transstr_buf: fix length comparison #8681

closes #8466
closes #8664
Regression by 0d7daaad98d5.

- Fix length comparison.
- Fix loop(s) which iterated over all fields of array `pcc` even if it
  was not filled up (try unicode 0x9f as statusline character).

Note about the tests:
- To input unicode with more than two hex digits you can use <C-v>U...:
  a + U+fe20: a︠
  a + U+fe20 + U+fe21: a︠︡

2 files changed, 75 insertions, 9 deletions

diff --git a/src/nvim/charset.c b/src/nvim/charset.c
index ab20996df7..a02d2a812d 100644
--- a/src/nvim/charset.c
+++ b/src/nvim/charset.c
@@ -331,14 +331,14 @@ size_t transstr_len(const char *const s)
   while (*p) {
     const size_t l = (size_t)utfc_ptr2len((const char_u *)p);
     if (l > 1) {
-      int pcc[MAX_MCO + 2];
+      int pcc[MAX_MCO + 1];
       pcc[0] = utfc_ptr2char((const char_u *)p, &pcc[1]);
 
       if (vim_isprintc(pcc[0])) {
         len += l;
       } else {
-        for (size_t i = 0; i < ARRAY_SIZE(pcc); i++) {
-          char hexbuf[11];
+        for (size_t i = 0; i < ARRAY_SIZE(pcc) && pcc[i]; i++) {
+          char hexbuf[9];
           len += transchar_hex(hexbuf, pcc[i]);
         }
       }
@@ -370,20 +370,20 @@ size_t transstr_buf(const char *const s, char *const buf, const size_t len)
   while (*p != NUL && buf_p < buf_e) {
     const size_t l = (size_t)utfc_ptr2len((const char_u *)p);
     if (l > 1) {
-      if (buf_p + l >= buf_e) {
-        break;
+      if (buf_p + l > buf_e) {
+        break;  // Exceeded `buf` size.
       }
-      int pcc[MAX_MCO + 2];
+      int pcc[MAX_MCO + 1];
       pcc[0] = utfc_ptr2char((const char_u *)p, &pcc[1]);
 
       if (vim_isprintc(pcc[0])) {
         memmove(buf_p, p, l);
         buf_p += l;
       } else {
-        for (size_t i = 0; i < ARRAY_SIZE(pcc); i++) {
-          char hexbuf[11];
+        for (size_t i = 0; i < ARRAY_SIZE(pcc) && pcc[i]; i++) {
+          char hexbuf[9];  // <up to 6 bytes>NUL
           const size_t hexlen = transchar_hex(hexbuf, pcc[i]);
-          if (buf_p + hexlen >= buf_e) {
+          if (buf_p + hexlen > buf_e) {
             break;
           }
           memmove(buf_p, hexbuf, hexlen);
@@ -394,6 +394,9 @@ size_t transstr_buf(const char *const s, char *const buf, const size_t len)
     } else {
       const char *const tb = (const char *)transchar_byte((uint8_t)(*p++));
       const size_t tb_len = strlen(tb);
+      if (buf_p + tb_len > buf_e) {
+        break;  // Exceeded `buf` size.
+      }
       memmove(buf_p, tb, tb_len);
       buf_p += tb_len;
     }
diff --git a/test/functional/ui/multibyte_spec.lua b/test/functional/ui/multibyte_spec.lua
index 278480ec4f..3e63353ad2 100644
--- a/test/functional/ui/multibyte_spec.lua
+++ b/test/functional/ui/multibyte_spec.lua
@@ -1,6 +1,7 @@
 local helpers = require('test.functional.helpers')(after_each)
 local Screen = require('test.functional.ui.screen')
 local clear = helpers.clear
+local command = helpers.command
 local feed = helpers.feed
 local feed_command = helpers.feed_command
 local insert = helpers.insert
@@ -120,3 +121,65 @@ describe("multibyte rendering", function()
   end)
 end)
 
+describe('multibyte rendering: statusline', function()
+  local screen
+
+  before_each(function()
+    clear()
+    screen = Screen.new(40, 4)
+    screen:attach()
+    command('set laststatus=2')
+  end)
+
+  after_each(function()
+    screen:detach()
+  end)
+
+  it('last char shows (multibyte)', function()
+    command('set statusline=你好')
+    screen:expect([[
+    ^                                        |
+    ~                                       |
+    你好                                    |
+                                            |
+    ]])
+  end)
+  it('last char shows (single byte)', function()
+    command('set statusline=abc')
+    screen:expect([[
+    ^                                        |
+    ~                                       |
+    abc                                     |
+                                            |
+    ]])
+  end)
+  it('unicode control points', function()
+    command('set statusline=Ÿ')
+    screen:expect([[
+    ^                                        |
+    ~                                       |
+    <9f>                                    |
+                                            |
+    ]])
+  end)
+  it('MAX_MCO (6) unicode combination points', function()
+    command('set statusline=o̸⃯ᷰ⃐⃧⃝')
+    -- o + U+1DF0 + U+20EF + U+0338 + U+20D0 + U+20E7 + U+20DD
+    screen:expect([[
+    ^                                        |
+    ~                                       |
+    o̸⃯ᷰ⃐⃧⃝                                       |
+                                            |
+    ]])
+  end)
+  it('non-printable followed by MAX_MCO unicode combination points', function()
+    command('set statusline=Ÿ̸⃯ᷰ⃐⃧⃝')
+    -- U+9F + U+1DF0 + U+20EF + U+0338 + U+20D0 + U+20E7 + U+20DD
+    screen:expect([[
+    ^                                        |
+    ~                                       |
+    <9f><1df0><20ef><0338><20d0><20e7><20dd>|
+                                            |
+    ]])
+  end)
+end)