charset,*: Refactor transstr()

9 files changed, 190 insertions, 130 deletions

diff --git a/src/nvim/buffer.c b/src/nvim/buffer.c
index 11c3b75d70..02d4cd30d4 100644
--- a/src/nvim/buffer.c
+++ b/src/nvim/buffer.c
@@ -2886,15 +2886,13 @@ static char_u *lasticon = NULL;
 
 void maketitle(void)
 {
-  char_u      *p;
   char_u      *t_str = NULL;
   char_u      *i_name;
   char_u      *i_str = NULL;
   int maxlen = 0;
   int len;
   int mustset;
-  char_u buf[IOSIZE];
-  int off;
+  char buf[IOSIZE];
 
   if (!redrawing()) {
     /* Postpone updating the title when 'lazyredraw' is set. */
@@ -2921,10 +2919,10 @@ void maketitle(void)
 
         use_sandbox = was_set_insecurely((char_u *)"titlestring", 0);
         called_emsg = FALSE;
-        build_stl_str_hl(curwin, buf, sizeof(buf),
-            p_titlestring, use_sandbox,
-            0, maxlen, NULL, NULL);
-        t_str = buf;
+        build_stl_str_hl(curwin, (char_u *)buf, sizeof(buf),
+                         p_titlestring, use_sandbox,
+                         0, maxlen, NULL, NULL);
+        t_str = (char_u *)buf;
         if (called_emsg) {
           set_string_option_direct((char_u *)"titlestring", -1, (char_u *)"",
                                    OPT_FREE, SID_ERROR);
@@ -2934,76 +2932,77 @@ void maketitle(void)
         t_str = p_titlestring;
       }
     } else {
-      /* format: "fname + (path) (1 of 2) - VIM" */
+      // Format: "fname + (path) (1 of 2) - VIM".
 
 #define SPACE_FOR_FNAME (sizeof(buf) - 100)
 #define SPACE_FOR_DIR   (sizeof(buf) - 20)
-#define SPACE_FOR_ARGNR (sizeof(buf) - 10)  // At least room for " - VIM"
+#define SPACE_FOR_ARGNR (sizeof(buf) - 10)  // At least room for " - VIM".
       if (curbuf->b_fname == NULL) {
-        STRLCPY(buf, _("[No Name]"), SPACE_FOR_FNAME + 1);
+        xstrlcpy(buf, _("[No Name]"), SPACE_FOR_FNAME + 1);
       } else {
-        p = transstr(path_tail(curbuf->b_fname));
-        STRLCPY(buf, p, SPACE_FOR_FNAME + 1);
-        xfree(p);
+        transstr_buf((const char *)path_tail(curbuf->b_fname),
+                     buf, SPACE_FOR_FNAME + 1);
       }
 
       switch (bufIsChanged(curbuf)
               | (curbuf->b_p_ro << 1)
               | (!MODIFIABLE(curbuf) << 2)) {
         case 0: break;
-        case 1: STRCAT(buf, " +"); break;
-        case 2: STRCAT(buf, " ="); break;
-        case 3: STRCAT(buf, " =+"); break;
+        case 1: strcat(buf, " +"); break;
+        case 2: strcat(buf, " ="); break;
+        case 3: strcat(buf, " =+"); break;
         case 4:
-        case 6: STRCAT(buf, " -"); break;
+        case 6: strcat(buf, " -"); break;
         case 5:
-        case 7: STRCAT(buf, " -+"); break;
+        case 7: strcat(buf, " -+"); break;
         default: assert(false);
       }
 
       if (curbuf->b_fname != NULL) {
         // Get path of file, replace home dir with ~.
-        off = (int)STRLEN(buf);
+        size_t off = strlen(buf);
         buf[off++] = ' ';
         buf[off++] = '(';
         home_replace(curbuf, curbuf->b_ffname,
-                     buf + off, (size_t)(SPACE_FOR_DIR - off), true);
+                     (char_u *)buf + off, (size_t)(SPACE_FOR_DIR - off), true);
 #ifdef BACKSLASH_IN_FILENAME
         /* avoid "c:/name" to be reduced to "c" */
         if (isalpha(buf[off]) && buf[off + 1] == ':')
           off += 2;
 #endif
-        /* remove the file name */
-        p = path_tail_with_sep(buf + off);
-        if (p == buf + off)
-          /* must be a help buffer */
-          STRLCPY(buf + off, _("help"), SPACE_FOR_DIR - off);
-        else
+        // Remove the file name.
+        char *p = (char *)path_tail_with_sep((char_u *)buf + off);
+        if (p == buf + off) {
+          // Must be a help buffer.
+          xstrlcpy(buf + off, _("help"), SPACE_FOR_DIR - off);
+        } else {
           *p = NUL;
+        }
 
-        /* Translate unprintable chars and concatenate.  Keep some
-         * room for the server name.  When there is no room (very long
-         * file name) use (...). */
+        // Translate unprintable chars and concatenate.  Keep some
+        // room for the server name.  When there is no room (very long
+        // file name) use (...).
         if (off < SPACE_FOR_DIR) {
           p = transstr(buf + off);
-          STRLCPY(buf + off, p, SPACE_FOR_DIR - off + 1);
+          xstrlcpy(buf + off, p, SPACE_FOR_DIR - off + 1);
           xfree(p);
         } else {
-          STRLCPY(buf + off, "...", SPACE_FOR_ARGNR - off + 1);
+          xstrlcpy(buf + off, "...", SPACE_FOR_ARGNR - off + 1);
         }
-        STRCAT(buf, ")");
+        strcat(buf, ")");
       }
 
-      append_arg_number(curwin, buf, SPACE_FOR_ARGNR, FALSE);
+      append_arg_number(curwin, (char_u *)buf, SPACE_FOR_ARGNR, false);
 
-      STRCAT(buf, " - NVIM");
+      strcat(buf, " - NVIM");
 
       if (maxlen > 0) {
-        /* make it shorter by removing a bit in the middle */
-        if (vim_strsize(buf) > maxlen)
-          trunc_string(buf, buf, maxlen, IOSIZE);
+        // Make it shorter by removing a bit in the middle.
+        if (vim_strsize((char_u *)buf) > maxlen) {
+          trunc_string((char_u *)buf, (char_u *)buf, maxlen, sizeof(buf));
+        }
       }
-      t_str = buf;
+      t_str = (char_u *)buf;
 #undef SPACE_FOR_FNAME
 #undef SPACE_FOR_DIR
 #undef SPACE_FOR_ARGNR
@@ -3012,7 +3011,7 @@ void maketitle(void)
   mustset = ti_change(t_str, &lasttitle);
 
   if (p_icon) {
-    i_str = buf;
+    i_str = (char_u *)buf;
     if (*p_iconstring != NUL) {
       if (stl_syntax & STL_IN_ICON) {
         int use_sandbox = FALSE;
diff --git a/src/nvim/charset.c b/src/nvim/charset.c
index 980b4ed426..2604c09ca4 100644
--- a/src/nvim/charset.c
+++ b/src/nvim/charset.c
@@ -313,69 +313,110 @@ void trans_characters(char_u *buf, int bufsize)
   }
 }
 
-/// Translate a string into allocated memory, replacing special chars with
-/// printable chars.
+/// Find length of a string capable of holding s with all specials replaced
 ///
-/// @param s
+/// Assumes replacing special characters with printable ones just like
+/// strtrans() does.
+///
+/// @param[in]  s  String to check.
 ///
-/// @return translated string
-char_u *transstr(char_u *s) FUNC_ATTR_NONNULL_RET
+/// @return number of bytes needed to hold a translation of `s`, NUL byte not
+///         included.
+size_t transstr_len(const char *const s)
+  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_PURE
 {
-  char_u *res;
-  char_u *p;
-  int c;
-  size_t l;
-  char_u hexbuf[11];
-
-  // Compute the length of the result, taking account of unprintable
-  // multi-byte characters.
+  const char *p = s;
   size_t len = 0;
-  p = s;
 
-  while (*p != NUL) {
-    if ((l = (size_t)(*mb_ptr2len)(p)) > 1) {
-      c = (*mb_ptr2char)(p);
-      p += l;
+  while (*p) {
+    const size_t l = (size_t)utfc_ptr2len((const char_u *)p);
+    if (l > 1) {
+      int pcc[MAX_MCO + 2];
+      pcc[0] = utfc_ptr2char((const char_u *)p, &pcc[1]);
 
-      if (vim_isprintc(c)) {
+      if (vim_isprintc(pcc[0])) {
         len += l;
       } else {
-        transchar_hex(hexbuf, c);
-        len += STRLEN(hexbuf);
+        for (size_t i = 0; i < ARRAY_SIZE(pcc); i++) {
+          char hexbuf[11];
+          len += transchar_hex(hexbuf, pcc[i]);
+        }
       }
+      p += l;
     } else {
-      l = (size_t)byte2cells(*p++);
-
-      if (l > 0) {
-        len += l;
-      } else {
-        // illegal byte sequence
-        len += 4;
-      }
+      const int b2c_l = byte2cells((uint8_t)(*p++));
+      // Illegal byte sequence may occupy up to 4 characters.
+      len += (size_t)(b2c_l > 0 ? b2c_l : 4);
     }
   }
-  res = xmallocz(len);
-
-  *res = NUL;
-  p = s;
+  return len;
+}
 
-  while (*p != NUL) {
-    if ((l = (size_t)(*mb_ptr2len)(p)) > 1) {
-      c = (*mb_ptr2char)(p);
+/// Replace special characters with printable ones
+///
+/// @param[in]  s  String to replace characters from.
+/// @param[out]  buf  Buffer to which result should be saved.
+/// @param[in]  len  Buffer length. Resulting string may not occupy more then
+///                  len - 1 bytes (one for trailing NUL byte).
+///
+/// @return `buf` argument unchanged.
+char *transstr_buf(const char *const s, char *const buf, const size_t len)
+  FUNC_ATTR_NONNULL_RET FUNC_ATTR_NONNULL_ALL
+{
+  const char *p = s;
+  char *buf_p = buf;
+  char *const buf_e = buf_p + len - 1;
+
+  while (*p != NUL && buf_p < buf_e) {
+    const size_t l = (size_t)utfc_ptr2len((const char_u *)p);
+    if (l > 1) {
+      if (buf_p + l >= buf_e) {
+        break;
+      }
+      int pcc[MAX_MCO + 2];
+      pcc[0] = utfc_ptr2char((const char_u *)p, &pcc[1]);
 
-      if (vim_isprintc(c)) {
-        // append printable multi-byte char
-        STRNCAT(res, p, l);
+      if (vim_isprintc(pcc[0])) {
+        memmove(buf_p, p, l);
+        buf_p += l;
       } else {
-        transchar_hex(res + STRLEN(res), c);
+        for (size_t i = 0; i < ARRAY_SIZE(pcc); i++) {
+          char hexbuf[11];
+          const size_t hexlen = transchar_hex(hexbuf, pcc[i]);
+          if (buf_p + hexlen >= buf_e) {
+            break;
+          }
+          memmove(buf_p, hexbuf, hexlen);
+          buf_p += hexlen;
+        }
       }
       p += l;
     } else {
-      STRCAT(res, transchar_byte(*p++));
+      const char *const tb = (const char *)transchar_byte((uint8_t)(*p++));
+      const size_t tb_len = strlen(tb);
+      memmove(buf_p, tb, tb_len);
+      buf_p += tb_len;
     }
   }
+  *buf_p = NUL;
+  assert(buf_p <= buf_e);
+  return buf;
+}
 
-  return res;
+/// Copy string and replace special characters with printable characters
+///
+/// Works like `strtrans()` does, used for that and in some other places.
+///
+/// @param[in]  s  String to replace characters from.
+///
+/// @return [allocated] translated string
+char *transstr(const char *const s)
+  FUNC_ATTR_NONNULL_RET
+{
+  // Compute the length of the result, taking account of unprintable
+  // multi-byte characters.
+  const size_t len = transstr_len((const char *)s) + 1;
+  return transstr_buf(s, xmalloc(len), len);
 }
 
 /// Convert the string "str[orglen]" to do ignore-case comparing.
@@ -504,13 +545,15 @@ char_u* transchar(int c)
   return transchar_buf;
 }
 
-/// Like transchar(), but called with a byte instead of a character.  Checks
-/// for an illegal UTF-8 byte.
+/// Like transchar(), but called with a byte instead of a character
 ///
-/// @param c
+/// Checks for an illegal UTF-8 byte.
+///
+/// @param[in]  c  Byte to translate.
 ///
 /// @return pointer to translated character in transchar_buf.
-char_u* transchar_byte(int c)
+char_u *transchar_byte(const int c)
+  FUNC_ATTR_WARN_UNUSED_RESULT
 {
   if (c >= 0x80) {
     transchar_nonprint(transchar_buf, c);
@@ -519,12 +562,14 @@ char_u* transchar_byte(int c)
   return transchar(c);
 }
 
-/// Convert non-printable character to two or more printable characters in
-/// "buf[]".  "buf" needs to be able to hold five bytes.
-/// Does NOT work for multi-byte characters, c must be <= 255.
+/// Convert non-printable characters to 2..4 printable ones
 ///
-/// @param buf
-/// @param c
+/// @warning Does not work for multi-byte characters, c must be <= 255.
+///
+/// @param[out]  buf  Buffer to store result in, must be able to hold at least
+///                   5 bytes (conversion result + NUL).
+/// @param[in]  c  Character to convert. NUL is assumed to be NL according to
+///                `:h NL-used-for-NUL`.
 void transchar_nonprint(char_u *buf, int c)
 {
   if (c == NL) {
@@ -534,54 +579,63 @@ void transchar_nonprint(char_u *buf, int c)
     // we use CR in place of  NL in this case
     c = NL;
   }
+  assert(c <= 0xff);
 
-  if (dy_flags & DY_UHEX) {
+  if (dy_flags & DY_UHEX || c > 0x7f) {
     // 'display' has "uhex"
-    transchar_hex(buf, c);
-  } else if (c <= 0x7f) {
+    transchar_hex((char *)buf, c);
+  } else {
     // 0x00 - 0x1f and 0x7f
     buf[0] = '^';
     // DEL displayed as ^?
     buf[1] = (char_u)(c ^ 0x40);
 
     buf[2] = NUL;
-  } else {
-    transchar_hex(buf, c);
   }
 }
 
-/// Convert a non-printable character to hex.
+/// Convert a non-printable character to hex C string like "<FFFF>"
 ///
-/// @param buf
-/// @param c
-void transchar_hex(char_u *buf, int c)
+/// @param[out]  buf  Buffer to store result in.
+/// @param[in]  c  Character to convert.
+///
+/// @return Number of bytes stored in buffer, excluding trailing NUL byte.
+size_t transchar_hex(char *const buf, const int c)
+  FUNC_ATTR_NONNULL_ALL
 {
-  int i = 0;
+  size_t i = 0;
 
-  buf[0] = '<';
+  buf[i++] = '<';
   if (c > 255) {
-    buf[++i] = (char_u)nr2hex((unsigned)c >> 12);
-    buf[++i] = (char_u)nr2hex((unsigned)c >> 8);
-  }
-  buf[++i] = (char_u)(nr2hex((unsigned)c >> 4));
-  buf[++i] = (char_u)(nr2hex((unsigned)c));
-  buf[++i] = '>';
-  buf[++i] = NUL;
+    if (c > 255 * 256) {
+      buf[i++] = (char)nr2hex((unsigned)c >> 20);
+      buf[i++] = (char)nr2hex((unsigned)c >> 16);
+    }
+    buf[i++] = (char)nr2hex((unsigned)c >> 12);
+    buf[i++] = (char)nr2hex((unsigned)c >> 8);
+  }
+  buf[i++] = (char)(nr2hex((unsigned)c >> 4));
+  buf[i++] = (char)(nr2hex((unsigned)c));
+  buf[i++] = '>';
+  buf[i] = NUL;
+  return i;
 }
 
-/// Convert the lower 4 bits of byte "c" to its hex character.
+/// Convert the lower 4 bits of byte "c" to its hex character
+///
 /// Lower case letters are used to avoid the confusion of <F1> being 0xf1 or
 /// function key 1.
 ///
-/// @param c
+/// @param[in]  n  Number to convert.
 ///
 /// @return the hex character.
-static unsigned nr2hex(unsigned c)
+static inline unsigned nr2hex(unsigned n)
+  FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT
 {
-  if ((c & 0xf) <= 9) {
-    return (c & 0xf) + '0';
+  if ((n & 0xf) <= 9) {
+    return (n & 0xf) + '0';
   }
-  return (c & 0xf) - 10 + 'a';
+  return (n & 0xf) - 10 + 'a';
 }
 
 /// Return number of display cells occupied by byte "b".
diff --git a/src/nvim/eval.c b/src/nvim/eval.c
index 7bdfe7c9ee..9a5b508cb4 100644
--- a/src/nvim/eval.c
+++ b/src/nvim/eval.c
@@ -16090,7 +16090,7 @@ static void f_strridx(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 static void f_strtrans(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 {
   rettv->v_type = VAR_STRING;
-  rettv->vval.v_string = transstr((char_u *)tv_get_string(&argvars[0]));
+  rettv->vval.v_string = (char_u *)transstr(tv_get_string(&argvars[0]));
 }
 
 /*
diff --git a/src/nvim/fold.c b/src/nvim/fold.c
index db88791967..ad9cd4d562 100644
--- a/src/nvim/fold.c
+++ b/src/nvim/fold.c
@@ -1775,7 +1775,7 @@ char_u *get_foldtext(win_T *wp, linenr_T lnum, linenr_T lnume,
           break;
       }
       if (*p != NUL) {
-        p = transstr(text);
+        p = (char_u *)transstr((const char *)text);
         xfree(text);
         text = p;
       }
diff --git a/src/nvim/globals.h b/src/nvim/globals.h
index 776f131437..400f46eb79 100644
--- a/src/nvim/globals.h
+++ b/src/nvim/globals.h
@@ -16,8 +16,6 @@
 
 #define IOSIZE         (1024+1)          // file I/O and sprintf buffer size
 
-#define MAX_MCO        6                 // maximum value for 'maxcombine'
-
 # define MSG_BUF_LEN 480                 // length of buffer for small messages
 # define MSG_BUF_CLEN  (MSG_BUF_LEN / 6) // cell length (worst case: utf-8
                                          // takes 6 bytes for one cell)
diff --git a/src/nvim/mbyte.c b/src/nvim/mbyte.c
index 008bce6df6..446063e33b 100644
--- a/src/nvim/mbyte.c
+++ b/src/nvim/mbyte.c
@@ -704,12 +704,14 @@ bool utf_composinglike(const char_u *p1, const char_u *p2)
   return arabic_combine(utf_ptr2char(p1), c2);
 }
 
-/*
- * Convert a UTF-8 byte string to a wide character. Also get up to MAX_MCO
- * composing characters.
- *
- * @param [out] pcc: composing chars, last one is 0
- */
+/// Convert a UTF-8 string to a wide character
+///
+/// Also gets up to #MAX_MCO composing characters.
+///
+/// @param[out]  pcc  Location where to store composing characters. Must have
+///                   space at least for #MAX_MCO + 1 elements.
+///
+/// @return leading character.
 int utfc_ptr2char(const char_u *p, int *pcc)
 {
   int len;
diff --git a/src/nvim/mbyte.h b/src/nvim/mbyte.h
index a5ce1b0a15..dd8e44b3f9 100644
--- a/src/nvim/mbyte.h
+++ b/src/nvim/mbyte.h
@@ -19,6 +19,13 @@
 #define MB_BYTE2LEN(b)         utf8len_tab[b]
 #define MB_BYTE2LEN_CHECK(b)   (((b) < 0 || (b) > 255) ? 1 : utf8len_tab[b])
 
+/// Maximum value for 'maxcombine'
+///
+/// At most that number of composing characters may be attached to the leading
+/// character by various `utfc_*` functions. Note that some functions do not
+/// have this limit.
+enum { MAX_MCO = 6 };
+
 // max length of an unicode char
 #define MB_MAXCHAR     6
 
diff --git a/src/nvim/popupmnu.c b/src/nvim/popupmnu.c
index 348daf028a..4c2fc6d906 100644
--- a/src/nvim/popupmnu.c
+++ b/src/nvim/popupmnu.c
@@ -387,7 +387,7 @@ void pum_redraw(void)
             char_u saved = *p;
 
             *p = NUL;
-            st = transstr(s);
+            st = (char_u *)transstr((const char *)s);
             *p = saved;
 
             if (curwin->w_p_rl) {
diff --git a/src/nvim/screen.c b/src/nvim/screen.c
index 05ad126fa0..e232939403 100644
--- a/src/nvim/screen.c
+++ b/src/nvim/screen.c
@@ -3213,9 +3213,9 @@ win_line (
               || (mb_l > 1 && (!vim_isprintc(mb_c)))) {
             // Illegal UTF-8 byte: display as <xx>.
             // Non-BMP character : display as ? or fullwidth ?.
-            transchar_hex(extra, mb_c);
-            if (wp->w_p_rl) {                         // reverse
-                rl_mirror(extra);
+            transchar_hex((char *)extra, mb_c);
+            if (wp->w_p_rl) {  // reverse
+              rl_mirror(extra);
             }
 
             p_extra = extra;
@@ -5232,8 +5232,8 @@ win_redr_custom (
   xfree(stl);
   ewp->w_p_crb = p_crb_save;
 
-  /* Make all characters printable. */
-  p = transstr(buf);
+  // Make all characters printable.
+  p = (char_u *)transstr((const char *)buf);
   len = STRLCPY(buf, p, sizeof(buf));
   len = (size_t)len < sizeof(buf) ? len : (int)sizeof(buf) - 1;
   xfree(p);