vim-patch:8.2.3522: cannot use \x and \u when setting 'listchars' (#16049)

Problem:    Cannot use \x and \u when setting 'listchars'.
Solution:   Support hex and unicode in hex form. (closes vim/vim#9006)
https://github.com/vim/vim/commit/93ff6720fe4427341bc426b6d46e6324f226c270

3 files changed, 54 insertions, 15 deletions

diff --git a/src/nvim/charset.c b/src/nvim/charset.c
index 5e18f9d86e..eb0903b594 100644
--- a/src/nvim/charset.c
+++ b/src/nvim/charset.c
@@ -1642,6 +1642,16 @@ int hex2nr(int c)
   return c - '0';
 }
 
+/// Convert two hex characters to a byte.
+/// Return -1 if one of the characters is not hex.
+int hexhex2nr(char_u *p)
+{
+  if (!ascii_isxdigit(p[0]) || !ascii_isxdigit(p[1])) {
+    return -1;
+  }
+  return (hex2nr(p[0]) << 4) + hex2nr(p[1]);
+}
+
 /// Check that "str" starts with a backslash that should be removed.
 /// For Windows this is only done when the character after the
 /// backslash is not a normal file name character.
diff --git a/src/nvim/option.c b/src/nvim/option.c
index 5dd7d708f9..b85649a5cc 100644
--- a/src/nvim/option.c
+++ b/src/nvim/option.c
@@ -3422,6 +3422,37 @@ void check_blending(win_T *wp)
     wp->w_p_winbl > 0 || (wp->w_floating && wp->w_float_config.shadow);
 }
 
+/// Calls mb_cptr2char_adv(p) and returns the character.
+/// If "p" starts with "\x", "\u" or "\U" the hex or unicode value is used.
+/// Returns 0 for invalid hex or invalid UTF-8 byte.
+static int get_encoded_char_adv(char_u **p)
+{
+  char_u *s = *p;
+
+  if (s[0] == '\\' && (s[1] == 'x' || s[1] == 'u' || s[1] == 'U')) {
+    int64_t num = 0;
+    int bytes;
+    int n;
+    for (bytes = s[1] == 'x' ? 1 : s[1] == 'u' ? 2 : 4; bytes > 0; bytes--) {
+      *p += 2;
+      n = hexhex2nr(*p);
+      if (n < 0) {
+        return 0;
+      }
+      num = num * 256 + n;
+    }
+    *p += 2;
+    return (int)num;
+  }
+
+  // TODO(bfredl): use schar_T representation and utfc_ptr2len
+  int clen = utf_ptr2len(s);
+  int c = mb_cptr2char_adv((const char_u **)p);
+  if (clen == 1 && c > 127) {  // Invalid UTF-8 byte
+    return 0;
+  }
+  return c;
+}
 
 /// Handle setting 'listchars' or 'fillchars'.
 /// Assume monocell characters
@@ -3526,26 +3557,21 @@ static char *set_chars_option(win_T *wp, char_u **varp, bool set)
             && p[len + 1] != NUL) {
           c2 = c3 = 0;
           s = p + len + 1;
-
-          // TODO(bfredl): use schar_T representation and utfc_ptr2len
-          int c1len = utf_ptr2len(s);
-          c1 = mb_cptr2char_adv((const char_u **)&s);
-          if (utf_char2cells(c1) > 1 || (c1len == 1 && c1 > 127)) {
+          c1 = get_encoded_char_adv(&s);
+          if (c1 == 0 || utf_char2cells(c1) > 1) {
             return e_invarg;
           }
           if (tab[i].cp == &wp->w_p_lcs_chars.tab2) {
             if (*s == NUL) {
               return e_invarg;
             }
-            int c2len = utf_ptr2len(s);
-            c2 = mb_cptr2char_adv((const char_u **)&s);
-            if (utf_char2cells(c2) > 1 || (c2len == 1 && c2 > 127)) {
+            c2 = get_encoded_char_adv(&s);
+            if (c2 == 0 || utf_char2cells(c2) > 1) {
               return e_invarg;
             }
             if (!(*s == ',' || *s == NUL)) {
-              int c3len = utf_ptr2len(s);
-              c3 = mb_cptr2char_adv((const char_u **)&s);
-              if (utf_char2cells(c3) > 1 || (c3len == 1 && c3 > 127)) {
+              c3 = get_encoded_char_adv(&s);
+              if (c3 == 0 || utf_char2cells(c3) > 1) {
                 return e_invarg;
               }
             }
@@ -3578,9 +3604,8 @@ static char *set_chars_option(win_T *wp, char_u **varp, bool set)
             last_multispace = p;
             multispace_len = 0;
             while (*s != NUL && *s != ',') {
-              int c1len = utf_ptr2len(s);
-              c1 = mb_cptr2char_adv((const char_u **)&s);
-              if (utf_char2cells(c1) > 1 || (c1len == 1 && c1 > 127)) {
+              c1 = get_encoded_char_adv(&s);
+              if (c1 == 0 || utf_char2cells(c1) > 1) {
                 return e_invarg;
               }
               multispace_len++;
@@ -3593,7 +3618,7 @@ static char *set_chars_option(win_T *wp, char_u **varp, bool set)
           } else {
             int multispace_pos = 0;
             while (*s != NUL && *s != ',') {
-              c1 = mb_cptr2char_adv((const char_u **)&s);
+              c1 = get_encoded_char_adv(&s);
               if (p == last_multispace) {
                 wp->w_p_lcs_chars.multispace[multispace_pos++] = c1;
               }
diff --git a/src/nvim/testdir/test_listchars.vim b/src/nvim/testdir/test_listchars.vim
index 8a1393a45d..751be8eff5 100644
--- a/src/nvim/testdir/test_listchars.vim
+++ b/src/nvim/testdir/test_listchars.vim
@@ -286,6 +286,10 @@ func Test_listchars_unicode()
   call cursor(1, 1)
   call assert_equal(expected, ScreenLines(1, virtcol('$')))
 
+  set listchars=eol:\\u21d4,space:\\u2423,multispace:≡\\u2262\\U00002263,nbsp:\\U00002260,tab:←↔\\u2192
+  redraw!
+  call assert_equal(expected, ScreenLines(1, virtcol('$')))
+
   set listchars+=lead:⇨,trail:⇦
   let expected = ['⇨⇨⇨⇨⇨⇨⇨⇨a←↔↔↔↔↔→b␣c≠d⇦⇦⇔']
   redraw!