eval/decode: Parse strings with NUL to special dictionaries

2 files changed, 100 insertions, 10 deletions

diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c
index 5a3c5709ad..8c95b34326 100644
--- a/src/nvim/eval/decode.c
+++ b/src/nvim/eval/decode.c
@@ -342,6 +342,7 @@ int json_decode_string(const char *const buf, const size_t len,
         char *str = xmalloc(len + 1);
         int fst_in_pair = 0;
         char *str_end = str;
+        bool hasnul = false;
         for (const char *t = s; t < p; t++) {
           if (t[0] != '\\' || t[1] != 'u') {
             if (fst_in_pair != 0) {
@@ -357,6 +358,9 @@ int json_decode_string(const char *const buf, const size_t len,
                 t += 4;
                 unsigned long ch;
                 vim_str2nr((char_u *) ubuf, NULL, NULL, 0, 0, 2, NULL, &ch);
+                if (ch == 0) {
+                  hasnul = true;
+                }
                 if (SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) {
                   fst_in_pair = (int) ch;
                 } else if (SURROGATE_LO_START <= ch && ch <= SURROGATE_LO_END
@@ -405,9 +409,9 @@ int json_decode_string(const char *const buf, const size_t len,
           str_end += utf_char2bytes((int) fst_in_pair, (char_u *) str_end);
         }
         if (conv.vc_type != CONV_NONE) {
-          size_t len = (size_t) (str_end - str);
+          size_t str_len = (size_t) (str_end - str);
           char *const new_str = (char *) string_convert(&conv, (char_u *) str,
-                                                        &len);
+                                                        &str_len);
           if (new_str == NULL) {
             EMSG2(_("E474: Failed to convert string \"%s\" from UTF-8"), str);
             xfree(str);
@@ -415,14 +419,31 @@ int json_decode_string(const char *const buf, const size_t len,
           }
           xfree(str);
           str = new_str;
-          str_end = new_str + len;
+          str_end = new_str + str_len;
+        }
+        if (hasnul) {
+          typval_T obj;
+          list_T *const list = list_alloc();
+          list->lv_refcount++;
+          create_special_dict(&obj, kMPString, ((typval_T) {
+            .v_type = VAR_LIST,
+            .v_lock = 0,
+            .vval = { .v_list = list },
+          }));
+          if (encode_list_write((void *) list, str, (size_t) (str_end - str))
+              == -1) {
+            clear_tv(&obj);
+            goto json_decode_string_fail;
+          }
+          POP(obj);
+        } else {
+          *str_end = NUL;
+          // TODO: return special string in case of NUL bytes
+          POP(((typval_T) {
+            .v_type = VAR_STRING,
+            .vval = { .v_string = (char_u *) str, },
+          }));
         }
-        *str_end = NUL;
-        // TODO: return special string in case of NUL bytes
-        POP(((typval_T) {
-          .v_type = VAR_STRING,
-          .vval = { .v_string = (char_u *) str, },
-        }));
         break;
       }
       case '-':
diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua
index 651f6c27b3..7916bc829c 100644
--- a/test/functional/eval/json_functions_spec.lua
+++ b/test/functional/eval/json_functions_spec.lua
@@ -1,13 +1,63 @@
 local helpers = require('test.functional.helpers')
 local clear = helpers.clear
 local funcs = helpers.funcs
+local meths = helpers.meths
 local eq = helpers.eq
 local eval = helpers.eval
 local execute = helpers.execute
 local exc_exec = helpers.exc_exec
 
 describe('jsondecode() function', function()
-  before_each(clear)
+  before_each(function()
+    clear()
+    execute([[
+      function Eq(exp, act)
+        let act = a:act
+        let exp = a:exp
+        if type(exp) != type(act)
+          return 0
+        endif
+        if type(exp) == type({})
+          if sort(keys(exp)) !=# sort(keys(act))
+            return 0
+          endif
+          if sort(keys(exp)) ==# ['_TYPE', '_VAL']
+            let exp_typ = v:msgpack_types[exp._TYPE]
+            let act_typ = act._TYPE
+            if exp_typ isnot act_typ
+              return 0
+            endif
+            return Eq(exp._VAL, act._VAL)
+          else
+            return empty(filter(copy(exp), '!Eq(v:val, act[v:key])'))
+          endif
+        else
+          if type(exp) == type([])
+            if len(exp) != len(act)
+              return 0
+            endif
+            return empty(filter(copy(exp), '!Eq(v:val, act[v:key])'))
+          endif
+          return exp ==# act
+        endif
+        return 1
+      endfunction
+    ]])
+    execute([[
+      function EvalEq(exp, act_expr)
+        let act = eval(a:act_expr)
+        if Eq(a:exp, act)
+          return 1
+        else
+          return string(act)
+        endif
+      endfunction
+    ]])
+  end)
+
+  local speq = function(expected, actual_expr)
+    eq(1, funcs.EvalEq(expected, actual_expr))
+  end
 
   it('accepts readfile()-style list', function()
     eq({Test=1}, funcs.jsondecode({
@@ -221,6 +271,14 @@ describe('jsondecode() function', function()
        exc_exec('call jsondecode("\\t\\"abc\\\\u00")'))
     eq('Vim(call):E474: Unfinished unicode escape sequence: \t"abc\\u000',
        exc_exec('call jsondecode("\\t\\"abc\\\\u000")'))
+    eq('Vim(call):E474: Expected four hex digits after \\u: \\u"    ',
+       exc_exec('call jsondecode("\\t\\"abc\\\\u\\"    ")'))
+    eq('Vim(call):E474: Expected four hex digits after \\u: \\u0"    ',
+       exc_exec('call jsondecode("\\t\\"abc\\\\u0\\"    ")'))
+    eq('Vim(call):E474: Expected four hex digits after \\u: \\u00"    ',
+       exc_exec('call jsondecode("\\t\\"abc\\\\u00\\"    ")'))
+    eq('Vim(call):E474: Expected four hex digits after \\u: \\u000"    ',
+       exc_exec('call jsondecode("\\t\\"abc\\\\u000\\"    ")'))
     eq('Vim(call):E474: Expected string end: \t"abc\\u0000',
        exc_exec('call jsondecode("\\t\\"abc\\\\u0000")'))
   end)
@@ -315,6 +373,17 @@ describe('jsondecode() function', function()
     eq('a\xED\xB0\x80', funcs.jsondecode('"a\\uDC00"'))
     eq('\t\xED\xB0\x80', funcs.jsondecode('"\\t\\uDC00"'))
   end)
+
+  local sp_decode_eq = function(expected, json)
+    meths.set_var('__json', json)
+    speq(expected, 'jsondecode(g:__json)')
+    execute('unlet! g:__json')
+  end
+
+  it('parses strings with NUL properly', function()
+    sp_decode_eq({_TYPE='string', _VAL={'\n'}}, '"\\u0000"')
+    sp_decode_eq({_TYPE='string', _VAL={'\n', '\n'}}, '"\\u0000\\n\\u0000"')
+  end)
 end)
 
 describe('jsonencode() function', function()