feat(decode_string): decode binary string with NULs to Blob

Strings that previously decoded into a msgpack special for representing
BINs with NULs now convert to Blobs. It shouldn't be possible to decode
into this special anymore after this change?

Notably, Lua strings with NULs now convert to Blobs when passed to VimL.

6 files changed, 46 insertions, 42 deletions

diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index e7820f5313..70f970ffef 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -6750,9 +6750,9 @@ msgpackparse({list})					*msgpackparse()*
 			zero byte or if string is a mapping key and mapping is
 			being represented as special dictionary for other
 			reasons.
-		binary	|readfile()|-style list of strings. This value will
-			appear in |msgpackparse()| output if binary string
-			contains zero byte.
+		binary	|String|, or |Blob| if binary string contains zero
+			byte. This value cannot appear in |msgpackparse()|
+			output since blobs were introduced.
 		array	|List|. This value cannot appear in |msgpackparse()|
 			output.
 							*msgpack-special-map*
diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
index 032c28c659..53d68fa5e6 100644
--- a/runtime/doc/lua.txt
+++ b/runtime/doc/lua.txt
@@ -326,7 +326,8 @@ semantically equivalent in Lua to:
     end
 
 Lua nils, numbers, strings, tables and booleans are converted to their
-respective Vimscript types. Conversion of other Lua types is an error.
+respective Vimscript types. If a Lua string contains a NUL byte, it will be
+converted to a |Blob|. Conversion of other Lua types is an error.
 
 The magic global "_A" contains the second argument to luaeval().
 
diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c
index 89e1f04bfd..5c03f55621 100644
--- a/src/nvim/eval/decode.c
+++ b/src/nvim/eval/decode.c
@@ -246,14 +246,15 @@ list_T *decode_create_map_special_dict(typval_T *const ret_tv,
 /// Convert char* string to typval_T
 ///
 /// Depending on whether string has (no) NUL bytes, it may use a special
-/// dictionary or decode string to VAR_STRING.
+/// dictionary, VAR_BLOB, or decode string to VAR_STRING.
 ///
 /// @param[in]  s  String to decode.
 /// @param[in]  len  String length.
 /// @param[in]  hasnul  Whether string has NUL byte, not or it was not yet
 ///                     determined.
-/// @param[in]  binary  If true, save special string type as kMPBinary,
-///                     otherwise kMPString.
+/// @param[in]  binary  Determines decode type if string has NUL bytes.
+///                     If true convert string to VAR_BLOB, otherwise to the
+///                     kMPString special type.
 /// @param[in]  s_allocated  If true, then `s` was allocated and can be saved in
 ///                          a returned structure. If it is not saved there, it
 ///                          will be freed.
@@ -269,21 +270,28 @@ typval_T decode_string(const char *const s, const size_t len,
                               ? ((s != NULL) && (memchr(s, NUL, len) != NULL))
                               : (bool)hasnul);
   if (really_hasnul) {
-    list_T *const list = tv_list_alloc(kListLenMayKnow);
-    tv_list_ref(list);
     typval_T tv;
-    create_special_dict(&tv, binary ? kMPBinary : kMPString, ((typval_T) {
-      .v_type = VAR_LIST,
-      .v_lock = VAR_UNLOCKED,
-      .vval = { .v_list = list },
-    }));
-    const int elw_ret = encode_list_write((void *)list, s, len);
-    if (s_allocated) {
-      xfree((void *)s);
-    }
-    if (elw_ret == -1) {
-      tv_clear(&tv);
-      return (typval_T) { .v_type = VAR_UNKNOWN, .v_lock = VAR_UNLOCKED };
+    tv.v_lock = VAR_UNLOCKED;
+    if (binary) {
+      tv_blob_alloc_ret(&tv);
+      ga_concat_len(&tv.vval.v_blob->bv_ga, s, len);
+    } else {
+      list_T *const list = tv_list_alloc(kListLenMayKnow);
+      tv_list_ref(list);
+      create_special_dict(&tv, kMPString,
+                          ((typval_T){
+                              .v_type = VAR_LIST,
+                              .v_lock = VAR_UNLOCKED,
+                              .vval = { .v_list = list },
+                          }));
+      const int elw_ret = encode_list_write((void *)list, s, len);
+      if (s_allocated) {
+        xfree((void *)s);
+      }
+      if (elw_ret == -1) {
+        tv_clear(&tv);
+        return (typval_T) { .v_type = VAR_UNKNOWN, .v_lock = VAR_UNLOCKED };
+      }
     }
     return tv;
   } else {
diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua
index 90e3ccc0e4..d95cea57e4 100644
--- a/test/functional/eval/msgpack_functions_spec.lua
+++ b/test/functional/eval/msgpack_functions_spec.lua
@@ -364,8 +364,7 @@ describe('msgpack*() functions', function()
     command('let dumped = ["\\xC4\\x01\\n"]')
     command('let parsed = msgpackparse(dumped)')
     command('let dumped2 = msgpackdump(parsed)')
-    eq({{_TYPE={}, _VAL={'\n'}}}, eval('parsed'))
-    eq(1, eval('parsed[0]._TYPE is v:msgpack_types.binary'))
+    eq({'\000'}, eval('parsed'))
     eq(1, eval('dumped ==# dumped2'))
   end)
 
diff --git a/test/functional/lua/api_spec.lua b/test/functional/lua/api_spec.lua
index b2c343096a..fdf79d55b2 100644
--- a/test/functional/lua/api_spec.lua
+++ b/test/functional/lua/api_spec.lua
@@ -15,7 +15,7 @@ describe('luaeval(vim.api.…)', function()
     describe('nvim_buf_get_lines', function()
       it('works', function()
         funcs.setline(1, {"abc", "def", "a\nb", "ttt"})
-        eq({{_TYPE={}, _VAL={'a\nb'}}},
+        eq({'a\000b'},
            funcs.luaeval('vim.api.nvim_buf_get_lines(1, 2, 3, false)'))
       end)
     end)
@@ -23,7 +23,7 @@ describe('luaeval(vim.api.…)', function()
       it('works', function()
         funcs.setline(1, {"abc", "def", "a\nb", "ttt"})
         eq(NIL, funcs.luaeval('vim.api.nvim_buf_set_lines(1, 1, 2, false, {"b\\0a"})'))
-        eq({'abc', {_TYPE={}, _VAL={'b\na'}}, {_TYPE={}, _VAL={'a\nb'}}, 'ttt'},
+        eq({'abc', 'b\000a', 'a\000b', 'ttt'},
            funcs.luaeval('vim.api.nvim_buf_get_lines(1, 0, 4, false)'))
       end)
     end)
@@ -68,6 +68,7 @@ describe('luaeval(vim.api.…)', function()
     eq({}, funcs.luaeval('vim.api.nvim_eval("[]")'))
     eq({}, funcs.luaeval('vim.api.nvim_eval("{}")'))
     eq(1, funcs.luaeval('vim.api.nvim_eval("1.0")'))
+    eq('\000', funcs.luaeval('vim.api.nvim_eval("0z00")'))
     eq(true, funcs.luaeval('vim.api.nvim_eval("v:true")'))
     eq(false, funcs.luaeval('vim.api.nvim_eval("v:false")'))
     eq(NIL, funcs.luaeval('vim.api.nvim_eval("v:null")'))
diff --git a/test/functional/lua/luaeval_spec.lua b/test/functional/lua/luaeval_spec.lua
index 36f6d4025f..255e99032f 100644
--- a/test/functional/lua/luaeval_spec.lua
+++ b/test/functional/lua/luaeval_spec.lua
@@ -63,11 +63,10 @@ describe('luaeval()', function()
       eq('\n[[...@0]]', funcs.execute('echo luaeval("l")'))
     end)
   end)
-  describe('strings', function()
-    it('are successfully converted to special dictionaries', function()
+  describe('strings with NULs', function()
+    it('are successfully converted to blobs', function()
       command([[let s = luaeval('"\0"')]])
-      eq({_TYPE={}, _VAL={'\n'}}, meths.get_var('s'))
-      eq(1, funcs.eval('s._TYPE is v:msgpack_types.binary'))
+      eq('\000', meths.get_var('s'))
     end)
     it('are successfully converted to special dictionaries in table keys',
     function()
@@ -76,13 +75,10 @@ describe('luaeval()', function()
       eq(1, funcs.eval('d._TYPE is v:msgpack_types.map'))
       eq(1, funcs.eval('d._VAL[0][0]._TYPE is v:msgpack_types.string'))
     end)
-    it('are successfully converted to special dictionaries from a list',
+    it('are successfully converted to blobs from a list',
     function()
       command([[let l = luaeval('{"abc", "a\0b", "c\0d", "def"}')]])
-      eq({'abc', {_TYPE={}, _VAL={'a\nb'}}, {_TYPE={}, _VAL={'c\nd'}}, 'def'},
-         meths.get_var('l'))
-      eq(1, funcs.eval('l[1]._TYPE is v:msgpack_types.binary'))
-      eq(1, funcs.eval('l[2]._TYPE is v:msgpack_types.binary'))
+      eq({'abc', 'a\000b', 'c\000d', 'def'}, meths.get_var('l'))
     end)
   end)
 
@@ -100,9 +96,9 @@ describe('luaeval()', function()
     eq(1, eval('type(luaeval("\'test\'"))'))
 
     eq('', funcs.luaeval('""'))
-    eq({_TYPE={}, _VAL={'\n'}}, funcs.luaeval([['\0']]))
-    eq({_TYPE={}, _VAL={'\n', '\n'}}, funcs.luaeval([['\0\n\0']]))
-    eq(1, eval([[luaeval('"\0\n\0"')._TYPE is v:msgpack_types.binary]]))
+    eq('\000', funcs.luaeval([['\0']]))
+    eq('\000\n\000', funcs.luaeval([['\0\n\0']]))
+    eq(10, eval([[type(luaeval("'\\0\\n\\0'"))]]))
 
     eq(true, funcs.luaeval('true'))
     eq(false, funcs.luaeval('false'))
@@ -122,12 +118,11 @@ describe('luaeval()', function()
     local level = 30
     eq(nested_by_level[level].o, funcs.luaeval(nested_by_level[level].s))
 
-    eq({_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, {_TYPE={}, _VAL={'\n', '\n\n'}}}}},
+    eq({_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, '\000\n\000\000'}}},
        funcs.luaeval([[{['\0\n\0']='\0\n\0\0'}]]))
     eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._TYPE is v:msgpack_types.map]]))
     eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._VAL[0][0]._TYPE is v:msgpack_types.string]]))
-    eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._VAL[0][1]._TYPE is v:msgpack_types.binary]]))
-    eq({nested={{_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, {_TYPE={}, _VAL={'\n', '\n\n'}}}}}}},
+    eq({nested={{_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, '\000\n\000\000'}}}}},
        funcs.luaeval([[{nested={{['\0\n\0']='\0\n\0\0'}}}]]))
   end)
 
@@ -175,8 +170,8 @@ describe('luaeval()', function()
   end
 
   it('correctly passes special dictionaries', function()
-    eq({'binary', {'\n', '\n'}}, luaevalarg(sp('binary', '["\\n", "\\n"]')))
-    eq({'binary', {'\n', '\n'}}, luaevalarg(sp('string', '["\\n", "\\n"]')))
+    eq({0, '\000\n\000'}, luaevalarg(sp('binary', '["\\n", "\\n"]')))
+    eq({0, '\000\n\000'}, luaevalarg(sp('string', '["\\n", "\\n"]')))
     eq({0, true}, luaevalarg(sp('boolean', 1)))
     eq({0, false}, luaevalarg(sp('boolean', 0)))
     eq({0, NIL}, luaevalarg(sp('nil', 0)))