feat(f_msgpackparse): support parsing from Blob

Note that it is not possible for msgpack_unpack_next() and
msgpack_unpacker_next() to return MSGPACK_UNPACK_EXTRA_BYTES, so it
should be fine to abort() on that.

Lua 5.1 doesn't support string hex escapes (\xXX) like VimL does (though
LuaJIT does), so convert them to decimal escapes (\DDD) in tests.

3 files changed, 137 insertions, 73 deletions

diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index 013bff3d20..93826660b1 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -2535,7 +2535,7 @@ mkdir({name} [, {path} [, {prot}]])
 				Number	create directory {name}
 mode([expr])			String	current editing mode
 msgpackdump({list} [, {type}])	List/Blob  dump objects to msgpack
-msgpackparse({list})		List	parse msgpack to a list of objects
+msgpackparse({data})		List	parse msgpack to a list of objects
 nextnonblank({lnum})		Number	line nr of non-blank line >= {lnum}
 nr2char({expr}[, {utf8}])	String	single char with ASCII/UTF8 value {expr}
 nvim_...({args}...)		any	call nvim |api| functions
@@ -6843,8 +6843,9 @@ msgpackdump({list} [, {type}])				*msgpackdump()*
 		4. Other strings and |Blob|s are always dumped as BIN strings.
 		5. Points 3. and 4. do not apply to |msgpack-special-dict|s.
 
-msgpackparse({list})					*msgpackparse()*
-		Convert a |readfile()|-style list to a list of VimL objects.
+msgpackparse({data})					*msgpackparse()*
+		Convert a |readfile()|-style list or a |Blob| to a list of
+		VimL objects.
 		Example: >
 			let fname = expand('~/.config/nvim/shada/main.shada')
 			let mpack = readfile(fname, 'b')
diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c
index a2ea74b19b..fda33fdfdd 100644
--- a/src/nvim/eval/funcs.c
+++ b/src/nvim/eval/funcs.c
@@ -6530,16 +6530,43 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv, FunPtr fptr)
   msgpack_packer_free(packer);
 }
 
-/// "msgpackparse" function
-static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr)
+static int msgpackparse_convert_item(const msgpack_object data,
+                                     const msgpack_unpack_return result,
+                                     list_T *const ret_list,
+                                     const bool fail_if_incomplete)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (argvars[0].v_type != VAR_LIST) {
-    EMSG2(_(e_listarg), "msgpackparse()");
-    return;
+  switch (result) {
+    case MSGPACK_UNPACK_PARSE_ERROR:
+      EMSG2(_(e_invarg2), "Failed to parse msgpack string");
+      return FAIL;
+    case MSGPACK_UNPACK_NOMEM_ERROR:
+      EMSG(_(e_outofmem));
+      return FAIL;
+    case MSGPACK_UNPACK_CONTINUE:
+      if (fail_if_incomplete) {
+        EMSG2(_(e_invarg2), "Incomplete msgpack string");
+        return FAIL;
+      }
+      return NOTDONE;
+    case MSGPACK_UNPACK_SUCCESS: {
+      typval_T tv = { .v_type = VAR_UNKNOWN };
+      if (msgpack_to_vim(data, &tv) == FAIL) {
+        EMSG2(_(e_invarg2), "Failed to convert msgpack string");
+        return FAIL;
+      }
+      tv_list_append_owned_tv(ret_list, tv);
+      return OK;
+    }
+    default:
+      abort();
   }
-  list_T *const ret_list = tv_list_alloc_ret(rettv, kListLenMayKnow);
-  const list_T *const list = argvars[0].vval.v_list;
+}
+
+static void msgpackparse_unpack_list(const list_T *const list,
+                                     list_T *const ret_list)
+  FUNC_ATTR_NONNULL_ARG(2)
+{
   if (tv_list_len(list) == 0) {
     return;
   }
@@ -6558,43 +6585,28 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr)
   do {
     if (!msgpack_unpacker_reserve_buffer(unpacker, IOSIZE)) {
       EMSG(_(e_outofmem));
-      goto f_msgpackparse_exit;
+      goto end;
     }
     size_t read_bytes;
     const int rlret = encode_read_from_list(
         &lrstate, msgpack_unpacker_buffer(unpacker), IOSIZE, &read_bytes);
     if (rlret == FAIL) {
       EMSG2(_(e_invarg2), "List item is not a string");
-      goto f_msgpackparse_exit;
+      goto end;
     }
     msgpack_unpacker_buffer_consumed(unpacker, read_bytes);
     if (read_bytes == 0) {
       break;
     }
     while (unpacker->off < unpacker->used) {
-      const msgpack_unpack_return result = msgpack_unpacker_next(unpacker,
-                                                                 &unpacked);
-      if (result == MSGPACK_UNPACK_PARSE_ERROR) {
-        EMSG2(_(e_invarg2), "Failed to parse msgpack string");
-        goto f_msgpackparse_exit;
-      }
-      if (result == MSGPACK_UNPACK_NOMEM_ERROR) {
-        EMSG(_(e_outofmem));
-        goto f_msgpackparse_exit;
-      }
-      if (result == MSGPACK_UNPACK_SUCCESS) {
-        typval_T tv = { .v_type = VAR_UNKNOWN };
-        if (msgpack_to_vim(unpacked.data, &tv) == FAIL) {
-          EMSG2(_(e_invarg2), "Failed to convert msgpack string");
-          goto f_msgpackparse_exit;
-        }
-        tv_list_append_owned_tv(ret_list, tv);
-      }
-      if (result == MSGPACK_UNPACK_CONTINUE) {
-        if (rlret == OK) {
-          EMSG2(_(e_invarg2), "Incomplete msgpack string");
-        }
+      const msgpack_unpack_return result
+          = msgpack_unpacker_next(unpacker, &unpacked);
+      const int conv_result = msgpackparse_convert_item(unpacked.data, result,
+                                                        ret_list, rlret == OK);
+      if (conv_result == NOTDONE) {
         break;
+      } else if (conv_result == FAIL) {
+        goto end;
       }
     }
     if (rlret == OK) {
@@ -6602,10 +6614,47 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr)
     }
   } while (true);
 
-f_msgpackparse_exit:
-  msgpack_unpacked_destroy(&unpacked);
+end:
   msgpack_unpacker_free(unpacker);
-  return;
+  msgpack_unpacked_destroy(&unpacked);
+}
+
+static void msgpackparse_unpack_blob(const blob_T *const blob,
+                                     list_T *const ret_list)
+  FUNC_ATTR_NONNULL_ARG(2)
+{
+  const int len = tv_blob_len(blob);
+  if (len == 0) {
+    return;
+  }
+  msgpack_unpacked unpacked;
+  msgpack_unpacked_init(&unpacked);
+  for (size_t offset = 0; offset < (size_t)len;) {
+    const msgpack_unpack_return result
+        = msgpack_unpack_next(&unpacked, blob->bv_ga.ga_data, len, &offset);
+    if (msgpackparse_convert_item(unpacked.data, result, ret_list, true)
+        != OK) {
+      break;
+    }
+  }
+
+  msgpack_unpacked_destroy(&unpacked);
+}
+
+/// "msgpackparse" function
+static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr)
+  FUNC_ATTR_NONNULL_ALL
+{
+  if (argvars[0].v_type != VAR_LIST && argvars[0].v_type != VAR_BLOB) {
+    EMSG2(_(e_listblobarg), "msgpackparse()");
+    return;
+  }
+  list_T *const ret_list = tv_list_alloc_ret(rettv, kListLenMayKnow);
+  if (argvars[0].v_type == VAR_LIST) {
+    msgpackparse_unpack_list(argvars[0].vval.v_list, ret_list);
+  } else {
+    msgpackparse_unpack_blob(argvars[0].vval.v_blob, ret_list);
+  }
 }
 
 /*
diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua
index 3db977257c..837b629858 100644
--- a/test/functional/eval/msgpack_functions_spec.lua
+++ b/test/functional/eval/msgpack_functions_spec.lua
@@ -1,5 +1,6 @@
 local helpers = require('test.functional.helpers')(after_each)
 local clear = helpers.clear
+local funcs = helpers.funcs
 local eval, eq = helpers.eval, helpers.eq
 local command = helpers.command
 local nvim = helpers.nvim
@@ -12,6 +13,7 @@ describe('msgpack*() functions', function()
     it(msg, function()
       nvim('set_var', 'obj', obj)
       eq(obj, eval('msgpackparse(msgpackdump(g:obj))'))
+      eq(obj, eval('msgpackparse(msgpackdump(g:obj, "B"))'))
     end)
   end
 
@@ -390,56 +392,61 @@ describe('msgpack*() functions', function()
   end)
 end)
 
+local blobstr = function(list)
+  local l = {}
+  for i,v in ipairs(list) do
+    l[i] = v:gsub('\n', '\000')
+  end
+  return table.concat(l, '\n')
+end
+
+-- Test msgpackparse() with a readfile()-style list and a blob argument
+local parse_eq = function(expect, list_arg)
+  local blob_expr = '0z' .. blobstr(list_arg):gsub('(.)', function(c)
+    return ('%.2x'):format(c:byte())
+  end)
+  eq(expect, funcs.msgpackparse(list_arg))
+  command('let g:parsed = msgpackparse(' .. blob_expr .. ')')
+  eq(expect, eval('g:parsed'))
+end
+
 describe('msgpackparse() function', function()
   before_each(clear)
 
   it('restores nil as v:null', function()
-    command('let dumped = ["\\xC0"]')
-    command('let parsed = msgpackparse(dumped)')
-    eq('[v:null]', eval('string(parsed)'))
+    parse_eq(eval('[v:null]'), {'\192'})
   end)
 
   it('restores boolean false as v:false', function()
-    command('let dumped = ["\\xC2"]')
-    command('let parsed = msgpackparse(dumped)')
-    eq({false}, eval('parsed'))
+    parse_eq({false}, {'\194'})
   end)
 
   it('restores boolean true as v:true', function()
-    command('let dumped = ["\\xC3"]')
-    command('let parsed = msgpackparse(dumped)')
-    eq({true}, eval('parsed'))
+    parse_eq({true}, {'\195'})
   end)
 
   it('restores FIXSTR as special dict', function()
-    command('let dumped = ["\\xa2ab"]')
-    command('let parsed = msgpackparse(dumped)')
-    eq({{_TYPE={}, _VAL={'ab'}}}, eval('parsed'))
+    parse_eq({{_TYPE={}, _VAL={'ab'}}}, {'\162ab'})
     eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.string'))
   end)
 
   it('restores BIN 8 as string', function()
-    command('let dumped = ["\\xC4\\x02ab"]')
-    eq({'ab'}, eval('msgpackparse(dumped)'))
+    parse_eq({'ab'}, {'\196\002ab'})
   end)
 
   it('restores FIXEXT1 as special dictionary', function()
-    command('let dumped = ["\\xD4\\x10", ""]')
-    command('let parsed = msgpackparse(dumped)')
-    eq({{_TYPE={}, _VAL={0x10, {"", ""}}}}, eval('parsed'))
+    parse_eq({{_TYPE={}, _VAL={0x10, {"", ""}}}}, {'\212\016', ''})
     eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.ext'))
   end)
 
   it('restores MAP with BIN key as special dictionary', function()
-    command('let dumped = ["\\x81\\xC4\\x01a\\xC4\\n"]')
-    command('let parsed = msgpackparse(dumped)')
-    eq({{_TYPE={}, _VAL={{'a', ''}}}}, eval('parsed'))
+    parse_eq({{_TYPE={}, _VAL={{'a', ''}}}}, {'\129\196\001a\196\n'})
     eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map'))
   end)
 
   it('restores MAP with duplicate STR keys as special dictionary', function()
     command('let dumped = ["\\x82\\xA1a\\xC4\\n\\xA1a\\xC4\\n"]')
-    -- FIXME Internal error bug
+    -- FIXME Internal error bug, can't use parse_eq() here
     command('silent! let parsed = msgpackparse(dumped)')
     eq({{_TYPE={}, _VAL={ {{_TYPE={}, _VAL={'a'}}, ''},
                           {{_TYPE={}, _VAL={'a'}}, ''}}} }, eval('parsed'))
@@ -449,9 +456,7 @@ describe('msgpackparse() function', function()
   end)
 
   it('restores MAP with MAP key as special dictionary', function()
-    command('let dumped = ["\\x81\\x80\\xC4\\n"]')
-    command('let parsed = msgpackparse(dumped)')
-    eq({{_TYPE={}, _VAL={{{}, ''}}}}, eval('parsed'))
+    parse_eq({{_TYPE={}, _VAL={{{}, ''}}}}, {'\129\128\196\n'})
     eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map'))
   end)
 
@@ -476,48 +481,57 @@ describe('msgpackparse() function', function()
   end)
 
   it('fails to parse a string', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse("abcdefghijklmnopqrstuvwxyz")'))
   end)
 
   it('fails to parse a number', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse(127)'))
   end)
 
   it('fails to parse a dictionary', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse({})'))
   end)
 
   it('fails to parse a funcref', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse(function("tr"))'))
   end)
 
   it('fails to parse a partial', function()
     command('function T() dict\nendfunction')
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse(function("T", [1, 2], {}))'))
   end)
 
   it('fails to parse a float', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse(0.0)'))
   end)
+
+  it('fails on incomplete msgpack string', function()
+    local expected = 'Vim(call):E475: Invalid argument: Incomplete msgpack string'
+    eq(expected, exc_exec([[call msgpackparse(["\xc4"])]]))
+    eq(expected, exc_exec([[call msgpackparse(["\xca", "\x02\x03"])]]))
+    eq(expected, exc_exec('call msgpackparse(0zc4)'))
+    eq(expected, exc_exec('call msgpackparse(0zca0a0203)'))
+  end)
+
+  it('fails when unable to parse msgpack string', function()
+    local expected = 'Vim(call):E475: Invalid argument: Failed to parse msgpack string'
+    eq(expected, exc_exec([[call msgpackparse(["\xc1"])]]))
+    eq(expected, exc_exec('call msgpackparse(0zc1)'))
+  end)
 end)
 
 describe('msgpackdump() function', function()
   before_each(clear)
 
   local dump_eq = function(exp_list, arg_expr)
-    local l = {}
-    for i,v in ipairs(exp_list) do
-      l[i] = v:gsub('\n', '\000')
-    end
-    local exp_blobstr = table.concat(l, '\n')
     eq(exp_list, eval('msgpackdump(' .. arg_expr .. ')'))
-    eq(exp_blobstr, eval('msgpackdump(' .. arg_expr .. ', "B")'))
+    eq(blobstr(exp_list), eval('msgpackdump(' .. arg_expr .. ', "B")'))
   end
 
   it('dumps string as BIN 8', function()