eval/encode: Fix writing strings starting with NL to list

Error [found][1] by oni-link.

[1]: https://github.com/neovim/neovim/pull/4131/files#r52239384

5 files changed, 192 insertions, 17 deletions

diff --git a/src/nvim/eval.h b/src/nvim/eval.h
index 50c11011c6..9d45b780a9 100644
--- a/src/nvim/eval.h
+++ b/src/nvim/eval.h
@@ -1,8 +1,6 @@
 #ifndef NVIM_EVAL_H
 #define NVIM_EVAL_H
 
-#include <msgpack.h>
-
 #include "nvim/profile.h"
 #include "nvim/hashtab.h"  // For hashtab_T
 #include "nvim/garray.h"  // For garray_T
diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c
index 8280889fbe..6026189235 100644
--- a/src/nvim/eval/encode.c
+++ b/src/nvim/eval/encode.c
@@ -79,11 +79,9 @@ int encode_list_write(void *data, const char *buf, size_t len)
   do {
     const char *line_start = line_end;
     line_end = xmemscan(line_start, NL, (size_t) (end - line_start));
-    if (line_end == line_start) {
-      list_append_allocated_string(list, NULL);
-    } else {
+    char *str = NULL;
+    if (line_end != line_start) {
       const size_t line_length = (size_t) (line_end - line_start);
-      char *str;
       if (li == NULL) {
         str = xmemdupz(line_start, line_length);
       } else {
@@ -93,7 +91,7 @@ int encode_list_write(void *data, const char *buf, size_t len)
         li->li_tv.vval.v_string = xrealloc(li->li_tv.vval.v_string,
                                            li_len + line_length + 1);
         str = (char *) li->li_tv.vval.v_string + li_len;
-        memmove(str, line_start, line_length);
+        memcpy(str, line_start, line_length);
         str[line_length] = 0;
       }
       for (size_t i = 0; i < line_length; i++) {
@@ -101,14 +99,14 @@ int encode_list_write(void *data, const char *buf, size_t len)
           str[i] = NL;
         }
       }
-      if (li == NULL) {
-        list_append_allocated_string(list, str);
-      } else {
-        li = NULL;
-      }
-      if (line_end == end - 1) {
-        list_append_allocated_string(list, NULL);
-      }
+    }
+    if (li == NULL) {
+      list_append_allocated_string(list, str);
+    } else {
+      li = NULL;
+    }
+    if (line_end == end - 1) {
+      list_append_allocated_string(list, NULL);
     }
     line_end++;
   } while (line_end < end);
diff --git a/test/unit/eval/encode_spec.lua b/test/unit/eval/encode_spec.lua
new file mode 100644
index 0000000000..f151a191fb
--- /dev/null
+++ b/test/unit/eval/encode_spec.lua
@@ -0,0 +1,100 @@
+local helpers = require('test.unit.helpers')
+local eval_helpers = require('test.unit.eval.helpers')
+
+local cimport = helpers.cimport
+local to_cstr = helpers.to_cstr
+local eq = helpers.eq
+
+local list = eval_helpers.list
+local lst2tbl = eval_helpers.lst2tbl
+local type_key = eval_helpers.type_key
+local list_type = eval_helpers.list_type
+local null_string = eval_helpers.null_string
+
+local encode = cimport('./src/nvim/eval/encode.h')
+
+describe('encode_list_write()', function()
+  local encode_list_write = function(l, s)
+    return encode.encode_list_write(l, to_cstr(s), #s)
+  end
+
+  it('writes empty string', function()
+    local l = list()
+    eq(0, encode_list_write(l, ''))
+    eq({[type_key]=list_type}, lst2tbl(l))
+  end)
+
+  it('writes ASCII string literal with printable characters', function()
+    local l = list()
+    eq(0, encode_list_write(l, 'abc'))
+    eq({[type_key]=list_type, 'abc'}, lst2tbl(l))
+  end)
+
+  it('writes string starting with NL', function()
+    local l = list()
+    eq(0, encode_list_write(l, '\nabc'))
+    eq({[type_key]=list_type, null_string, 'abc'}, lst2tbl(l))
+  end)
+
+  it('writes string starting with NL twice', function()
+    local l = list()
+    eq(0, encode_list_write(l, '\nabc'))
+    eq({[type_key]=list_type, null_string, 'abc'}, lst2tbl(l))
+    eq(0, encode_list_write(l, '\nabc'))
+    eq({[type_key]=list_type, null_string, 'abc', 'abc'}, lst2tbl(l))
+  end)
+
+  it('writes string ending with NL', function()
+    local l = list()
+    eq(0, encode_list_write(l, 'abc\n'))
+    eq({[type_key]=list_type, 'abc', null_string}, lst2tbl(l))
+  end)
+
+  it('writes string ending with NL twice', function()
+    local l = list()
+    eq(0, encode_list_write(l, 'abc\n'))
+    eq({[type_key]=list_type, 'abc', null_string}, lst2tbl(l))
+    eq(0, encode_list_write(l, 'abc\n'))
+    eq({[type_key]=list_type, 'abc', 'abc', null_string}, lst2tbl(l))
+  end)
+
+  it('writes string starting, ending and containing NL twice', function()
+    local l = list()
+    eq(0, encode_list_write(l, '\na\nb\n'))
+    eq({[type_key]=list_type, null_string, 'a', 'b', null_string}, lst2tbl(l))
+    eq(0, encode_list_write(l, '\na\nb\n'))
+    eq({[type_key]=list_type, null_string, 'a', 'b', null_string, 'a', 'b', null_string}, lst2tbl(l))
+  end)
+
+  it('writes string starting, ending and containing NUL with NL between twice', function()
+    local l = list()
+    eq(0, encode_list_write(l, '\0\n\0\n\0'))
+    eq({[type_key]=list_type, '\n', '\n', '\n'}, lst2tbl(l))
+    eq(0, encode_list_write(l, '\0\n\0\n\0'))
+    eq({[type_key]=list_type, '\n', '\n', '\n\n', '\n', '\n'}, lst2tbl(l))
+  end)
+
+  it('writes string starting, ending and containing NL with NUL between twice', function()
+    local l = list()
+    eq(0, encode_list_write(l, '\n\0\n\0\n'))
+    eq({[type_key]=list_type, null_string, '\n', '\n', null_string}, lst2tbl(l))
+    eq(0, encode_list_write(l, '\n\0\n\0\n'))
+    eq({[type_key]=list_type, null_string, '\n', '\n', null_string, '\n', '\n', null_string}, lst2tbl(l))
+  end)
+
+  it('writes string containing a single NL twice', function()
+    local l = list()
+    eq(0, encode_list_write(l, '\n'))
+    eq({[type_key]=list_type, null_string, null_string}, lst2tbl(l))
+    eq(0, encode_list_write(l, '\n'))
+    eq({[type_key]=list_type, null_string, null_string, null_string}, lst2tbl(l))
+  end)
+
+  it('writes string containing a few NLs twice', function()
+    local l = list()
+    eq(0, encode_list_write(l, '\n\n\n'))
+    eq({[type_key]=list_type, null_string, null_string, null_string, null_string}, lst2tbl(l))
+    eq(0, encode_list_write(l, '\n\n\n'))
+    eq({[type_key]=list_type, null_string, null_string, null_string, null_string, null_string, null_string, null_string}, lst2tbl(l))
+  end)
+end)
diff --git a/test/unit/eval/helpers.lua b/test/unit/eval/helpers.lua
new file mode 100644
index 0000000000..da2f5626ff
--- /dev/null
+++ b/test/unit/eval/helpers.lua
@@ -0,0 +1,72 @@
+local helpers = require('test.unit.helpers')
+
+local cimport = helpers.cimport
+local to_cstr = helpers.to_cstr
+local ffi = helpers.ffi
+local eq = helpers.eq
+
+local eval = cimport('./src/nvim/eval.h', './src/nvim/eval_defs.h')
+
+local null_string = {[true]='NULL string'}
+local null_list = {[true]='NULL list'}
+local type_key = {[true]='type key'}
+local list_type = {[true]='list type'}
+
+local list = function(...)
+  local ret = ffi.gc(eval.list_alloc(), eval.list_unref)
+  eq(0, ret.lv_refcount)
+  ret.lv_refcount = 1
+  for i = 1, select('#', ...) do
+    local val = select(i, ...)
+    local typ = type(val)
+    if typ == 'string' then
+      eval.list_append_string(ret, to_cstr(val))
+    elseif typ == 'table' and val == null_string then
+      eval.list_append_string(ret, nil)
+    elseif typ == 'table' and val == null_list then
+      eval.list_append_list(ret, nil)
+    elseif typ == 'table' and val[type_key] == list_type then
+      local itemlist = ffi.gc(list(table.unpack(val)), nil)
+      eq(1, itemlist.lv_refcount)
+      itemlist.lv_refcount = 0
+      eval.list_append_list(ret, itemlist)
+    else
+      assert(false, 'Not implemented yet')
+    end
+  end
+  return ret
+end
+
+local lst2tbl = function(l)
+  local ret = {[type_key]=list_type}
+  if l == nil then
+    return ret
+  end
+  local li = l.lv_first
+  -- (listitem_T *) NULL is equal to nil, but yet it is not false.
+  while li ~= nil do
+    local typ = li.li_tv.v_type
+    if typ == eval.VAR_STRING then
+      str = li.li_tv.vval.v_string
+      if str == nil then
+        ret[#ret + 1] = null_string
+      else
+        ret[#ret + 1] = ffi.string(str)
+      end
+    else
+      assert(false, 'Not implemented yet')
+    end
+    li = li.li_next
+  end
+  return ret
+end
+
+return {
+  null_string=null_string,
+  null_list=null_list,
+  list_type=list_type,
+  type_key=type_key,
+
+  list=list,
+  lst2tbl=lst2tbl,
+}
diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua
index 7b43b2218c..9b9c1fef0f 100644
--- a/test/unit/helpers.lua
+++ b/test/unit/helpers.lua
@@ -28,8 +28,10 @@ local function filter_complex_blocks(body)
   local result = {}
 
   for line in body:gmatch("[^\r\n]+") do
-    if not (string.find(line, "(^)", 1, true) ~= nil or
-      string.find(line, "_ISwupper", 1, true)) then
+    if not (string.find(line, "(^)", 1, true) ~= nil
+            or string.find(line, "_ISwupper", 1, true)
+            or string.find(line, "msgpack_zone_push_finalizer")
+            or string.find(line, "msgpack_unpacker_reserve_buffer")) then
       result[#result + 1] = line
     end
   end
@@ -103,6 +105,11 @@ local function cimport(...)
   -- request a sorted version of the new lines (same relative order as the
   -- original preprocessed file) and feed that to the LuaJIT ffi
   local new_lines = new_cdefs:to_table()
+  if os.getenv('NVIM_TEST_PRINT_CDEF') == '1' then
+    for lnum, line in ipairs(new_lines) do
+      print(lnum, line)
+    end
+  end
   ffi.cdef(table.concat(new_lines, "\n"))
 
   return libnvim