shada,functests: Test compatibility support

For compatibility the following things are done:

1. Items with type greater then greatest type are ignored when reading and
   copied when writing.
2. Registers with unknown name are ignored when reading and blindly copied when
   writing.
3. Registers with unknown type are ignored when reading and merged as usual when
   writing.
4. Local and global marks with unknown names are ignored when reading. When
   writing global marks are blindly copied and local marks are also blindly
   copied, but only if file they are attached to fits in the `'N` limit defined
   in &shada. Unknown local mark’s timestamp is also taken into account when
   calculating which files exactly should fit into this limit.
5. History items with unknown type are ignored when reading and blindly copied
   when writing.
6. Unknown keys found in register, local marks, global marks, changes, jumps and
   search pattern entries are read to additional_data Dictionary and dumped (of
   course, unless any of these elements were not overwritten later). It
   obviously works only for values conversible to Object type.
7. Additional elements found in replacement string and history entries are read
   to additional_elements Array and dumped (same: only if they were not
   overwritten later). Again this works only for elements conversible to Object
   type.
8. Additional elements found in variable entries are simply ignored when
   reading. When writing *new* variables they will be preserved during merging,
   but that’s all. Variable values dumped from current NeoVim session never have
   additional elements.

3 files changed, 451 insertions, 18 deletions

diff --git a/src/nvim/mark.c b/src/nvim/mark.c
index 5afa24e621..7e1ae42c47 100644
--- a/src/nvim/mark.c
+++ b/src/nvim/mark.c
@@ -1351,7 +1351,7 @@ bool mark_set_global(const char name, const xfmark_T fm, const bool update)
   if (fm_tgt == &namedfm[0] - 1) {
     return false;
   }
-  if (update && fm.fmark.timestamp < fm_tgt->fmark.timestamp) {
+  if (update && fm.fmark.timestamp <= fm_tgt->fmark.timestamp) {
     return false;
   }
   if (fm_tgt->fmark.mark.lnum != 0) {
@@ -1386,7 +1386,7 @@ bool mark_set_local(const char name, buf_T *const buf,
   } else {
     return false;
   }
-  if (update && fm.timestamp < fm_tgt->timestamp) {
+  if (update && fm.timestamp <= fm_tgt->timestamp) {
     return false;
   }
   if (fm_tgt->mark.lnum != 0) {
diff --git a/src/nvim/shada.c b/src/nvim/shada.c
index be049fbb20..770a5a89f4 100644
--- a/src/nvim/shada.c
+++ b/src/nvim/shada.c
@@ -1567,6 +1567,11 @@ static char *shada_filename(const char *file)
       msgpack_pack_str(spacker, sizeof(s) - 1); \
       msgpack_pack_str_body(spacker, s, sizeof(s) - 1); \
     } while (0)
+#define PACK_STRING(s) \
+    do { \
+      msgpack_pack_str(spacker, s.size); \
+      msgpack_pack_str_body(spacker, s.data, s.size); \
+    } while (0)
 
 /// Write single ShaDa entry
 ///
@@ -1587,7 +1592,7 @@ static bool shada_pack_entry(msgpack_packer *const packer,
     if ((src) != NULL) { \
       for (listitem_T *li = (src)->lv_first; li != NULL; li = li->li_next) { \
         if (vim_to_msgpack(spacker, &li->li_tv) == FAIL) { \
-          return false; \
+          goto shada_pack_entry_error; \
         } \
       } \
     } \
@@ -1605,7 +1610,7 @@ static bool shada_pack_entry(msgpack_packer *const packer,
           msgpack_pack_str(spacker, key_len); \
           msgpack_pack_str_body(spacker, (const char *) hi->hi_key, key_len); \
           if (vim_to_msgpack(spacker, &di->di_tv) == FAIL) { \
-            return false; \
+            goto shada_pack_entry_error; \
           } \
         } \
       } \
@@ -1616,12 +1621,9 @@ static bool shada_pack_entry(msgpack_packer *const packer,
       assert(false);
     }
     case kSDItemUnknown: {
-      if ((msgpack_pack_uint64(packer, (uint64_t) entry.data.unknown_item.size)
-           == -1)
-          || (packer->callback(packer->data, entry.data.unknown_item.contents,
-                               (unsigned) entry.data.unknown_item.size)
-              == -1)) {
-        return false;
+      if (spacker->callback(spacker->data, entry.data.unknown_item.contents,
+                            (unsigned) entry.data.unknown_item.size) == -1) {
+        goto shada_pack_entry_error;
       }
       break;
     }
@@ -1651,7 +1653,7 @@ static bool shada_pack_entry(msgpack_packer *const packer,
       msgpack_rpc_from_string(cstr_as_string(entry.data.global_var.name),
                               spacker);
       if (vim_to_msgpack(spacker, &entry.data.global_var.value) == FAIL) {
-        return false;
+        goto shada_pack_entry_error;
       }
       DUMP_ADDITIONAL_ELEMENTS(entry.data.global_var.additional_elements);
       break;
@@ -1826,30 +1828,34 @@ static bool shada_pack_entry(msgpack_packer *const packer,
   }
   if (!max_kbyte || sbuf.size <= max_kbyte * 1024) {
     if (entry.type == kSDItemUnknown) {
-      if (msgpack_pack_uint64(packer, (uint64_t) entry.data.unknown_item.type)
-          == -1) {
-        return false;
+      if (msgpack_pack_uint64(packer, entry.data.unknown_item.type) == -1) {
+        goto shada_pack_entry_error;
       }
     } else {
       if (msgpack_pack_uint64(packer, (uint64_t) entry.type) == -1) {
-        return false;
+        goto shada_pack_entry_error;
       }
     }
     if (msgpack_pack_uint64(packer, (uint64_t) entry.timestamp) == -1) {
-      return false;
+      goto shada_pack_entry_error;
     }
     if (sbuf.size > 0) {
       if ((msgpack_pack_uint64(packer, (uint64_t) sbuf.size) == -1)
           || (packer->callback(packer->data, sbuf.data,
                                (unsigned) sbuf.size) == -1)) {
-        return false;
+        goto shada_pack_entry_error;
       }
     }
   }
   msgpack_packer_free(spacker);
   msgpack_sbuffer_destroy(&sbuf);
   return true;
+shada_pack_entry_error:
+  msgpack_packer_free(spacker);
+  msgpack_sbuffer_destroy(&sbuf);
+  return false;
 }
+#undef PACK_STRING
 
 /// Write single ShaDa entry, converting it if needed
 ///
@@ -3260,7 +3266,14 @@ shada_read_next_item_start:
     entry->data.unknown_item.size = length;
     entry->data.unknown_item.type = type_u64;
     entry->data.unknown_item.contents = xmalloc(length);
-    return fread_len(sd_reader, entry->data.unknown_item.contents, length);
+    const ShaDaReadResult fl_ret = fread_len(sd_reader,
+                                             entry->data.unknown_item.contents,
+                                             length);
+    if (fl_ret != kSDReadStatusSuccess) {
+      shada_free_shada_entry(entry);
+      entry->type = kSDItemMissing;
+    }
+    return fl_ret;
   }
 
   char *const buf = xmalloc(length);
diff --git a/test/functional/shada/compatibility_spec.lua b/test/functional/shada/compatibility_spec.lua
new file mode 100644
index 0000000000..5c48d82f4a
--- /dev/null
+++ b/test/functional/shada/compatibility_spec.lua
@@ -0,0 +1,420 @@
+-- ShaDa compatibility support
+local helpers = require('test.functional.helpers')
+local nvim, nvim_window, nvim_curwin, nvim_command, nvim_feed, nvim_eval, eq =
+  helpers.nvim, helpers.window, helpers.curwin, helpers.command, helpers.feed,
+  helpers.eval, helpers.eq
+local write_file = helpers.write_file
+
+local shada_helpers = require('test.functional.shada.helpers')
+local reset, set_additional_cmd, clear, exc_exec =
+  shada_helpers.reset, shada_helpers.set_additional_cmd,
+  shada_helpers.clear, shada_helpers.exc_exec
+
+local shada_fname = 'Xtest-functional-shada-additional.shada'
+local wshada = function(text)
+  write_file(shada_fname, text, true)
+end
+local sdrcmd = function(bang)
+  return 'rshada' .. (bang and '!' or '') .. ' ' .. shada_fname
+end
+
+local msgpack = require('MessagePack')
+local mpack_keys = {'type', 'timestamp', 'length', 'value'}
+local read_shada_file = function(fname)
+  local fd = io.open(fname, 'r')
+  local mstring = fd:read('*a')
+  fd:close()
+  local unpacker = msgpack.unpacker(mstring)
+  local ret = {}
+  local cur
+  local i = 0
+  while true do
+    local off, val = unpacker()
+    if not off then break end
+    if i % 4 == 0 then
+      cur = {}
+      ret[#ret + 1] = cur
+    end
+    cur[mpack_keys[(i % 4) + 1]] = val
+    i = i + 1
+  end
+  return ret
+end
+
+describe('ShaDa forward compatibility support code', function()
+  before_each(reset)
+  after_each(function()
+    clear()
+    os.remove(shada_fname)
+  end)
+
+  it('works with search pattern item with BOOL unknown (sX) key value', function()
+    wshada('\002\001\011\130\162sX\194\162sp\196\001-')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 2 and not v.value.ss then
+        eq(false, v.value.sX)
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('silent! /---/')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 2 and v.value.ss then
+        eq(nil, v.value.sX)
+        found = true
+      end
+    end
+    eq(true, found)
+  end)
+
+  it('works with s/search pattern item with BOOL unknown (sX) key value', function()
+    wshada('\002\001\015\131\162sX\194\162ss\195\162sp\196\001-')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 2 and v.value.ss then
+        eq(false, v.value.sX)
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('silent! s/--/---/ge')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 2 and v.value.ss then
+        eq(nil, v.value.sX)
+        found = true
+      end
+    end
+    eq(true, found)
+  end)
+
+  it('works with replacement item with BOOL additional value in list', function()
+    wshada('\003\000\005\146\196\001-\194')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 3 then
+        eq(2, #v.value)
+        eq(false, v.value[2])
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('silent! s/--/---/ge')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 3 then
+        eq(1, #v.value)
+        found = true
+      end
+    end
+    eq(true, found)
+  end)
+
+  for _, v in ipairs({{name='global mark', mpack='\007\001\018\131\162mX\195\161f\196\006/a/b/c\161nA'},
+                      {name='jump', mpack='\008\001\018\131\162mX\195\161f\196\006/a/b/c\161l\002'},
+                      {name='local mark', mpack='\010\001\018\131\162mX\195\161f\196\006/a/b/c\161na'},
+                      {name='change', mpack='\011\001\015\130\162mX\195\161f\196\006/a/b/c'},
+                     }) do
+    it('works with ' .. v.name .. ' item with BOOL unknown (mX) key value', function()
+      nvim_command('silent noautocmd edit /a/b/c')
+      eq('/a/b/c', nvim_eval('bufname("%")'))
+      nvim_command('call setline(".", ["1", "2", "3"])')
+      wshada(v.mpack)
+      eq(0, exc_exec(sdrcmd(true)))
+      os.remove(shada_fname)
+      nvim_command('wshada ' .. shada_fname)
+      local found = false
+      for _, subv in ipairs(read_shada_file(shada_fname)) do
+        if subv.type == v.mpack:byte() then
+          if subv.value.mX == true then
+            found = true
+          end
+        end
+      end
+      eq(true, found)
+      eq(0, exc_exec(sdrcmd()))
+      nvim_command('bwipeout!')
+      nvim_eval('setpos("\'A", [0, 1, 1, 0])')
+      os.remove(shada_fname)
+      nvim_command('wshada ' .. shada_fname)
+      found = false
+      for _, subv in ipairs(read_shada_file(shada_fname)) do
+        if subv.type == v.mpack:byte() then
+          if subv.value.mX == true then
+            found = true
+          end
+        end
+      end
+      eq(false, found)
+    end)
+
+    if v.name == 'global mark' or v.name == 'local mark' then
+      it('works with ' .. v.name .. ' item with <C-a> name', function()
+        nvim_command('silent noautocmd edit /a/b/c')
+        eq('/a/b/c', nvim_eval('bufname("%")'))
+        nvim_command('call setline(".", ["1", "2", "3"])')
+        wshada(v.mpack:gsub('n.$', 'n\001')
+               .. v.mpack:gsub('n.$', 'n\002')
+               .. v.mpack:gsub('n.$', 'n\003'):gsub('/a/b/c', '/d/e/f'))
+        eq(0, exc_exec(sdrcmd(true)))
+        nvim_command('wshada ' .. shada_fname)
+        local found = 0
+        for i, subv in ipairs(read_shada_file(shada_fname)) do
+          if i == 1 then
+            eq(1, subv.type)
+          end
+          if subv.type == v.mpack:byte() then
+            if subv.value.mX == true and subv.value.n <= 3 then
+              found = found + 1
+            end
+          end
+        end
+        eq(3, found)
+        nvim_command('wshada! ' .. shada_fname)
+        local found = 0
+        for i, subv in ipairs(read_shada_file(shada_fname)) do
+          if i == 1 then
+            eq(1, subv.type)
+          end
+          if subv.type == v.mpack:byte() then
+            if subv.value.mX == true and subv.value.n <= 3 then
+              found = found + 1
+            end
+          end
+        end
+        eq(0, found)
+      end)
+    end
+  end
+
+  it('works with register item with BOOL unknown (rX) key', function()
+    wshada('\005\001\015\131\161na\162rX\194\162rc\145\196\001-')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 5 and v.value.rX == false then
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('let @a = "Test"')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 5 and v.value.rX == false then
+        found = true
+      end
+    end
+    eq(false, found)
+  end)
+
+  it('works with register item with <C-a> name', function()
+    wshada('\005\001\015\131\161n\001\162rX\194\162rc\145\196\001-')
+    eq(0, exc_exec(sdrcmd(true)))
+    nvim_command('wshada ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 5 then
+        if v.value.rX == false and v.value.n == 1 then
+          found = found + 1
+        end
+      end
+    end
+    eq(1, found)
+    nvim_command('wshada! ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 5 then
+        if v.value.rX == false and v.value.n == 1 then
+          found = found + 1
+        end
+      end
+    end
+    eq(0, found)
+  end)
+
+  it('works with register item with type 10', function()
+    wshada('\005\001\019\132\161na\162rX\194\162rc\145\196\001-\162rt\010')
+    eq(0, exc_exec(sdrcmd(true)))
+    eq({{}, ''}, nvim_eval('[getreg("a", 1, 1)[:], getregtype("a")]'))
+    nvim_command('wshada ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 5 then
+        if v.value.rX == false and v.value.rt == 10 then
+          found = found + 1
+        end
+      end
+    end
+    eq(1, found)
+    nvim_command('wshada! ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 5 then
+        if v.value.rX == false and v.value.rt == 10 then
+          found = found + 1
+        end
+      end
+    end
+    eq(0, found)
+  end)
+
+  it('works with buffer list item with BOOL unknown (bX) key', function()
+    nvim_command('set shada+=%')
+    wshada('\009\000\016\145\130\161f\196\006/a/b/c\162bX\195')
+    eq(0, exc_exec(sdrcmd()))
+    eq(2, nvim_eval('bufnr("$")'))
+    eq('/a/b/c', nvim_eval('bufname(2)'))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 9 and #v.value == 1 and v.value[1].bX == true then
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('buffer 2')
+    nvim_command('edit!')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 5 and v.value.rX == false then
+        found = true
+      end
+    end
+    eq(false, found)
+    nvim_command('bwipeout!')
+  end)
+
+  it('works with history item with BOOL additional value in list', function()
+    wshada('\004\000\006\147\000\196\001-\194')
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_command('wshada ' .. shada_fname)
+    local found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 4 and v.value[1] == 0 and v.value[2] == '-' then
+        eq(false, v.value[3])
+        eq(3, #v.value)
+        found = true
+      end
+    end
+    eq(true, found)
+    eq(0, exc_exec(sdrcmd()))
+    os.remove(shada_fname)
+    nvim_eval('histadd(":", "--")')
+    nvim_eval('histadd(":", "-")')
+    nvim_command('wshada ' .. shada_fname)
+    found = false
+    for _, v in ipairs(read_shada_file(shada_fname)) do
+      if v.type == 4 and v.value[1] == 0 and v.value[2] == '-' then
+        eq(2, #v.value)
+        found = true
+      end
+    end
+    eq(true, found)
+  end)
+
+  it('works with history item with type 10', function()
+    wshada('\004\000\006\147\010\196\001-\194')
+    eq(0, exc_exec(sdrcmd()))
+    nvim_command('wshada ' .. shada_fname)
+    eq(0, exc_exec(sdrcmd()))
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 4 then
+        if v.value[1] == 10 and #v.value == 3 and v.value[3] == false then
+          found = found + 1
+        end
+      end
+    end
+    eq(1, found)
+    nvim_command('wshada! ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 4 then
+        if v.value[1] == 10 and #v.value == 3 and v.value[3] == false then
+          found = found + 1
+        end
+      end
+    end
+    eq(0, found)
+  end)
+
+  it('works with item with 100 type', function()
+    wshada('\100\000\006\147\010\196\001-\194')
+    eq(0, exc_exec(sdrcmd()))
+    nvim_command('wshada ' .. shada_fname)
+    eq(0, exc_exec(sdrcmd()))
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 100 then
+        if v.value[1] == 10 and #v.value == 3 and v.value[3] == false then
+          found = found + 1
+        end
+      end
+    end
+    eq(1, found)
+    nvim_command('wshada! ' .. shada_fname)
+    local found = 0
+    for i, v in ipairs(read_shada_file(shada_fname)) do
+      if i == 1 then
+        eq(1, v.type)
+      end
+      if v.type == 100 then
+        if v.value[1] == 10 and #v.value == 3 and v.value[3] == false then
+          found = found + 1
+        end
+      end
+    end
+    eq(0, found)
+  end)
+end)