refactor(shada): rework msgpack decoding without msgpack-c

This also makes shada reading slightly faster due to avoiding
some copying and allocation.

Use keysets to drive decoding of msgpack maps for shada entries.

42 files changed, 1019 insertions, 1159 deletions

diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua
index b490e7b480..fc2fadc440 100755
--- a/scripts/gen_eval_files.lua
+++ b/scripts/gen_eval_files.lua
@@ -376,6 +376,9 @@ end
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
 local function render_api_keyset_meta(_f, fun, write)
+  if string.sub(fun.name, 1, 1) == '_' then
+    return -- not exported
+  end
   write('')
   write('--- @class vim.api.keyset.' .. fun.name)
   for _, p in ipairs(fun.params) do
diff --git a/src/nvim/api/command.c b/src/nvim/api/command.c
index 5ad439af9c..78be0a2cf7 100644
--- a/src/nvim/api/command.c
+++ b/src/nvim/api/command.c
@@ -515,7 +515,7 @@ String nvim_cmd(uint64_t channel_id, Dict(cmd) *cmd, Dict(cmd_opts) *opts, Arena
 
   if (HAS_KEY(cmd, cmd, magic)) {
     Dict(cmd_magic) magic[1] = KEYDICT_INIT;
-    if (!api_dict_to_keydict(magic, KeyDict_cmd_magic_get_field, cmd->magic, err)) {
+    if (!api_dict_to_keydict(magic, DictHash(cmd_magic), cmd->magic, err)) {
       goto end;
     }
 
@@ -533,14 +533,14 @@ String nvim_cmd(uint64_t channel_id, Dict(cmd) *cmd, Dict(cmd_opts) *opts, Arena
 
   if (HAS_KEY(cmd, cmd, mods)) {
     Dict(cmd_mods) mods[1] = KEYDICT_INIT;
-    if (!api_dict_to_keydict(mods, KeyDict_cmd_mods_get_field, cmd->mods, err)) {
+    if (!api_dict_to_keydict(mods, DictHash(cmd_mods), cmd->mods, err)) {
       goto end;
     }
 
     if (HAS_KEY(mods, cmd_mods, filter)) {
       Dict(cmd_mods_filter) filter[1] = KEYDICT_INIT;
 
-      if (!api_dict_to_keydict(&filter, KeyDict_cmd_mods_filter_get_field,
+      if (!api_dict_to_keydict(&filter, DictHash(cmd_mods_filter),
                                mods->filter, err)) {
         goto end;
       }
diff --git a/src/nvim/api/keysets_defs.h b/src/nvim/api/keysets_defs.h
index cc2ef981b5..ce4e3d56e1 100644
--- a/src/nvim/api/keysets_defs.h
+++ b/src/nvim/api/keysets_defs.h
@@ -103,7 +103,7 @@ typedef struct {
   Object nargs;
   Object preview;
   Object range;
-  Boolean register_;
+  Boolean register_ DictKey(register);
 } Dict(user_command);
 
 typedef struct {
@@ -170,7 +170,7 @@ typedef struct {
   Boolean reverse;
   Boolean altfont;
   Boolean nocombine;
-  Boolean default_;
+  Boolean default_ DictKey(default);
   Object cterm;
   Object foreground;
   Object fg;
@@ -392,3 +392,41 @@ typedef struct {
   OptionalKeys is_set__ns_opts_;
   Array wins;
 } Dict(ns_opts);
+
+typedef struct {
+  OptionalKeys is_set___shada_search_pat_;
+  Boolean magic DictKey(sm);
+  Boolean smartcase DictKey(sc);
+  Boolean has_line_offset DictKey(sl);
+  Boolean place_cursor_at_end DictKey(se);
+  Boolean is_last_used DictKey(su);
+  Boolean is_substitute_pattern DictKey(ss);
+  Boolean highlighted DictKey(sh);
+  Boolean search_backward DictKey(sb);
+  Integer offset DictKey(so);
+  String pat DictKey(sp);
+} Dict(_shada_search_pat);
+
+typedef struct {
+  OptionalKeys is_set___shada_mark_;
+  Integer n;
+  Integer l;
+  Integer c;
+  String f;
+} Dict(_shada_mark);
+
+typedef struct {
+  OptionalKeys is_set___shada_register_;
+  StringArray rc;
+  Boolean ru;
+  Integer rt;
+  Integer n;
+  Integer rw;
+} Dict(_shada_register);
+
+typedef struct {
+  OptionalKeys is_set___shada_buflist_item_;
+  Integer l;
+  Integer c;
+  String f;
+} Dict(_shada_buflist_item);
diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index 94e2d76cbe..ac3404cf86 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -19,6 +19,8 @@
 # define ArrayOf(...) Array
 # define DictionaryOf(...) Dictionary
 # define Dict(name) KeyDict_##name
+# define DictHash(name) KeyDict_##name##_get_field
+# define DictKey(name)
 # include "api/private/defs.h.inline.generated.h"
 #endif
 
@@ -88,6 +90,8 @@ typedef kvec_t(Object) Array;
 typedef struct key_value_pair KeyValuePair;
 typedef kvec_t(KeyValuePair) Dictionary;
 
+typedef kvec_t(String) StringArray;
+
 typedef enum {
   kObjectTypeNil = 0,
   kObjectTypeBoolean,
@@ -103,6 +107,10 @@ typedef enum {
   kObjectTypeTabpage,
 } ObjectType;
 
+typedef enum {
+  kUnpackTypeStringArray = -1,
+} UnpackType;
+
 /// Value by which objects represented as EXT type are shifted
 ///
 /// Subtracted when packing, added when unpacking. Used to allow moving
@@ -140,7 +148,7 @@ typedef struct {
 typedef struct {
   char *str;
   size_t ptr_off;
-  ObjectType type;  // kObjectTypeNil == untyped
+  int type;  // ObjectType or UnpackType. kObjectTypeNil == untyped
   int opt_index;
   bool is_hlgroup;
 } KeySetLink;
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index ac621b1486..017265e0d3 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -1,6 +1,5 @@
 #include <assert.h>
 #include <limits.h>
-#include <msgpack/unpack.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stddef.h>
@@ -922,15 +921,17 @@ bool api_dict_to_keydict(void *retval, FieldHashfn hashy, Dictionary dict, Error
       } else if (value->type == kObjectTypeDictionary) {
         *val = value->data.dictionary;
       } else {
-        api_err_exp(err, field->str, api_typename(field->type), api_typename(value->type));
+        api_err_exp(err, field->str, api_typename((ObjectType)field->type),
+                    api_typename(value->type));
         return false;
       }
     } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
                || field->type == kObjectTypeTabpage) {
-      if (value->type == kObjectTypeInteger || value->type == field->type) {
+      if (value->type == kObjectTypeInteger || value->type == (ObjectType)field->type) {
         *(handle_T *)mem = (handle_T)value->data.integer;
       } else {
-        api_err_exp(err, field->str, api_typename(field->type), api_typename(value->type));
+        api_err_exp(err, field->str, api_typename((ObjectType)field->type),
+                    api_typename(value->type));
         return false;
       }
     } else if (field->type == kObjectTypeLuaRef) {
@@ -980,7 +981,7 @@ Dictionary api_keydict_to_dict(void *value, KeySetLink *table, size_t max_size,
     } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
                || field->type == kObjectTypeTabpage) {
       val.data.integer = *(handle_T *)mem;
-      val.type = field->type;
+      val.type = (ObjectType)field->type;
     } else if (field->type == kObjectTypeLuaRef) {
       // do nothing
     } else {
diff --git a/src/nvim/api/ui.c b/src/nvim/api/ui.c
index 852b8b9b48..82a5ff5f8e 100644
--- a/src/nvim/api/ui.c
+++ b/src/nvim/api/ui.c
@@ -1,6 +1,5 @@
 #include <assert.h>
 #include <inttypes.h>
-#include <msgpack/pack.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
diff --git a/src/nvim/buffer.c b/src/nvim/buffer.c
index 2fc684b412..0c301bd5e2 100644
--- a/src/nvim/buffer.c
+++ b/src/nvim/buffer.c
@@ -868,7 +868,7 @@ static void free_buffer(buf_T *buf)
   }
   unref_var_dict(buf->b_vars);
   aubuflocal_remove(buf);
-  tv_dict_unref(buf->additional_data);
+  xfree(buf->additional_data);
   xfree(buf->b_prompt_text);
   callback_free(&buf->b_prompt_callback);
   callback_free(&buf->b_prompt_interrupt);
diff --git a/src/nvim/buffer_defs.h b/src/nvim/buffer_defs.h
index 221a86a907..8e076646c3 100644
--- a/src/nvim/buffer_defs.h
+++ b/src/nvim/buffer_defs.h
@@ -712,7 +712,7 @@ struct file_buffer {
 
   Terminal *terminal;           // Terminal instance associated with the buffer
 
-  dict_T *additional_data;      // Additional data from shada file if any.
+  AdditionalData *additional_data;      // Additional data from shada file if any.
 
   int b_mapped_ctrl_c;          // modes where CTRL-C is mapped
 
diff --git a/src/nvim/cmdhist.c b/src/nvim/cmdhist.c
index ab05ae1cfc..47a4ffba9e 100644
--- a/src/nvim/cmdhist.c
+++ b/src/nvim/cmdhist.c
@@ -190,7 +190,7 @@ static inline void hist_free_entry(histentry_T *hisptr)
   FUNC_ATTR_NONNULL_ALL
 {
   xfree(hisptr->hisstr);
-  tv_list_unref(hisptr->additional_elements);
+  xfree(hisptr->additional_data);
   clear_hist_entry(hisptr);
 }
 
@@ -237,7 +237,7 @@ static int in_history(int type, const char *str, int move_to_front, int sep)
     return false;
   }
 
-  list_T *const list = history[type][i].additional_elements;
+  AdditionalData *ad = history[type][i].additional_data;
   char *const save_hisstr = history[type][i].hisstr;
   while (i != hisidx[type]) {
     if (++i >= hislen) {
@@ -246,11 +246,11 @@ static int in_history(int type, const char *str, int move_to_front, int sep)
     history[type][last_i] = history[type][i];
     last_i = i;
   }
-  tv_list_unref(list);
+  xfree(ad);
   history[type][i].hisnum = ++hisnum[type];
   history[type][i].hisstr = save_hisstr;
   history[type][i].timestamp = os_time();
-  history[type][i].additional_elements = NULL;
+  history[type][i].additional_data = NULL;
   return true;
 }
 
@@ -337,7 +337,7 @@ void add_to_history(int histype, const char *new_entry, size_t new_entrylen, boo
   // Store the separator after the NUL of the string.
   hisptr->hisstr = xstrnsave(new_entry, new_entrylen + 2);
   hisptr->timestamp = os_time();
-  hisptr->additional_elements = NULL;
+  hisptr->additional_data = NULL;
   hisptr->hisstr[new_entrylen + 1] = (char)sep;
 
   hisptr->hisnum = ++hisnum[histype];
diff --git a/src/nvim/cmdhist.h b/src/nvim/cmdhist.h
index 43be397cee..4df4b09e68 100644
--- a/src/nvim/cmdhist.h
+++ b/src/nvim/cmdhist.h
@@ -24,7 +24,7 @@ typedef struct {
   int hisnum;           ///< Entry identifier number.
   char *hisstr;         ///< Actual entry, separator char after the NUL.
   Timestamp timestamp;  ///< Time when entry was added.
-  list_T *additional_elements;  ///< Additional entries from ShaDa file.
+  AdditionalData *additional_data;  ///< Additional entries from ShaDa file.
 } histentry_T;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
diff --git a/src/nvim/context.h b/src/nvim/context.h
index 0bb7138b6c..4375030fbc 100644
--- a/src/nvim/context.h
+++ b/src/nvim/context.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <msgpack/sbuffer.h>
 #include <stddef.h>
 
 #include "klib/kvec.h"
diff --git a/src/nvim/eval.c b/src/nvim/eval.c
index 0180d37ad8..31e4953a5c 100644
--- a/src/nvim/eval.c
+++ b/src/nvim/eval.c
@@ -4788,19 +4788,6 @@ bool garbage_collect(bool testing)
   FOR_ALL_BUFFERS(buf) {
     // buffer-local variables
     ABORTING(set_ref_in_item)(&buf->b_bufvar.di_tv, copyID, NULL, NULL);
-    // buffer marks (ShaDa additional data)
-    ABORTING(set_ref_in_fmark)(buf->b_last_cursor, copyID);
-    ABORTING(set_ref_in_fmark)(buf->b_last_insert, copyID);
-    ABORTING(set_ref_in_fmark)(buf->b_last_change, copyID);
-    for (size_t i = 0; i < NMARKS; i++) {
-      ABORTING(set_ref_in_fmark)(buf->b_namedm[i], copyID);
-    }
-    // buffer change list (ShaDa additional data)
-    for (int i = 0; i < buf->b_changelistlen; i++) {
-      ABORTING(set_ref_in_fmark)(buf->b_changelist[i], copyID);
-    }
-    // buffer ShaDa additional data
-    ABORTING(set_ref_dict)(buf->additional_data, copyID);
 
     // buffer callback functions
     ABORTING(set_ref_in_callback)(&buf->b_prompt_callback, copyID, NULL, NULL);
@@ -4823,10 +4810,6 @@ bool garbage_collect(bool testing)
   FOR_ALL_TAB_WINDOWS(tp, wp) {
     // window-local variables
     ABORTING(set_ref_in_item)(&wp->w_winvar.di_tv, copyID, NULL, NULL);
-    // window jump list (ShaDa additional data)
-    for (int i = 0; i < wp->w_jumplistlen; i++) {
-      ABORTING(set_ref_in_fmark)(wp->w_jumplist[i].fmark, copyID);
-    }
   }
   // window-local variables in autocmd windows
   for (int i = 0; i < AUCMD_WIN_COUNT; i++) {
@@ -4843,9 +4826,6 @@ bool garbage_collect(bool testing)
       char name = NUL;
       bool is_unnamed = false;
       reg_iter = op_global_reg_iter(reg_iter, &name, &reg, &is_unnamed);
-      if (name != NUL) {
-        ABORTING(set_ref_dict)(reg.additional_data, copyID);
-      }
     } while (reg_iter != NULL);
   }
 
@@ -4856,9 +4836,6 @@ bool garbage_collect(bool testing)
       xfmark_T fm;
       char name = NUL;
       mark_iter = mark_global_iter(mark_iter, &name, &fm);
-      if (name != NUL) {
-        ABORTING(set_ref_dict)(fm.fmark.additional_data, copyID);
-      }
     } while (mark_iter != NULL);
   }
 
@@ -4900,36 +4877,6 @@ bool garbage_collect(bool testing)
   // v: vars
   ABORTING(set_ref_in_ht)(&vimvarht, copyID, NULL);
 
-  // history items (ShaDa additional elements)
-  if (p_hi) {
-    for (int i = 0; i < HIST_COUNT; i++) {
-      const void *iter = NULL;
-      do {
-        histentry_T hist;
-        iter = hist_iter(iter, (uint8_t)i, false, &hist);
-        if (hist.hisstr != NULL) {
-          ABORTING(set_ref_list)(hist.additional_elements, copyID);
-        }
-      } while (iter != NULL);
-    }
-  }
-
-  // previously used search/substitute patterns (ShaDa additional data)
-  {
-    SearchPattern pat;
-    get_search_pattern(&pat);
-    ABORTING(set_ref_dict)(pat.additional_data, copyID);
-    get_substitute_pattern(&pat);
-    ABORTING(set_ref_dict)(pat.additional_data, copyID);
-  }
-
-  // previously used replacement string
-  {
-    SubReplacementString sub;
-    sub_get_replacement(&sub);
-    ABORTING(set_ref_list)(sub.additional_elements, copyID);
-  }
-
   ABORTING(set_ref_in_quickfix)(copyID);
 
   bool did_free = false;
@@ -5211,52 +5158,6 @@ bool set_ref_in_item(typval_T *tv, int copyID, ht_stack_T **ht_stack, list_stack
   return abort;
 }
 
-/// Mark all lists and dicts referenced in given mark
-///
-/// @return  true if setting references failed somehow.
-static inline bool set_ref_in_fmark(fmark_T fm, int copyID)
-  FUNC_ATTR_WARN_UNUSED_RESULT
-{
-  if (fm.additional_data != NULL
-      && fm.additional_data->dv_copyID != copyID) {
-    fm.additional_data->dv_copyID = copyID;
-    return set_ref_in_ht(&fm.additional_data->dv_hashtab, copyID, NULL);
-  }
-  return false;
-}
-
-/// Mark all lists and dicts referenced in given list and the list itself
-///
-/// @return  true if setting references failed somehow.
-static inline bool set_ref_list(list_T *list, int copyID)
-  FUNC_ATTR_WARN_UNUSED_RESULT
-{
-  if (list != NULL) {
-    typval_T tv = (typval_T) {
-      .v_type = VAR_LIST,
-      .vval = { .v_list = list }
-    };
-    return set_ref_in_item(&tv, copyID, NULL, NULL);
-  }
-  return false;
-}
-
-/// Mark all lists and dicts referenced in given dict and the dict itself
-///
-/// @return  true if setting references failed somehow.
-static inline bool set_ref_dict(dict_T *dict, int copyID)
-  FUNC_ATTR_WARN_UNUSED_RESULT
-{
-  if (dict != NULL) {
-    typval_T tv = (typval_T) {
-      .v_type = VAR_DICT,
-      .vval = { .v_dict = dict }
-    };
-    return set_ref_in_item(&tv, copyID, NULL, NULL);
-  }
-  return false;
-}
-
 /// Get the key for #{key: val} into "tv" and advance "arg".
 ///
 /// @return  FAIL when there is no valid key.
diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c
index 13cd3274dd..1ff8716763 100644
--- a/src/nvim/eval/decode.c
+++ b/src/nvim/eval/decode.c
@@ -1,5 +1,4 @@
 #include <assert.h>
-#include <msgpack/object.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -7,6 +6,7 @@
 #include <string.h>
 
 #include "klib/kvec.h"
+#include "mpack/object.h"
 #include "nvim/ascii_defs.h"
 #include "nvim/charset.h"
 #include "nvim/eval.h"
@@ -885,173 +885,250 @@ json_decode_string_ret:
 
 #undef DICT_LEN
 
-/// Convert msgpack object to a Vimscript one
-int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+static void positive_integer_to_special_typval(typval_T *rettv, uint64_t val)
 {
-  switch (mobj.type) {
-  case MSGPACK_OBJECT_NIL:
+  if (val <= VARNUMBER_MAX) {
     *rettv = (typval_T) {
+      .v_type = VAR_NUMBER,
+      .v_lock = VAR_UNLOCKED,
+      .vval = { .v_number = (varnumber_T)val },
+    };
+  } else {
+    list_T *const list = tv_list_alloc(4);
+    tv_list_ref(list);
+    create_special_dict(rettv, kMPInteger, ((typval_T) {
+      .v_type = VAR_LIST,
+      .v_lock = VAR_UNLOCKED,
+      .vval = { .v_list = list },
+    }));
+    tv_list_append_number(list, 1);
+    tv_list_append_number(list, (varnumber_T)((val >> 62) & 0x3));
+    tv_list_append_number(list, (varnumber_T)((val >> 31) & 0x7FFFFFFF));
+    tv_list_append_number(list, (varnumber_T)(val & 0x7FFFFFFF));
+  }
+}
+
+static void typval_parse_enter(mpack_parser_t *parser, mpack_node_t *node)
+{
+  typval_T *result = NULL;
+
+  mpack_node_t *parent = MPACK_PARENT_NODE(node);
+  if (parent) {
+    switch (parent->tok.type) {
+    case MPACK_TOKEN_ARRAY: {
+      list_T *list = parent->data[1].p;
+      result = tv_list_append_owned_tv(list, (typval_T) { .v_type = VAR_UNKNOWN });
+      break;
+    }
+    case MPACK_TOKEN_MAP: {
+      typval_T(*items)[2] = parent->data[1].p;
+      result = &items[parent->pos][parent->key_visited];
+      break;
+    }
+
+    case MPACK_TOKEN_STR:
+    case MPACK_TOKEN_BIN:
+    case MPACK_TOKEN_EXT:
+      assert(node->tok.type == MPACK_TOKEN_CHUNK);
+      break;
+
+    default:
+      abort();
+    }
+  } else {
+    result = parser->data.p;
+  }
+
+  // for types that are completed in typval_parse_exit
+  node->data[0].p = result;
+  node->data[1].p = NULL;  // free on error if non-NULL
+
+  switch (node->tok.type) {
+  case MPACK_TOKEN_NIL:
+    *result = (typval_T) {
       .v_type = VAR_SPECIAL,
       .v_lock = VAR_UNLOCKED,
       .vval = { .v_special = kSpecialVarNull },
     };
     break;
-  case MSGPACK_OBJECT_BOOLEAN:
-    *rettv = (typval_T) {
+  case MPACK_TOKEN_BOOLEAN:
+    *result = (typval_T) {
       .v_type = VAR_BOOL,
       .v_lock = VAR_UNLOCKED,
       .vval = {
-        .v_bool = mobj.via.boolean ? kBoolVarTrue : kBoolVarFalse
+        .v_bool = mpack_unpack_boolean(node->tok) ? kBoolVarTrue : kBoolVarFalse
       },
     };
     break;
-  case MSGPACK_OBJECT_POSITIVE_INTEGER:
-    if (mobj.via.u64 <= VARNUMBER_MAX) {
-      *rettv = (typval_T) {
-        .v_type = VAR_NUMBER,
-        .v_lock = VAR_UNLOCKED,
-        .vval = { .v_number = (varnumber_T)mobj.via.u64 },
-      };
-    } else {
-      list_T *const list = tv_list_alloc(4);
-      tv_list_ref(list);
-      create_special_dict(rettv, kMPInteger, ((typval_T) {
-        .v_type = VAR_LIST,
-        .v_lock = VAR_UNLOCKED,
-        .vval = { .v_list = list },
-      }));
-      uint64_t n = mobj.via.u64;
-      tv_list_append_number(list, 1);
-      tv_list_append_number(list, (varnumber_T)((n >> 62) & 0x3));
-      tv_list_append_number(list, (varnumber_T)((n >> 31) & 0x7FFFFFFF));
-      tv_list_append_number(list, (varnumber_T)(n & 0x7FFFFFFF));
-    }
+  case MPACK_TOKEN_SINT: {
+    *result = (typval_T) {
+      .v_type = VAR_NUMBER,
+      .v_lock = VAR_UNLOCKED,
+      .vval = { .v_number = mpack_unpack_sint(node->tok) },
+    };
     break;
-  case MSGPACK_OBJECT_NEGATIVE_INTEGER:
-    if (mobj.via.i64 >= VARNUMBER_MIN) {
-      *rettv = (typval_T) {
-        .v_type = VAR_NUMBER,
-        .v_lock = VAR_UNLOCKED,
-        .vval = { .v_number = (varnumber_T)mobj.via.i64 },
-      };
-    } else {
-      list_T *const list = tv_list_alloc(4);
-      tv_list_ref(list);
-      create_special_dict(rettv, kMPInteger, ((typval_T) {
-        .v_type = VAR_LIST,
-        .v_lock = VAR_UNLOCKED,
-        .vval = { .v_list = list },
-      }));
-      uint64_t n = -((uint64_t)mobj.via.i64);
-      tv_list_append_number(list, -1);
-      tv_list_append_number(list, (varnumber_T)((n >> 62) & 0x3));
-      tv_list_append_number(list, (varnumber_T)((n >> 31) & 0x7FFFFFFF));
-      tv_list_append_number(list, (varnumber_T)(n & 0x7FFFFFFF));
-    }
+  }
+  case MPACK_TOKEN_UINT:
+    positive_integer_to_special_typval(result, mpack_unpack_uint(node->tok));
     break;
-  case MSGPACK_OBJECT_FLOAT32:
-  case MSGPACK_OBJECT_FLOAT64:
-    *rettv = (typval_T) {
+  case MPACK_TOKEN_FLOAT:
+    *result = (typval_T) {
       .v_type = VAR_FLOAT,
       .v_lock = VAR_UNLOCKED,
-      .vval = { .v_float = mobj.via.f64 },
+      .vval = { .v_float = mpack_unpack_float(node->tok) },
     };
     break;
-  case MSGPACK_OBJECT_STR:
-  case MSGPACK_OBJECT_BIN:
-    *rettv = decode_string(mobj.via.bin.ptr, mobj.via.bin.size, false, false);
+
+  case MPACK_TOKEN_BIN:
+  case MPACK_TOKEN_STR:
+  case MPACK_TOKEN_EXT:
+    // actually converted in typval_parse_exit after the data chunks
+    node->data[1].p = xmallocz(node->tok.length);
     break;
-  case MSGPACK_OBJECT_ARRAY: {
-    list_T *const list = tv_list_alloc((ptrdiff_t)mobj.via.array.size);
+  case MPACK_TOKEN_CHUNK: {
+    char *data = parent->data[1].p;
+    memcpy(data + parent->pos,
+           node->tok.data.chunk_ptr, node->tok.length);
+    break;
+  }
+
+  case MPACK_TOKEN_ARRAY: {
+    list_T *const list = tv_list_alloc((ptrdiff_t)node->tok.length);
     tv_list_ref(list);
-    *rettv = (typval_T) {
+    *result = (typval_T) {
       .v_type = VAR_LIST,
       .v_lock = VAR_UNLOCKED,
       .vval = { .v_list = list },
     };
-    for (size_t i = 0; i < mobj.via.array.size; i++) {
-      // Not populated yet, need to create list item to push.
-      tv_list_append_owned_tv(list, (typval_T) { .v_type = VAR_UNKNOWN });
-      if (msgpack_to_vim(mobj.via.array.ptr[i],
-                         TV_LIST_ITEM_TV(tv_list_last(list)))
-          == FAIL) {
-        return FAIL;
-      }
+    node->data[1].p = list;
+    break;
+  }
+  case MPACK_TOKEN_MAP:
+    // we don't know if this will be safe to convert to a typval dict yet
+    node->data[1].p = xmallocz(node->tok.length * 2 * sizeof(typval_T));
+    break;
+  }
+}
+
+/// free node which was entered but never exited, due to a nested error
+///
+/// Don't bother with typvals as these will be GC:d eventually
+void typval_parser_error_free(mpack_parser_t *parser)
+{
+  for (uint32_t i = 0; i < parser->size; i++) {
+    mpack_node_t *node = &parser->items[i];
+    switch (node->tok.type) {
+    case MPACK_TOKEN_BIN:
+    case MPACK_TOKEN_STR:
+    case MPACK_TOKEN_EXT:
+    case MPACK_TOKEN_MAP:
+      XFREE_CLEAR(node->data[1].p);
+      break;
+    default:
+      break;
     }
+  }
+}
+
+static void typval_parse_exit(mpack_parser_t *parser, mpack_node_t *node)
+{
+  typval_T *result = node->data[0].p;
+  switch (node->tok.type) {
+  case MPACK_TOKEN_BIN:
+  case MPACK_TOKEN_STR:
+    *result = decode_string(node->data[1].p, node->tok.length, false, true);
+    node->data[1].p = NULL;
     break;
+
+  case MPACK_TOKEN_EXT: {
+    list_T *const list = tv_list_alloc(2);
+    tv_list_ref(list);
+    tv_list_append_number(list, node->tok.data.ext_type);
+    list_T *const ext_val_list = tv_list_alloc(kListLenMayKnow);
+    tv_list_append_list(list, ext_val_list);
+    create_special_dict(result, kMPExt, ((typval_T) { .v_type = VAR_LIST,
+                                                      .v_lock = VAR_UNLOCKED,
+                                                      .vval = { .v_list = list } }));
+    // TODO(bfredl): why not use BLOB?
+    encode_list_write((void *)ext_val_list, node->data[1].p, node->tok.length);
+    XFREE_CLEAR(node->data[1].p);
   }
-  case MSGPACK_OBJECT_MAP: {
-    for (size_t i = 0; i < mobj.via.map.size; i++) {
-      if ((mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR
-           && mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_BIN)
-          || mobj.via.map.ptr[i].key.via.str.size == 0
-          || memchr(mobj.via.map.ptr[i].key.via.str.ptr, NUL,
-                    mobj.via.map.ptr[i].key.via.str.size) != NULL) {
+  break;
+
+  case MPACK_TOKEN_MAP: {
+    typval_T(*items)[2] = node->data[1].p;
+    for (size_t i = 0; i < node->tok.length; i++) {
+      typval_T *key = &items[i][0];
+      if (key->v_type != VAR_STRING
+          || key->vval.v_string == NULL
+          || key->vval.v_string[0] == NUL) {
         goto msgpack_to_vim_generic_map;
       }
     }
     dict_T *const dict = tv_dict_alloc();
     dict->dv_refcount++;
-    *rettv = (typval_T) {
+    *result = (typval_T) {
       .v_type = VAR_DICT,
       .v_lock = VAR_UNLOCKED,
       .vval = { .v_dict = dict },
     };
-    for (size_t i = 0; i < mobj.via.map.size; i++) {
-      dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key)
-                                      + mobj.via.map.ptr[i].key.via.str.size);
-      memcpy(&di->di_key[0], mobj.via.map.ptr[i].key.via.str.ptr,
-             mobj.via.map.ptr[i].key.via.str.size);
+    for (size_t i = 0; i < node->tok.length; i++) {
+      char *key = items[i][0].vval.v_string;
+      size_t keylen = strlen(key);
+      dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key) + keylen);
+      memcpy(&di->di_key[0], key, keylen);
       di->di_tv.v_type = VAR_UNKNOWN;
       if (tv_dict_add(dict, di) == FAIL) {
         // Duplicate key: fallback to generic map
-        tv_clear(rettv);
+        TV_DICT_ITER(dict, d, {
+            d->di_tv.v_type = VAR_SPECIAL;  // don't free values in tv_clear(), they will be reused
+            d->di_tv.vval.v_special = kSpecialVarNull;
+          });
+        tv_clear(result);
         xfree(di);
         goto msgpack_to_vim_generic_map;
       }
-      if (msgpack_to_vim(mobj.via.map.ptr[i].val, &di->di_tv) == FAIL) {
-        return FAIL;
-      }
+      di->di_tv = items[i][1];
+    }
+    for (size_t i = 0; i < node->tok.length; i++) {
+      xfree(items[i][0].vval.v_string);
     }
+    XFREE_CLEAR(node->data[1].p);
     break;
 msgpack_to_vim_generic_map: {}
-    list_T *const list = decode_create_map_special_dict(rettv, (ptrdiff_t)mobj.via.map.size);
-    for (size_t i = 0; i < mobj.via.map.size; i++) {
+    list_T *const list = decode_create_map_special_dict(result, node->tok.length);
+    for (size_t i = 0; i < node->tok.length; i++) {
       list_T *const kv_pair = tv_list_alloc(2);
       tv_list_append_list(list, kv_pair);
 
-      typval_T key_tv = { .v_type = VAR_UNKNOWN };
-      if (msgpack_to_vim(mobj.via.map.ptr[i].key, &key_tv) == FAIL) {
-        tv_clear(&key_tv);
-        return FAIL;
-      }
-      tv_list_append_owned_tv(kv_pair, key_tv);
-
-      typval_T val_tv = { .v_type = VAR_UNKNOWN };
-      if (msgpack_to_vim(mobj.via.map.ptr[i].val, &val_tv) == FAIL) {
-        tv_clear(&val_tv);
-        return FAIL;
-      }
-      tv_list_append_owned_tv(kv_pair, val_tv);
+      tv_list_append_owned_tv(kv_pair, items[i][0]);
+      tv_list_append_owned_tv(kv_pair, items[i][1]);
     }
+    XFREE_CLEAR(node->data[1].p);
     break;
   }
-  case MSGPACK_OBJECT_EXT: {
-    list_T *const list = tv_list_alloc(2);
-    tv_list_ref(list);
-    tv_list_append_number(list, mobj.via.ext.type);
-    list_T *const ext_val_list = tv_list_alloc(kListLenMayKnow);
-    tv_list_append_list(list, ext_val_list);
-    create_special_dict(rettv, kMPExt, ((typval_T) { .v_type = VAR_LIST,
-                                                     .v_lock = VAR_UNLOCKED,
-                                                     .vval = { .v_list = list } }));
-    if (encode_list_write((void *)ext_val_list, mobj.via.ext.ptr,
-                          mobj.via.ext.size) == -1) {
-      return FAIL;
-    }
+
+  default:
+    // other kinds are handled completely in typval_parse_enter,
     break;
   }
+}
+
+int mpack_parse_typval(mpack_parser_t *parser, const char **data, size_t *size)
+{
+  return mpack_parse(parser, data, size, typval_parse_enter, typval_parse_exit);
+}
+
+int unpack_typval(const char **data, size_t *size, typval_T *ret)
+{
+  ret->v_type = VAR_UNKNOWN;
+  mpack_parser_t parser;
+  mpack_parser_init(&parser, 0);
+  parser.data.p = ret;
+  int status = mpack_parse_typval(&parser, data, size);
+  if (status != MPACK_OK) {
+    typval_parser_error_free(&parser);
+    tv_clear(ret);
   }
-  return OK;
+  return status;
 }
diff --git a/src/nvim/eval/decode.h b/src/nvim/eval/decode.h
index c0d10a469a..485cc65561 100644
--- a/src/nvim/eval/decode.h
+++ b/src/nvim/eval/decode.h
@@ -1,8 +1,8 @@
 #pragma once
 
-#include <msgpack.h>  // IWYU pragma: keep
 #include <stddef.h>  // IWYU pragma: keep
 
+#include "mpack/object.h"
 #include "nvim/eval/typval_defs.h"  // IWYU pragma: keep
 #include "nvim/types_defs.h"  // IWYU pragma: keep
 
diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c
index 92c5aaeffd..79f334601d 100644
--- a/src/nvim/eval/encode.c
+++ b/src/nvim/eval/encode.c
@@ -55,11 +55,11 @@ int encode_blob_write(void *const data, const char *const buf, const size_t len)
 }
 
 /// Msgpack callback for writing to readfile()-style list
-int encode_list_write(void *const data, const char *const buf, const size_t len)
+void encode_list_write(void *const data, const char *const buf, const size_t len)
   FUNC_ATTR_NONNULL_ARG(1)
 {
   if (len == 0) {
-    return 0;
+    return;
   }
   list_T *const list = (list_T *)data;
   const char *const end = buf + len;
@@ -97,7 +97,6 @@ int encode_list_write(void *const data, const char *const buf, const size_t len)
   if (line_end == end) {
     tv_list_append_allocated_string(list, NULL);
   }
-  return 0;
 }
 
 /// Abort conversion to string after a recursion error.
diff --git a/src/nvim/eval/encode.h b/src/nvim/eval/encode.h
index efccfcb5a6..2bacc82b0d 100644
--- a/src/nvim/eval/encode.h
+++ b/src/nvim/eval/encode.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <msgpack/pack.h>
 #include <string.h>
 
 #include "nvim/eval/typval_defs.h"
diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c
index d2f6af4d9e..2f51532ec4 100644
--- a/src/nvim/eval/funcs.c
+++ b/src/nvim/eval/funcs.c
@@ -4,9 +4,6 @@
 #include <inttypes.h>
 #include <limits.h>
 #include <math.h>
-#include <msgpack/object.h>
-#include <msgpack/pack.h>
-#include <msgpack/unpack.h>
 #include <signal.h>
 #include <stdarg.h>
 #include <stddef.h>
@@ -18,6 +15,7 @@
 #include <uv.h>
 
 #include "auto/config.h"
+#include "mpack/object.h"
 #include "nvim/api/private/converter.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/dispatch.h"
@@ -5723,36 +5721,24 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
   }
 }
 
-static int msgpackparse_convert_item(const msgpack_object data, const msgpack_unpack_return result,
-                                     list_T *const ret_list, const bool fail_if_incomplete)
-  FUNC_ATTR_NONNULL_ALL
+static void emsg_mpack_error(int status)
 {
-  switch (result) {
-  case MSGPACK_UNPACK_PARSE_ERROR:
+  switch (status) {
+  case MPACK_ERROR:
     semsg(_(e_invarg2), "Failed to parse msgpack string");
-    return FAIL;
-  case MSGPACK_UNPACK_NOMEM_ERROR:
-    emsg(_(e_outofmem));
-    return FAIL;
-  case MSGPACK_UNPACK_CONTINUE:
-    if (fail_if_incomplete) {
-      semsg(_(e_invarg2), "Incomplete msgpack string");
-      return FAIL;
-    }
-    return NOTDONE;
-  case MSGPACK_UNPACK_SUCCESS: {
-    typval_T tv = { .v_type = VAR_UNKNOWN };
-    if (msgpack_to_vim(data, &tv) == FAIL) {
-      semsg(_(e_invarg2), "Failed to convert msgpack string");
-      return FAIL;
-    }
-    tv_list_append_owned_tv(ret_list, tv);
-    return OK;
-  }
-  case MSGPACK_UNPACK_EXTRA_BYTES:
-    abort();
+    break;
+
+  case MPACK_EOF:
+    semsg(_(e_invarg2), "Incomplete msgpack string");
+    break;
+
+  case MPACK_NOMEM:
+    semsg(_(e_invarg2), "object was too deep to unpack");
+    break;
+
+  default:
+    break;
   }
-  UNREACHABLE;
 }
 
 static void msgpackparse_unpack_list(const list_T *const list, list_T *const ret_list)
@@ -5766,48 +5752,57 @@ static void msgpackparse_unpack_list(const list_T *const list, list_T *const ret
     return;
   }
   ListReaderState lrstate = encode_init_lrstate(list);
-  msgpack_unpacker *const unpacker = msgpack_unpacker_new(IOSIZE);
-  if (unpacker == NULL) {
-    emsg(_(e_outofmem));
-    return;
-  }
-  msgpack_unpacked unpacked;
-  msgpack_unpacked_init(&unpacked);
+  char *buf = alloc_block();
+  size_t buf_size = 0;
+
+  typval_T cur_item = { .v_type = VAR_UNKNOWN };
+  mpack_parser_t parser;
+  mpack_parser_init(&parser, 0);
+  parser.data.p = &cur_item;
+
+  int status = MPACK_OK;
   while (true) {
-    if (!msgpack_unpacker_reserve_buffer(unpacker, IOSIZE)) {
-      emsg(_(e_outofmem));
-      goto end;
-    }
     size_t read_bytes;
-    const int rlret = encode_read_from_list(&lrstate, msgpack_unpacker_buffer(unpacker), IOSIZE,
+    const int rlret = encode_read_from_list(&lrstate, buf + buf_size, ARENA_BLOCK_SIZE - buf_size,
                                             &read_bytes);
     if (rlret == FAIL) {
       semsg(_(e_invarg2), "List item is not a string");
       goto end;
     }
-    msgpack_unpacker_buffer_consumed(unpacker, read_bytes);
-    if (read_bytes == 0) {
-      break;
-    }
-    while (unpacker->off < unpacker->used) {
-      const msgpack_unpack_return result
-        = msgpack_unpacker_next(unpacker, &unpacked);
-      const int conv_result = msgpackparse_convert_item(unpacked.data, result,
-                                                        ret_list, rlret == OK);
-      if (conv_result == NOTDONE) {
+    buf_size += read_bytes;
+
+    const char *ptr = buf;
+    while (buf_size) {
+      status = mpack_parse_typval(&parser, &ptr, &buf_size);
+      if (status == MPACK_OK) {
+        tv_list_append_owned_tv(ret_list, cur_item);
+        cur_item.v_type = VAR_UNKNOWN;
+      } else {
         break;
-      } else if (conv_result == FAIL) {
-        goto end;
       }
     }
+
     if (rlret == OK) {
       break;
     }
+
+    if (status == MPACK_EOF) {
+      // move remaining data to front of buffer
+      if (buf_size && ptr > buf) {
+        memmove(buf, ptr, buf_size);
+      }
+    } else if (status != MPACK_OK) {
+      break;
+    }
+  }
+
+  if (status != MPACK_OK) {
+    typval_parser_error_free(&parser);
+    emsg_mpack_error(status);
   }
 
 end:
-  msgpack_unpacker_free(unpacker);
-  msgpack_unpacked_destroy(&unpacked);
+  free_block(buf);
 }
 
 static void msgpackparse_unpack_blob(const blob_T *const blob, list_T *const ret_list)
@@ -5817,18 +5812,19 @@ static void msgpackparse_unpack_blob(const blob_T *const blob, list_T *const ret
   if (len == 0) {
     return;
   }
-  msgpack_unpacked unpacked;
-  msgpack_unpacked_init(&unpacked);
-  for (size_t offset = 0; offset < (size_t)len;) {
-    const msgpack_unpack_return result
-      = msgpack_unpack_next(&unpacked, blob->bv_ga.ga_data, (size_t)len, &offset);
-    if (msgpackparse_convert_item(unpacked.data, result, ret_list, true)
-        != OK) {
-      break;
+
+  const char *data = blob->bv_ga.ga_data;
+  size_t remaining = (size_t)len;
+  while (remaining) {
+    typval_T tv;
+    int status = unpack_typval(&data, &remaining, &tv);
+    if (status != MPACK_OK) {
+      emsg_mpack_error(status);
+      return;
     }
-  }
 
-  msgpack_unpacked_destroy(&unpacked);
+    tv_list_append_owned_tv(ret_list, tv);
+  }
 }
 
 /// "msgpackparse" function
diff --git a/src/nvim/eval/typval.c b/src/nvim/eval/typval.c
index c00abe452c..e7b6a0feee 100644
--- a/src/nvim/eval/typval.c
+++ b/src/nvim/eval/typval.c
@@ -485,13 +485,14 @@ void tv_list_append_tv(list_T *const l, typval_T *const tv)
 /// Like tv_list_append_tv(), but tv is moved to a list
 ///
 /// This means that it is no longer valid to use contents of the typval_T after
-/// function exits.
-void tv_list_append_owned_tv(list_T *const l, typval_T tv)
+/// function exits. A pointer is returned to the allocated typval which can be used
+typval_T *tv_list_append_owned_tv(list_T *const l, typval_T tv)
   FUNC_ATTR_NONNULL_ALL
 {
   listitem_T *const li = tv_list_item_alloc();
   *TV_LIST_ITEM_TV(li) = tv;
   tv_list_append(l, li);
+  return TV_LIST_ITEM_TV(li);
 }
 
 /// Append a list to a list as one item
diff --git a/src/nvim/ex_cmds.c b/src/nvim/ex_cmds.c
index 1060650bf2..6a7a713d39 100644
--- a/src/nvim/ex_cmds.c
+++ b/src/nvim/ex_cmds.c
@@ -3068,8 +3068,8 @@ void sub_get_replacement(SubReplacementString *const ret_sub)
 void sub_set_replacement(SubReplacementString sub)
 {
   xfree(old_sub.sub);
-  if (sub.additional_elements != old_sub.additional_elements) {
-    tv_list_unref(old_sub.additional_elements);
+  if (sub.additional_data != old_sub.additional_data) {
+    xfree(old_sub.additional_data);
   }
   old_sub = sub;
 }
@@ -3395,7 +3395,7 @@ static int do_sub(exarg_T *eap, const proftime_T timeout, const int cmdpreview_n
       sub_set_replacement((SubReplacementString) {
         .sub = xstrdup(sub),
         .timestamp = os_time(),
-        .additional_elements = NULL,
+        .additional_data = NULL,
       });
     }
   } else if (!eap->skip) {    // use previous pattern and substitution
diff --git a/src/nvim/ex_cmds_defs.h b/src/nvim/ex_cmds_defs.h
index 07f92ca169..4924e86470 100644
--- a/src/nvim/ex_cmds_defs.h
+++ b/src/nvim/ex_cmds_defs.h
@@ -233,5 +233,5 @@ typedef struct {
 typedef struct {
   char *sub;            ///< Previous replacement string.
   Timestamp timestamp;  ///< Time when it was last set.
-  list_T *additional_elements;  ///< Additional data left from ShaDa file.
+  AdditionalData *additional_data;  ///< Additional data left from ShaDa file.
 } SubReplacementString;
diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
index 9ce9f3d7a6..4e7de7adb4 100644
--- a/src/nvim/generators/c_grammar.lua
+++ b/src/nvim/generators/c_grammar.lua
@@ -91,7 +91,9 @@ local c_proto = Ct(
     * (P(';') + (P('{') * nl + (impl_line ^ 0) * P('}')))
 )
 
-local c_field = Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * P(';') * fill)
+local dict_key = P('DictKey(') * Cg(rep1(any - P(')')), 'dict_key') * P(')')
+local keyset_field =
+  Ct(Cg(c_id, 'type') * ws * Cg(c_id, 'name') * fill * (dict_key ^ -1) * fill * P(';') * fill)
 local c_keyset = Ct(
   P('typedef')
     * ws
@@ -99,7 +101,7 @@ local c_keyset = Ct(
     * fill
     * P('{')
     * fill
-    * Cg(Ct(c_field ^ 1), 'fields')
+    * Cg(Ct(keyset_field ^ 1), 'fields')
     * P('}')
     * fill
     * P('Dict')
diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
index 61c80a3d2e..3567831c41 100644
--- a/src/nvim/generators/gen_api_dispatch.lua
+++ b/src/nvim/generators/gen_api_dispatch.lua
@@ -72,14 +72,19 @@ local keysets = {}
 local function add_keyset(val)
   local keys = {}
   local types = {}
+  local c_names = {}
   local is_set_name = 'is_set__' .. val.keyset_name .. '_'
   local has_optional = false
   for i, field in ipairs(val.fields) do
+    local dict_key = field.dict_key or field.name
     if field.type ~= 'Object' then
-      types[field.name] = field.type
+      types[dict_key] = field.type
     end
     if field.name ~= is_set_name and field.type ~= 'OptionalKeys' then
-      table.insert(keys, field.name)
+      table.insert(keys, dict_key)
+      if dict_key ~= field.name then
+        c_names[dict_key] = field.name
+      end
     else
       if i > 1 then
         error("'is_set__{type}_' must be first if present")
@@ -94,6 +99,7 @@ local function add_keyset(val)
   table.insert(keysets, {
     name = val.keyset_name,
     keys = keys,
+    c_names = c_names,
     types = types,
     has_optional = has_optional,
   })
@@ -332,19 +338,6 @@ output:write([[
 keysets_defs:write('// IWYU pragma: private, include "nvim/api/private/dispatch.h"\n\n')
 
 for _, k in ipairs(keysets) do
-  local c_name = {}
-
-  for i = 1, #k.keys do
-    -- some keys, like "register" are c keywords and get
-    -- escaped with a trailing _ in the struct.
-    if vim.endswith(k.keys[i], '_') then
-      local orig = k.keys[i]
-      k.keys[i] = string.sub(k.keys[i], 1, #k.keys[i] - 1)
-      c_name[k.keys[i]] = orig
-      k.types[k.keys[i]] = k.types[orig]
-    end
-  end
-
   local neworder, hashfun = hashy.hashy_hash(k.name, k.keys, function(idx)
     return k.name .. '_table[' .. idx .. '].str'
   end)
@@ -354,6 +347,8 @@ for _, k in ipairs(keysets) do
   local function typename(type)
     if type == 'HLGroupID' then
       return 'kObjectTypeInteger'
+    elseif type == 'StringArray' then
+      return 'kUnpackTypeStringArray'
     elseif type ~= nil then
       return 'kObjectType' .. type
     else
@@ -374,7 +369,7 @@ for _, k in ipairs(keysets) do
         .. '", offsetof(KeyDict_'
         .. k.name
         .. ', '
-        .. (c_name[key] or key)
+        .. (k.c_names[key] or key)
         .. '), '
         .. typename(k.types[key])
         .. ', '
diff --git a/src/nvim/generators/hashy.lua b/src/nvim/generators/hashy.lua
index 711e695742..ea35064962 100644
--- a/src/nvim/generators/hashy.lua
+++ b/src/nvim/generators/hashy.lua
@@ -73,11 +73,15 @@ function M.switcher(put, tab, maxlen, worst_buck_size)
           vim.list_extend(neworder, buck)
           local endidx = #neworder
           put("      case '" .. c .. "': ")
-          put('low = ' .. startidx .. '; ')
-          if bucky then
-            put('high = ' .. endidx .. '; ')
+          if len == 1 then
+            put('return ' .. startidx .. ';\n')
+          else
+            put('low = ' .. startidx .. '; ')
+            if bucky then
+              put('high = ' .. endidx .. '; ')
+            end
+            put 'break;\n'
           end
-          put 'break;\n'
         end
         put '      default: break;\n'
         put '    }\n    '
@@ -105,13 +109,19 @@ function M.hashy_hash(name, strings, access)
   end
   local len_pos_buckets, maxlen, worst_buck_size = M.build_pos_hash(strings)
   put('int ' .. name .. '_hash(const char *str, size_t len)\n{\n')
-  if worst_buck_size > 1 then
+  if maxlen == 1 then
+    put('\n') -- nothing
+  elseif worst_buck_size > 1 then
     put('  int low = 0, high = 0;\n')
   else
     put('  int low = -1;\n')
   end
   local neworder = M.switcher(put, len_pos_buckets, maxlen, worst_buck_size)
-  if worst_buck_size > 1 then
+  if maxlen == 1 then
+    put([[
+  return -1;
+]])
+  elseif worst_buck_size > 1 then
     put([[
   for (int i = low; i < high; i++) {
     if (!memcmp(str, ]] .. access('i') .. [[, len)) {
diff --git a/src/nvim/main.c b/src/nvim/main.c
index 72888757be..c507a201b0 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -6,7 +6,6 @@
 #endif
 #include <assert.h>
 #include <limits.h>
-#include <msgpack/pack.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
diff --git a/src/nvim/mark.c b/src/nvim/mark.c
index fae28ef6e1..a09ade2b03 100644
--- a/src/nvim/mark.c
+++ b/src/nvim/mark.c
@@ -71,7 +71,7 @@ int setmark(int c)
 /// Free fmark_T item
 void free_fmark(fmark_T fm)
 {
-  tv_dict_unref(fm.additional_data);
+  xfree(fm.additional_data);
 }
 
 /// Free xfmark_T item
diff --git a/src/nvim/mark_defs.h b/src/nvim/mark_defs.h
index ad437e8ce1..f953e26e4e 100644
--- a/src/nvim/mark_defs.h
+++ b/src/nvim/mark_defs.h
@@ -1,7 +1,11 @@
 #pragma once
 
-#include "nvim/eval/typval_defs.h"
+#include <stdbool.h>
+
+#include "nvim/func_attr.h"
 #include "nvim/os/time_defs.h"
+#include "nvim/pos_defs.h"
+#include "nvim/types_defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "mark_defs.h.inline.generated.h"
@@ -75,7 +79,7 @@ typedef struct {
   int fnum;             ///< File number.
   Timestamp timestamp;  ///< Time when this mark was last set.
   fmarkv_T view;  ///< View the mark was created on
-  dict_T *additional_data;  ///< Additional data from ShaDa file.
+  AdditionalData *additional_data;  ///< Additional data from ShaDa file.
 } fmark_T;
 
 #define INIT_FMARK { { 0, 0, 0 }, 0, 0, INIT_FMARKV, NULL }
diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 0930f0c7b3..e4aef4063d 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -1,8 +1,5 @@
 #include <assert.h>
 #include <inttypes.h>
-#include <msgpack/object.h>
-#include <msgpack/sbuffer.h>
-#include <msgpack/unpack.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/src/nvim/msgpack_rpc/channel_defs.h b/src/nvim/msgpack_rpc/channel_defs.h
index 7dc1374964..6c8b05b8f3 100644
--- a/src/nvim/msgpack_rpc/channel_defs.h
+++ b/src/nvim/msgpack_rpc/channel_defs.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <msgpack.h>
 #include <stdbool.h>
 #include <uv.h>
 
diff --git a/src/nvim/msgpack_rpc/unpacker.c b/src/nvim/msgpack_rpc/unpacker.c
index 28d27e8268..9372754b01 100644
--- a/src/nvim/msgpack_rpc/unpacker.c
+++ b/src/nvim/msgpack_rpc/unpacker.c
@@ -11,6 +11,7 @@
 #include "nvim/memory.h"
 #include "nvim/msgpack_rpc/channel_defs.h"
 #include "nvim/msgpack_rpc/unpacker.h"
+#include "nvim/strings.h"
 #include "nvim/ui_client.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
@@ -206,6 +207,7 @@ bool unpacker_parse_header(Unpacker *p)
 
   assert(!ERROR_SET(&p->unpack_error));
 
+  // TODO(bfredl): eliminate p->reader, we can use mpack_rtoken directly
 #define NEXT(tok) \
   result = mpack_read(&p->reader, &data, &size, &tok); \
   if (result) { goto error; }
@@ -522,3 +524,197 @@ bool unpacker_parse_redraw(Unpacker *p)
     abort();
   }
 }
+
+/// require complete string. safe to use e.g. in shada as we have loaded a complete shada item into
+/// a linear buffer.
+///
+/// data and size are preserved in cause of failure
+///
+/// @return "data" is NULL exact when failure (non-null data and size=0 for
+/// valid empty string)
+String unpack_string(const char **data, size_t *size)
+{
+  const char *data2 = *data;
+  size_t size2 = *size;
+  mpack_token_t tok;
+
+  // TODO(bfredl): this code is hot a f, specialize!
+  int result = mpack_rtoken(&data2, &size2, &tok);
+  if (result || (tok.type != MPACK_TOKEN_STR && tok.type != MPACK_TOKEN_BIN)) {
+    return (String)STRING_INIT;
+  }
+  if (*size < tok.length) {
+    // result = MPACK_EOF;
+    return (String)STRING_INIT;
+  }
+  (*data) = data2 + tok.length;
+  (*size) = size2 - tok.length;
+  return cbuf_as_string((char *)data2, tok.length);
+}
+
+/// @return -1 if not an array or EOF. otherwise size of valid array
+ssize_t unpack_array(const char **data, size_t *size)
+{
+  // TODO(bfredl): this code is hot, specialize!
+  mpack_token_t tok;
+  int result = mpack_rtoken(data, size, &tok);
+  if (result || tok.type != MPACK_TOKEN_ARRAY) {
+    return -1;
+  }
+  return tok.length;
+}
+
+/// does not keep "data" untouched on failure
+bool unpack_integer(const char **data, size_t *size, Integer *res)
+{
+  mpack_token_t tok;
+  int result = mpack_rtoken(data, size, &tok);
+  if (result) {
+    return false;
+  }
+  return unpack_uint_or_sint(tok, res);
+}
+
+bool unpack_uint_or_sint(mpack_token_t tok, Integer *res)
+{
+  if (tok.type == MPACK_TOKEN_UINT) {
+    *res = (Integer)mpack_unpack_uint(tok);
+    return true;
+  } else if (tok.type == MPACK_TOKEN_SINT) {
+    *res = (Integer)mpack_unpack_sint(tok);
+    return true;
+  }
+  return false;
+}
+
+static void parse_nop(mpack_parser_t *parser, mpack_node_t *node)
+{
+}
+
+int unpack_skip(const char **data, size_t *size)
+{
+  mpack_parser_t parser;
+  mpack_parser_init(&parser, 0);
+
+  return mpack_parse(&parser, data, size, parse_nop, parse_nop);
+}
+
+void push_additional_data(AdditionalDataBuilder *ad, const char *data, size_t size)
+{
+  if (kv_size(*ad) == 0) {
+    AdditionalData init = { 0 };
+    kv_concat_len(*ad, &init, sizeof(init));
+  }
+  AdditionalData *a = (AdditionalData *)ad->items;
+  a->nitems++;
+  a->nbytes += (uint32_t)size;
+  kv_concat_len(*ad, data, size);
+}
+
+// currently only used for shada, so not re-entrant like unpacker_parse_redraw
+bool unpack_keydict(void *retval, FieldHashfn hashy, AdditionalDataBuilder *ad, const char **data,
+                    size_t *restrict size, char **error)
+{
+  OptKeySet *ks = (OptKeySet *)retval;
+  mpack_token_t tok;
+
+  int result = mpack_rtoken(data, size, &tok);
+  if (result || tok.type != MPACK_TOKEN_MAP) {
+    *error = xstrdup("is not a dictionary");
+    return false;
+  }
+
+  size_t map_size = tok.length;
+
+  for (size_t i = 0; i < map_size; i++) {
+    const char *item_start = *data;
+    // TODO(bfredl): we could specialize a hot path for FIXSTR here
+    String key = unpack_string(data, size);
+    if (!key.data) {
+      *error = arena_printf(NULL, "has key value which is not a string").data;
+      return false;
+    } else if (key.size == 0) {
+      *error = arena_printf(NULL, "has empty key").data;
+      return false;
+    }
+    KeySetLink *field = hashy(key.data, key.size);
+
+    if (!field) {
+      int status = unpack_skip(data, size);
+      if (status) {
+        return false;
+      }
+
+      if (ad) {
+        push_additional_data(ad, item_start, (size_t)(*data - item_start));
+      }
+      continue;
+    }
+
+    assert(field->opt_index >= 0);
+    uint64_t flag = (1ULL << field->opt_index);
+    if (ks->is_set_ & flag) {
+      *error = xstrdup("duplicate key");
+      return false;
+    }
+    ks->is_set_ |= flag;
+
+    char *mem = ((char *)retval + field->ptr_off);
+    switch (field->type) {
+    case kObjectTypeBoolean:
+      if (*size == 0 || (**data & 0xfe) != 0xc2) {
+        *error = arena_printf(NULL, "has %.*s key value which is not a boolean", (int)key.size,
+                              key.data).data;
+        return false;
+      }
+      *(Boolean *)mem = **data & 0x01;
+      (*data)++; (*size)--;
+      break;
+
+    case kObjectTypeInteger:
+      if (!unpack_integer(data, size, (Integer *)mem)) {
+        *error = arena_printf(NULL, "has %.*s key value which is not an integer", (int)key.size,
+                              key.data).data;
+        return false;
+      }
+      break;
+
+    case kObjectTypeString: {
+      String val = unpack_string(data, size);
+      if (!val.data) {
+        *error = arena_printf(NULL, "has %.*s key value which is not a binary", (int)key.size,
+                              key.data).data;
+        return false;
+      }
+      *(String *)mem = val;
+      break;
+    }
+
+    case kUnpackTypeStringArray: {
+      ssize_t len = unpack_array(data, size);
+      if (len < 0) {
+        *error = arena_printf(NULL, "has %.*s key with non-array value", (int)key.size,
+                              key.data).data;
+        return false;
+      }
+      StringArray *a = (StringArray *)mem;
+      kv_ensure_space(*a, (size_t)len);
+      for (size_t j = 0; j < (size_t)len; j++) {
+        String item = unpack_string(data, size);
+        if (!item.data) {
+          *error = arena_printf(NULL, "has %.*s array with non-binary value", (int)key.size,
+                                key.data).data;
+          return false;
+        }
+        kv_push(*a, item);
+      }
+      break;
+    }
+
+    default:
+      abort();  // not supported
+    }
+  }
+
+  return true;
+}
diff --git a/src/nvim/msgpack_rpc/unpacker.h b/src/nvim/msgpack_rpc/unpacker.h
index ed55fdd4af..c29462292f 100644
--- a/src/nvim/msgpack_rpc/unpacker.h
+++ b/src/nvim/msgpack_rpc/unpacker.h
@@ -41,6 +41,8 @@ struct Unpacker {
   bool has_grid_line_event;
 };
 
+typedef kvec_t(char) AdditionalDataBuilder;
+
 // unrecovareble error. unpack_error should be set!
 #define unpacker_closed(p) ((p)->state < 0)
 
diff --git a/src/nvim/ops.c b/src/nvim/ops.c
index e418635d37..05b9db474e 100644
--- a/src/nvim/ops.c
+++ b/src/nvim/ops.c
@@ -930,16 +930,6 @@ int do_record(int c)
   return retval;
 }
 
-static void set_yreg_additional_data(yankreg_T *reg, dict_T *additional_data)
-  FUNC_ATTR_NONNULL_ARG(1)
-{
-  if (reg->additional_data == additional_data) {
-    return;
-  }
-  tv_dict_unref(reg->additional_data);
-  reg->additional_data = additional_data;
-}
-
 /// Stuff string "p" into yank register "regname" as a single line (append if
 /// uppercase). "p" must have been allocated.
 ///
@@ -969,7 +959,7 @@ static int stuff_yank(int regname, char *p)
     *pp = lp;
   } else {
     free_register(reg);
-    set_yreg_additional_data(reg, NULL);
+    reg->additional_data = NULL;
     reg->y_array = xmalloc(sizeof(char *));
     reg->y_array[0] = p;
     reg->y_size = 1;
@@ -2507,7 +2497,7 @@ void clear_registers(void)
 void free_register(yankreg_T *reg)
   FUNC_ATTR_NONNULL_ALL
 {
-  set_yreg_additional_data(reg, NULL);
+  XFREE_CLEAR(reg->additional_data);
   if (reg->y_array == NULL) {
     return;
   }
@@ -5144,7 +5134,7 @@ static void str_to_reg(yankreg_T *y_ptr, MotionType yank_type, const char *str,
   }
   y_ptr->y_type = yank_type;
   y_ptr->y_size = lnum;
-  set_yreg_additional_data(y_ptr, NULL);
+  XFREE_CLEAR(y_ptr->additional_data);
   y_ptr->timestamp = os_time();
   if (yank_type == kMTBlockWise) {
     y_ptr->y_width = (blocklen == -1 ? (colnr_T)maxlen - 1 : blocklen);
diff --git a/src/nvim/ops.h b/src/nvim/ops.h
index f2b38e5faf..dfa2d4b112 100644
--- a/src/nvim/ops.h
+++ b/src/nvim/ops.h
@@ -110,7 +110,7 @@ typedef struct {
   MotionType y_type;        ///< Register type
   colnr_T y_width;          ///< Register width (only valid for y_type == kBlockWise).
   Timestamp timestamp;      ///< Time when register was last modified.
-  dict_T *additional_data;  ///< Additional data from ShaDa file.
+  AdditionalData *additional_data;  ///< Additional data from ShaDa file.
 } yankreg_T;
 
 /// Modes for get_yank_register()
diff --git a/src/nvim/os/fileio.c b/src/nvim/os/fileio.c
index 89834bed80..1981d0dfd4 100644
--- a/src/nvim/os/fileio.c
+++ b/src/nvim/os/fileio.c
@@ -309,6 +309,22 @@ ptrdiff_t file_read(FileDescriptor *const fp, char *const ret_buf, const size_t
   return (ptrdiff_t)(size - read_remaining);
 }
 
+/// try to read already buffered data in place
+///
+/// @return NULL if enough data is not available
+///         valid pointer to chunk of "size". pointer becomes invalid in the next "file_read" call!
+char *file_try_read_buffered(FileDescriptor *const fp, const size_t size)
+  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  if ((size_t)(fp->write_pos - fp->read_pos) >= size) {
+    char *ret = fp->read_pos;
+    fp->read_pos += size;
+    fp->bytes_read += size;
+    return ret;
+  }
+  return NULL;
+}
+
 /// Write to a file
 ///
 /// @param[in]  fd  File descriptor to write to.
diff --git a/src/nvim/search.c b/src/nvim/search.c
index 57d151cf75..d1cb336905 100644
--- a/src/nvim/search.c
+++ b/src/nvim/search.c
@@ -290,7 +290,7 @@ void restore_search_patterns(void)
 static inline void free_spat(SearchPattern *const spat)
 {
   xfree(spat->pat);
-  tv_dict_unref(spat->additional_data);
+  xfree(spat->additional_data);
 }
 
 #if defined(EXITFREE)
diff --git a/src/nvim/search.h b/src/nvim/search.h
index 92ee5d6854..1b6c1a6375 100644
--- a/src/nvim/search.h
+++ b/src/nvim/search.h
@@ -89,7 +89,7 @@ typedef struct {
   bool no_scs;          ///< No smartcase for this pattern.
   Timestamp timestamp;  ///< Time of the last change.
   SearchOffset off;     ///< Pattern offset.
-  dict_T *additional_data;  ///< Additional data from ShaDa file.
+  AdditionalData *additional_data;  ///< Additional data from ShaDa file.
 } SearchPattern;
 
 /// Optional extra arguments for searchit().
diff --git a/src/nvim/shada.c b/src/nvim/shada.c
index 7dcc6025eb..e5479a7d40 100644
--- a/src/nvim/shada.c
+++ b/src/nvim/shada.c
@@ -1,9 +1,5 @@
 #include <assert.h>
 #include <inttypes.h>
-#include <msgpack/object.h>
-#include <msgpack/pack.h>
-#include <msgpack/sbuffer.h>
-#include <msgpack/unpack.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdio.h>
@@ -13,6 +9,7 @@
 #include <uv.h>
 
 #include "auto/config.h"
+#include "nvim/api/keysets_defs.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/ascii_defs.h"
@@ -43,6 +40,7 @@
 #include "nvim/memory.h"
 #include "nvim/message.h"
 #include "nvim/msgpack_rpc/packer.h"
+#include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/normal_defs.h"
 #include "nvim/ops.h"
 #include "nvim/option.h"
@@ -71,26 +69,26 @@
 # include ENDIAN_INCLUDE_FILE
 #endif
 
-#define SEARCH_KEY_MAGIC "sm"
-#define SEARCH_KEY_SMARTCASE "sc"
-#define SEARCH_KEY_HAS_LINE_OFFSET "sl"
-#define SEARCH_KEY_PLACE_CURSOR_AT_END "se"
-#define SEARCH_KEY_IS_LAST_USED "su"
-#define SEARCH_KEY_IS_SUBSTITUTE_PATTERN "ss"
-#define SEARCH_KEY_HIGHLIGHTED "sh"
-#define SEARCH_KEY_OFFSET "so"
-#define SEARCH_KEY_PAT "sp"
-#define SEARCH_KEY_BACKWARD "sb"
-
-#define REG_KEY_TYPE "rt"
-#define REG_KEY_WIDTH "rw"
-#define REG_KEY_CONTENTS "rc"
-#define REG_KEY_UNNAMED "ru"
-
-#define KEY_LNUM "l"
-#define KEY_COL "c"
-#define KEY_FILE "f"
-#define KEY_NAME_CHAR "n"
+#define SEARCH_KEY_MAGIC sm
+#define SEARCH_KEY_SMARTCASE sc
+#define SEARCH_KEY_HAS_LINE_OFFSET sl
+#define SEARCH_KEY_PLACE_CURSOR_AT_END se
+#define SEARCH_KEY_IS_LAST_USED su
+#define SEARCH_KEY_IS_SUBSTITUTE_PATTERN ss
+#define SEARCH_KEY_HIGHLIGHTED sh
+#define SEARCH_KEY_OFFSET so
+#define SEARCH_KEY_PAT sp
+#define SEARCH_KEY_BACKWARD sb
+
+#define REG_KEY_TYPE rt
+#define REG_KEY_WIDTH rw
+#define REG_KEY_CONTENTS rc
+#define REG_KEY_UNNAMED ru
+
+#define KEY_LNUM l
+#define KEY_COL c
+#define KEY_FILE f
+#define KEY_NAME_CHAR n
 
 // Error messages formerly used by viminfo code:
 //   E136: viminfo: Too many errors, skipping rest of file
@@ -238,26 +236,12 @@ typedef struct {
       char name;
       pos_T mark;
       char *fname;
-      dict_T *additional_data;
     } filemark;
-    struct search_pattern {
-      bool magic;
-      bool smartcase;
-      bool has_line_offset;
-      bool place_cursor_at_end;
-      int64_t offset;
-      bool is_last_used;
-      bool is_substitute_pattern;
-      bool highlighted;
-      bool search_backward;
-      char *pat;
-      dict_T *additional_data;
-    } search_pattern;
+    Dict(_shada_search_pat) search_pattern;
     struct history_item {
       uint8_t histtype;
       char *string;
       char sep;
-      list_T *additional_elements;
     } history_item;
     struct reg {  // yankreg_T
       char name;
@@ -266,12 +250,10 @@ typedef struct {
       bool is_unnamed;
       size_t contents_size;
       size_t width;
-      dict_T *additional_data;
     } reg;
     struct global_var {
       char *name;
       typval_T value;
-      list_T *additional_elements;
     } global_var;
     struct {
       uint64_t type;
@@ -280,17 +262,17 @@ typedef struct {
     } unknown_item;
     struct sub_string {
       char *sub;
-      list_T *additional_elements;
     } sub_string;
     struct buffer_list {
       size_t size;
       struct buffer_list_buffer {
         pos_T pos;
         char *fname;
-        dict_T *additional_data;
+        AdditionalData *additional_data;
       } *buffers;
     } buffer_list;
   } data;
+  AdditionalData *additional_data;
 } ShadaEntry;
 
 /// One entry in sized linked list
@@ -366,6 +348,7 @@ typedef struct {
   [kSDItem##name] = { \
     .timestamp = 0, \
     .type = kSDItem##name, \
+    .additional_data = NULL, \
     .data = { \
       .attr = { __VA_ARGS__ } \
     } \
@@ -385,49 +368,41 @@ static const ShadaEntry sd_default_values[] = {
           .is_substitute_pattern = false,
           .highlighted = false,
           .search_backward = false,
-          .pat = NULL,
-          .additional_data = NULL),
-  DEF_SDE(SubString, sub_string, .sub = NULL, .additional_elements = NULL),
+          .pat = STRING_INIT),
+  DEF_SDE(SubString, sub_string, .sub = NULL),
   DEF_SDE(HistoryEntry, history_item,
           .histtype = HIST_CMD,
           .string = NULL,
-          .sep = NUL,
-          .additional_elements = NULL),
+          .sep = NUL),
   DEF_SDE(Register, reg,
           .name = NUL,
           .type = kMTCharWise,
           .contents = NULL,
           .contents_size = 0,
           .is_unnamed = false,
-          .width = 0,
-          .additional_data = NULL),
+          .width = 0),
   DEF_SDE(Variable, global_var,
           .name = NULL,
-          .value = { .v_type = VAR_UNKNOWN, .vval = { .v_string = NULL } },
-          .additional_elements = NULL),
+          .value = { .v_type = VAR_UNKNOWN, .vval = { .v_string = NULL } }),
   DEF_SDE(GlobalMark, filemark,
           .name = '"',
           .mark = DEFAULT_POS,
-          .fname = NULL,
-          .additional_data = NULL),
+          .fname = NULL),
   DEF_SDE(Jump, filemark,
           .name = NUL,
           .mark = DEFAULT_POS,
-          .fname = NULL,
-          .additional_data = NULL),
+          .fname = NULL),
   DEF_SDE(BufferList, buffer_list,
           .size = 0,
           .buffers = NULL),
   DEF_SDE(LocalMark, filemark,
           .name = '"',
           .mark = DEFAULT_POS,
-          .fname = NULL,
-          .additional_data = NULL),
+          .fname = NULL),
   DEF_SDE(Change, filemark,
           .name = NUL,
           .mark = DEFAULT_POS,
-          .fname = NULL,
-          .additional_data = NULL),
+          .fname = NULL),
 };
 #undef DEFAULT_POS
 #undef DEF_SDE
@@ -685,9 +660,9 @@ static const void *shada_hist_iter(const void *const iter, const uint8_t history
           .sep = (char)(history_type == HIST_SEARCH
                         ? hist_he.hisstr[strlen(hist_he.hisstr) + 1]
                         : 0),
-          .additional_elements = hist_he.additional_elements,
         }
-      }
+      },
+      .additional_data = hist_he.additional_data,
     };
   }
   return ret;
@@ -809,8 +784,7 @@ static inline void hms_to_he_array(const HistoryMergerState *const hms_p,
     hist->timestamp = cur_entry->data.timestamp;
     hist->hisnum = (int)(hist - hist_array) + 1;
     hist->hisstr = cur_entry->data.data.history_item.string;
-    hist->additional_elements =
-      cur_entry->data.data.history_item.additional_elements;
+    hist->additional_data = cur_entry->data.additional_data;
     hist++;
   })
   *new_hisnum = (int)(hist - hist_array);
@@ -1057,9 +1031,9 @@ static void shada_read(FileDescriptor *const sd_reader, const int flags)
           .end = cur_entry.data.search_pattern.place_cursor_at_end,
           .off = cur_entry.data.search_pattern.offset,
         },
-        .pat = cur_entry.data.search_pattern.pat,
-        .patlen = strlen(cur_entry.data.search_pattern.pat),
-        .additional_data = cur_entry.data.search_pattern.additional_data,
+        .pat = cur_entry.data.search_pattern.pat.data,
+        .patlen = cur_entry.data.search_pattern.pat.size,
+        .additional_data = cur_entry.additional_data,
         .timestamp = cur_entry.timestamp,
       };
 
@@ -1087,7 +1061,7 @@ static void shada_read(FileDescriptor *const sd_reader, const int flags)
       sub_set_replacement((SubReplacementString) {
         .sub = cur_entry.data.sub_string.sub,
         .timestamp = cur_entry.timestamp,
-        .additional_elements = cur_entry.data.sub_string.additional_elements,
+        .additional_data = cur_entry.additional_data,
       });
       // Without using regtilde and without / &cpo flag previous substitute
       // string is close to useless: you can only use it with :& or :~ and
@@ -1125,7 +1099,7 @@ static void shada_read(FileDescriptor *const sd_reader, const int flags)
         .y_type = cur_entry.data.reg.type,
         .y_width = (colnr_T)cur_entry.data.reg.width,
         .timestamp = cur_entry.timestamp,
-        .additional_data = cur_entry.data.reg.additional_data,
+        .additional_data = cur_entry.additional_data,
       }, cur_entry.data.reg.is_unnamed)) {
         shada_free_shada_entry(&cur_entry);
       }
@@ -1150,7 +1124,7 @@ static void shada_read(FileDescriptor *const sd_reader, const int flags)
           .fnum = (buf == NULL ? 0 : buf->b_fnum),
           .timestamp = cur_entry.timestamp,
           .view = INIT_FMARKV,
-          .additional_data = cur_entry.data.filemark.additional_data,
+          .additional_data = cur_entry.additional_data,
         },
       };
       if (cur_entry.type == kSDItemGlobalMark) {
@@ -1192,8 +1166,9 @@ static void shada_read(FileDescriptor *const sd_reader, const int flags)
                       cur_entry.data.buffer_list.buffers[i].pos, 0, view);
           buflist_setfpos(buf, curwin, buf->b_last_cursor.mark.lnum,
                           buf->b_last_cursor.mark.col, false);
-          buf->additional_data =
-            cur_entry.data.buffer_list.buffers[i].additional_data;
+
+          xfree(buf->additional_data);
+          buf->additional_data = cur_entry.data.buffer_list.buffers[i].additional_data;
           cur_entry.data.buffer_list.buffers[i].additional_data = NULL;
         }
       }
@@ -1230,7 +1205,7 @@ static void shada_read(FileDescriptor *const sd_reader, const int flags)
         .fnum = 0,
         .timestamp = cur_entry.timestamp,
         .view = INIT_FMARKV,
-        .additional_data = cur_entry.data.filemark.additional_data,
+        .additional_data = cur_entry.additional_data,
       };
       if (cur_entry.type == kSDItemLocalMark) {
         if (!mark_set_local(cur_entry.data.filemark.name, buf, fm, !force)) {
@@ -1338,7 +1313,9 @@ static char *shada_filename(const char *file)
   return xstrdup(file);
 }
 
-#define PACK_STATIC_STR(s) mpack_str(STATIC_CSTR_AS_STRING(s), &sbuf);
+#define PACK_KEY(s) mpack_str(STATIC_CSTR_AS_STRING(KEY_NAME(s)), &sbuf);
+#define KEY_NAME(s) #s
+#define KEY_NAME2(s) KEY_NAME(s)
 
 #define SHADA_MPACK_FREE_SPACE (4 * MPACK_ITEM_SIZE)
 
@@ -1349,6 +1326,18 @@ static void shada_check_buffer(PackerBuffer *packer)
   }
 }
 
+static uint32_t additional_data_len(AdditionalData *src)
+{
+  return src ? src->nitems : 0;
+}
+
+static void dump_additional_data(AdditionalData *src, PackerBuffer *sbuf)
+{
+  if (src != NULL) {
+    mpack_raw(src->data, src->nbytes, sbuf);
+  }
+}
+
 /// Write single ShaDa entry
 ///
 /// @param[in]  packer     Packer used to write entry.
@@ -1363,37 +1352,7 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
 {
   ShaDaWriteResult ret = kSDWriteFailed;
   PackerBuffer sbuf = packer_string_buffer();
-#define DUMP_ADDITIONAL_ELEMENTS(src, what) \
-  do { \
-    if ((src) != NULL) { \
-      TV_LIST_ITER((src), li, { \
-        if (encode_vim_to_msgpack(&sbuf, TV_LIST_ITEM_TV(li), \
-                                  _("additional elements of ShaDa " what)) \
-            == FAIL) { \
-          goto shada_pack_entry_error; \
-        } \
-      }); \
-    } \
-  } while (0)
-#define DUMP_ADDITIONAL_DATA(src, what) \
-  do { \
-    dict_T *const d = (src); \
-    if (d != NULL) { \
-      size_t todo = d->dv_hashtab.ht_used; \
-      for (const hashitem_T *hi = d->dv_hashtab.ht_array; todo; hi++) { \
-        if (!HASHITEM_EMPTY(hi)) { \
-          todo--; \
-          dictitem_T *const di = TV_DICT_HI2DI(hi); \
-          mpack_str(cstr_as_string(hi->hi_key), &sbuf); \
-          if (encode_vim_to_msgpack(&sbuf, &di->di_tv, \
-                                    _("additional data of ShaDa " what)) \
-              == FAIL) { \
-            goto shada_pack_entry_error; \
-          } \
-        } \
-      } \
-    } \
-  } while (0)
+
 #define CHECK_DEFAULT(entry, attr) \
   (sd_default_values[(entry).type].data.attr == (entry).data.attr)
 #define ONE_IF_NOT_DEFAULT(entry, attr) \
@@ -1402,7 +1361,7 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
 #define PACK_BOOL(entry, name, attr) \
   do { \
     if (!CHECK_DEFAULT(entry, search_pattern.attr)) { \
-      PACK_STATIC_STR(name); \
+      PACK_KEY(name); \
       mpack_bool(&sbuf.ptr, !sd_default_values[(entry).type].data.search_pattern.attr); \
     } \
   } while (0)
@@ -1418,26 +1377,19 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
     const bool is_hist_search =
       entry.data.history_item.histtype == HIST_SEARCH;
     uint32_t arr_size = (2 + (uint32_t)is_hist_search
-                         + (uint32_t)(tv_list_len(entry.data.history_item.additional_elements)));
+                         + additional_data_len(entry.additional_data));
     mpack_array(&sbuf.ptr, arr_size);
     mpack_uint(&sbuf.ptr, entry.data.history_item.histtype);
     mpack_bin(cstr_as_string(entry.data.history_item.string), &sbuf);
     if (is_hist_search) {
       mpack_uint(&sbuf.ptr, (uint8_t)entry.data.history_item.sep);
     }
-    DUMP_ADDITIONAL_ELEMENTS(entry.data.history_item.additional_elements,
-                             "history entry item");
+    dump_additional_data(entry.additional_data, &sbuf);
     break;
   }
   case kSDItemVariable: {
-    if (entry.data.global_var.value.v_type == VAR_BLOB) {
-      // Strings and Blobs both pack as msgpack BINs; differentiate them by
-      // storing an additional VAR_TYPE_BLOB element alongside Blobs
-      list_T *const list = tv_list_alloc(1);
-      tv_list_append_number(list, VAR_TYPE_BLOB);
-      entry.data.global_var.additional_elements = list;
-    }
-    uint32_t arr_size = 2 + (uint32_t)(tv_list_len(entry.data.global_var.additional_elements));
+    bool is_blob = (entry.data.global_var.value.v_type == VAR_BLOB);
+    uint32_t arr_size = 2 + (is_blob ? 1 : 0) + additional_data_len(entry.additional_data);
     mpack_array(&sbuf.ptr, arr_size);
     const String varname = cstr_as_string(entry.data.global_var.name);
     mpack_bin(varname, &sbuf);
@@ -1451,16 +1403,18 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
             entry.data.global_var.name);
       goto shada_pack_entry_error;
     }
-    DUMP_ADDITIONAL_ELEMENTS(entry.data.global_var.additional_elements,
-                             "variable item");
+    if (is_blob) {
+      mpack_check_buffer(&sbuf);
+      mpack_integer(&sbuf.ptr, VAR_TYPE_BLOB);
+    }
+    dump_additional_data(entry.additional_data, &sbuf);
     break;
   }
   case kSDItemSubString: {
-    uint32_t arr_size = 1 + (uint32_t)(tv_list_len(entry.data.sub_string.additional_elements));
+    uint32_t arr_size = 1 + additional_data_len(entry.additional_data);
     mpack_array(&sbuf.ptr, arr_size);
     mpack_bin(cstr_as_string(entry.data.sub_string.sub), &sbuf);
-    DUMP_ADDITIONAL_ELEMENTS(entry.data.sub_string.additional_elements,
-                             "sub string item");
+    dump_additional_data(entry.additional_data, &sbuf);
     break;
   }
   case kSDItemSearchPattern: {
@@ -1475,14 +1429,10 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
                                + ONE_IF_NOT_DEFAULT(entry, search_pattern.highlighted)
                                + ONE_IF_NOT_DEFAULT(entry, search_pattern.offset)
                                + ONE_IF_NOT_DEFAULT(entry, search_pattern.search_backward)
-                               // finally, additional data:
-                               + (entry.data.search_pattern.additional_data
-                                  ? (uint32_t)entry.data.search_pattern.additional_data->dv_hashtab.
-                                  ht_used
-                                  : 0));
+                               + additional_data_len(entry.additional_data));
     mpack_map(&sbuf.ptr, entry_map_size);
-    PACK_STATIC_STR(SEARCH_KEY_PAT);
-    mpack_bin(cstr_as_string(entry.data.search_pattern.pat), &sbuf);
+    PACK_KEY(SEARCH_KEY_PAT);
+    mpack_bin(entry.data.search_pattern.pat, &sbuf);
     PACK_BOOL(entry, SEARCH_KEY_MAGIC, magic);
     PACK_BOOL(entry, SEARCH_KEY_IS_LAST_USED, is_last_used);
     PACK_BOOL(entry, SEARCH_KEY_SMARTCASE, smartcase);
@@ -1492,12 +1442,11 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
     PACK_BOOL(entry, SEARCH_KEY_HIGHLIGHTED, highlighted);
     PACK_BOOL(entry, SEARCH_KEY_BACKWARD, search_backward);
     if (!CHECK_DEFAULT(entry, search_pattern.offset)) {
-      PACK_STATIC_STR(SEARCH_KEY_OFFSET);
+      PACK_KEY(SEARCH_KEY_OFFSET);
       mpack_integer(&sbuf.ptr, entry.data.search_pattern.offset);
     }
 #undef PACK_BOOL
-    DUMP_ADDITIONAL_DATA(entry.data.search_pattern.additional_data,
-                         "search pattern item");
+    dump_additional_data(entry.additional_data, &sbuf);
     break;
   }
   case kSDItemChange:
@@ -1508,31 +1457,26 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
                              + ONE_IF_NOT_DEFAULT(entry, filemark.mark.lnum)
                              + ONE_IF_NOT_DEFAULT(entry, filemark.mark.col)
                              + ONE_IF_NOT_DEFAULT(entry, filemark.name)
-                             // Additional entries, if any:
-                             + (
-                                entry.data.filemark.additional_data == NULL
-                                ? 0
-                                : entry.data.filemark.additional_data->dv_hashtab.ht_used));
+                             + additional_data_len(entry.additional_data));
     mpack_map(&sbuf.ptr, (uint32_t)entry_map_size);
-    PACK_STATIC_STR(KEY_FILE);
+    PACK_KEY(KEY_FILE);
     mpack_bin(cstr_as_string(entry.data.filemark.fname), &sbuf);
     if (!CHECK_DEFAULT(entry, filemark.mark.lnum)) {
-      PACK_STATIC_STR(KEY_LNUM);
+      PACK_KEY(KEY_LNUM);
       mpack_integer(&sbuf.ptr, entry.data.filemark.mark.lnum);
     }
     if (!CHECK_DEFAULT(entry, filemark.mark.col)) {
-      PACK_STATIC_STR(KEY_COL);
+      PACK_KEY(KEY_COL);
       mpack_integer(&sbuf.ptr, entry.data.filemark.mark.col);
     }
     assert(entry.type == kSDItemJump || entry.type == kSDItemChange
            ? CHECK_DEFAULT(entry, filemark.name)
            : true);
     if (!CHECK_DEFAULT(entry, filemark.name)) {
-      PACK_STATIC_STR(KEY_NAME_CHAR);
+      PACK_KEY(KEY_NAME_CHAR);
       mpack_uint(&sbuf.ptr, (uint8_t)entry.data.filemark.name);
     }
-    DUMP_ADDITIONAL_DATA(entry.data.filemark.additional_data,
-                         "mark (change, jump, global or local) item");
+    dump_additional_data(entry.additional_data, &sbuf);
     break;
   }
   case kSDItemRegister: {
@@ -1540,31 +1484,29 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
                                + ONE_IF_NOT_DEFAULT(entry, reg.type)
                                + ONE_IF_NOT_DEFAULT(entry, reg.width)
                                + ONE_IF_NOT_DEFAULT(entry, reg.is_unnamed)
-                               // Additional entries, if any:
-                               + (entry.data.reg.additional_data == NULL
-                                  ? 0
-                                  : (uint32_t)entry.data.reg.additional_data->dv_hashtab.ht_used));
+                               + additional_data_len(entry.additional_data));
+
     mpack_map(&sbuf.ptr, entry_map_size);
-    PACK_STATIC_STR(REG_KEY_CONTENTS);
+    PACK_KEY(REG_KEY_CONTENTS);
     mpack_array(&sbuf.ptr, (uint32_t)entry.data.reg.contents_size);
     for (size_t i = 0; i < entry.data.reg.contents_size; i++) {
       mpack_bin(cstr_as_string(entry.data.reg.contents[i]), &sbuf);
     }
-    PACK_STATIC_STR(KEY_NAME_CHAR);
+    PACK_KEY(KEY_NAME_CHAR);
     mpack_uint(&sbuf.ptr, (uint8_t)entry.data.reg.name);
     if (!CHECK_DEFAULT(entry, reg.type)) {
-      PACK_STATIC_STR(REG_KEY_TYPE);
+      PACK_KEY(REG_KEY_TYPE);
       mpack_uint(&sbuf.ptr, (uint8_t)entry.data.reg.type);
     }
     if (!CHECK_DEFAULT(entry, reg.width)) {
-      PACK_STATIC_STR(REG_KEY_WIDTH);
+      PACK_KEY(REG_KEY_WIDTH);
       mpack_uint64(&sbuf.ptr, (uint64_t)entry.data.reg.width);
     }
     if (!CHECK_DEFAULT(entry, reg.is_unnamed)) {
-      PACK_STATIC_STR(REG_KEY_UNNAMED);
+      PACK_KEY(REG_KEY_UNNAMED);
       mpack_bool(&sbuf.ptr, entry.data.reg.is_unnamed);
     }
-    DUMP_ADDITIONAL_DATA(entry.data.reg.additional_data, "register item");
+    dump_additional_data(entry.additional_data, &sbuf);
     break;
   }
   case kSDItemBufferList:
@@ -1575,24 +1517,20 @@ static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry
                                           != default_pos.lnum)
                                + (size_t)(entry.data.buffer_list.buffers[i].pos.col
                                           != default_pos.col)
-                               // Additional entries, if any:
-                               + (entry.data.buffer_list.buffers[i].additional_data == NULL
-                                  ? 0
-                                  : (entry.data.buffer_list.buffers[i].additional_data
-                                     ->dv_hashtab.ht_used)));
+                               + additional_data_len(entry.data.buffer_list.buffers[i].
+                                                     additional_data));
       mpack_map(&sbuf.ptr, (uint32_t)entry_map_size);
-      PACK_STATIC_STR(KEY_FILE);
+      PACK_KEY(KEY_FILE);
       mpack_bin(cstr_as_string(entry.data.buffer_list.buffers[i].fname), &sbuf);
       if (entry.data.buffer_list.buffers[i].pos.lnum != 1) {
-        PACK_STATIC_STR(KEY_LNUM);
+        PACK_KEY(KEY_LNUM);
         mpack_uint64(&sbuf.ptr, (uint64_t)entry.data.buffer_list.buffers[i].pos.lnum);
       }
       if (entry.data.buffer_list.buffers[i].pos.col != 0) {
-        PACK_STATIC_STR(KEY_COL);
+        PACK_KEY(KEY_COL);
         mpack_uint64(&sbuf.ptr, (uint64_t)entry.data.buffer_list.buffers[i].pos.col);
       }
-      DUMP_ADDITIONAL_DATA(entry.data.buffer_list.buffers[i].additional_data,
-                           "buffer list subitem");
+      dump_additional_data(entry.data.buffer_list.buffers[i].additional_data, &sbuf);
     }
     break;
   case kSDItemHeader:
@@ -1686,76 +1624,29 @@ static int compare_file_marks(const void *a, const void *b)
 ///
 /// @return kSDReadStatusNotShaDa, kSDReadStatusReadError or
 ///         kSDReadStatusSuccess.
-static inline ShaDaReadResult shada_parse_msgpack(FileDescriptor *const sd_reader,
-                                                  const size_t length,
-                                                  msgpack_unpacked *ret_unpacked,
-                                                  char **const ret_buf)
-  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ARG(1)
+static ShaDaReadResult shada_check_status(uintmax_t initial_fpos, int status, size_t remaining)
+  FUNC_ATTR_WARN_UNUSED_RESULT
 {
-  const uintmax_t initial_fpos = sd_reader->bytes_read;
-  char *const buf = xmalloc(length);
-
-  const ShaDaReadResult fl_ret = fread_len(sd_reader, buf, length);
-  if (fl_ret != kSDReadStatusSuccess) {
-    xfree(buf);
-    return fl_ret;
-  }
-  bool did_try_to_free = false;
-shada_parse_msgpack_read_next: {}
-  size_t off = 0;
-  msgpack_unpacked unpacked;
-  msgpack_unpacked_init(&unpacked);
-  const msgpack_unpack_return result =
-    msgpack_unpack_next(&unpacked, buf, length, &off);
-  ShaDaReadResult ret = kSDReadStatusSuccess;
-  switch (result) {
-  case MSGPACK_UNPACK_SUCCESS:
-    if (off < length) {
-      goto shada_parse_msgpack_extra_bytes;
+  switch (status) {
+  case MPACK_OK:
+    if (remaining) {
+      semsg(_(RCERR "Failed to parse ShaDa file: extra bytes in msgpack string "
+              "at position %" PRIu64),
+            (uint64_t)initial_fpos);
+      return kSDReadStatusNotShaDa;
     }
-    break;
-  case MSGPACK_UNPACK_PARSE_ERROR:
-    semsg(_(RCERR "Failed to parse ShaDa file due to a msgpack parser error "
-            "at position %" PRIu64),
-          (uint64_t)initial_fpos);
-    ret = kSDReadStatusNotShaDa;
-    break;
-  case MSGPACK_UNPACK_NOMEM_ERROR:
-    if (!did_try_to_free) {
-      did_try_to_free = true;
-      try_to_free_memory();
-      goto shada_parse_msgpack_read_next;
-    }
-    emsg(_(e_outofmem));
-    ret = kSDReadStatusReadError;
-    break;
-  case MSGPACK_UNPACK_CONTINUE:
+    return kSDReadStatusSuccess;
+  case MPACK_EOF:
     semsg(_(RCERR "Failed to parse ShaDa file: incomplete msgpack string "
             "at position %" PRIu64),
           (uint64_t)initial_fpos);
-    ret = kSDReadStatusNotShaDa;
-    break;
-  case MSGPACK_UNPACK_EXTRA_BYTES:
-shada_parse_msgpack_extra_bytes:
-    semsg(_(RCERR "Failed to parse ShaDa file: extra bytes in msgpack string "
+    return kSDReadStatusNotShaDa;
+  default:
+    semsg(_(RCERR "Failed to parse ShaDa file due to a msgpack parser error "
             "at position %" PRIu64),
           (uint64_t)initial_fpos);
-    ret = kSDReadStatusNotShaDa;
-    break;
-  }
-  if (ret_buf != NULL && ret == kSDReadStatusSuccess) {
-    if (ret_unpacked == NULL) {
-      msgpack_unpacked_destroy(&unpacked);
-    } else {
-      *ret_unpacked = unpacked;
-    }
-    *ret_buf = buf;
-  } else {
-    assert(ret_buf == NULL || ret != kSDReadStatusSuccess);
-    msgpack_unpacked_destroy(&unpacked);
-    xfree(buf);
+    return kSDReadStatusNotShaDa;
   }
-  return ret;
 }
 
 /// Format shada entry for debugging purposes
@@ -1772,25 +1663,16 @@ static const char *shada_format_entry(const ShadaEntry entry)
   //                         ^ Space for `can_free_entry`
 #define FORMAT_MARK_ENTRY(entry_name, name_fmt, name_fmt_arg) \
   do { \
-    typval_T ad_tv = { \
-      .v_type = VAR_DICT, \
-      .vval.v_dict = entry.data.filemark.additional_data \
-    }; \
-    size_t ad_len; \
-    char *const ad = encode_tv2string(&ad_tv, &ad_len); \
     vim_snprintf_add(S_LEN(ret), \
                      entry_name " {" name_fmt " file=[%zu]\"%.512s\", " \
                      "pos={l=%" PRIdLINENR ",c=%" PRIdCOLNR ",a=%" PRIdCOLNR "}, " \
-                     "ad={%p:[%zu]%.64s} }", \
+                     "}", \
                      name_fmt_arg, \
                      strlen(entry.data.filemark.fname), \
                      entry.data.filemark.fname, \
                      entry.data.filemark.mark.lnum, \
                      entry.data.filemark.mark.col, \
-                     entry.data.filemark.mark.coladd, \
-                     (void *)entry.data.filemark.additional_data, \
-                     ad_len, \
-                     ad); \
+                     entry.data.filemark.mark.coladd); \
   } while (0)
   switch (entry.type) {
   case kSDItemMissing:
@@ -1974,8 +1856,8 @@ static inline ShaDaWriteResult shada_read_when_writing(FileDescriptor *const sd_
           }
           // Ignore duplicates.
           if (wms_entry.timestamp == entry.timestamp
-              && (wms_entry.data.filemark.additional_data == NULL
-                  && entry.data.filemark.additional_data == NULL)
+              && (wms_entry.additional_data == NULL
+                  && entry.additional_data == NULL)
               && marks_equal(wms_entry.data.filemark.mark,
                              entry.data.filemark.mark)
               && strcmp(wms_entry.data.filemark.fname,
@@ -2203,11 +2085,11 @@ static inline void add_search_pattern(PossiblyFreedShadaEntry *const ret_pse,
             .is_substitute_pattern = is_substitute_pattern,
             .highlighted = ((is_substitute_pattern ^ search_last_used)
                             && search_highlighted),
-            .pat = pat.pat,
-            .additional_data = pat.additional_data,
+            .pat = cstr_as_string(pat.pat),
             .search_backward = (!is_substitute_pattern && pat.off.dir == '?'),
           }
-        }
+        },
+        .additional_data = pat.additional_data,
       }
     };
   }
@@ -2244,11 +2126,11 @@ static inline void shada_initialize_registers(WriteMergerState *const wms, int m
             .contents_size = reg.y_size,
             .type = reg.y_type,
             .width = (size_t)(reg.y_type == kMTBlockWise ? reg.y_width : 0),
-            .additional_data = reg.additional_data,
             .name = name,
             .is_unnamed = is_unnamed,
           }
-        }
+        },
+        .additional_data = reg.additional_data,
       }
     };
   } while (reg_iter != NULL);
@@ -2494,9 +2376,9 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
           .global_var = {
             .name = (char *)name,
             .value = tgttv,
-            .additional_elements = NULL,
           }
-        }
+        },
+        .additional_data = NULL,
       }, max_kbyte)) == kSDWriteFailed) {
         tv_clear(&vartv);
         tv_clear(&tgttv);
@@ -2538,9 +2420,9 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
         .data = {
           .sub_string = {
             .sub = sub.sub,
-            .additional_elements = sub.additional_elements,
           }
-        }
+        },
+        .additional_data = sub.additional_data,
       }
     };
   }
@@ -2580,10 +2462,10 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
             .filemark = {
               .mark = fm.fmark.mark,
               .name = name,
-              .additional_data = fm.fmark.additional_data,
               .fname = (char *)fname,
             }
-          }
+          },
+          .additional_data = fm.fmark.additional_data,
         },
       };
       if (ascii_isdigit(name)) {
@@ -2629,9 +2511,9 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
                 .mark = fm.mark,
                 .name = name,
                 .fname = (char *)fname,
-                .additional_data = fm.additional_data,
               }
-            }
+            },
+            .additional_data = fm.additional_data,
           }
         };
         if (fm.timestamp > filemarks->greatest_timestamp) {
@@ -2649,9 +2531,9 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
               .filemark = {
                 .mark = fm.mark,
                 .fname = (char *)fname,
-                .additional_data = fm.additional_data,
               }
-            }
+            },
+            .additional_data = fm.additional_data,
           }
         };
         if (fm.timestamp > filemarks->greatest_timestamp) {
@@ -2682,10 +2564,10 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
           .filemark = {
             .mark = curwin->w_cursor,
             .name = '0',
-            .additional_data = NULL,
             .fname = curbuf->b_ffname,
           }
-        }
+        },
+        .additional_data = NULL,
       },
     });
   }
@@ -2799,7 +2681,7 @@ shada_write_exit:
   return ret;
 }
 
-#undef PACK_STATIC_STR
+#undef PACK_KEY
 
 /// Write ShaDa file to a given location
 ///
@@ -3035,41 +2917,36 @@ static void shada_free_shada_entry(ShadaEntry *const entry)
   case kSDItemJump:
   case kSDItemGlobalMark:
   case kSDItemLocalMark:
-    tv_dict_unref(entry->data.filemark.additional_data);
     xfree(entry->data.filemark.fname);
     break;
   case kSDItemSearchPattern:
-    tv_dict_unref(entry->data.search_pattern.additional_data);
-    xfree(entry->data.search_pattern.pat);
+    api_free_string(entry->data.search_pattern.pat);
     break;
   case kSDItemRegister:
-    tv_dict_unref(entry->data.reg.additional_data);
     for (size_t i = 0; i < entry->data.reg.contents_size; i++) {
       xfree(entry->data.reg.contents[i]);
     }
     xfree(entry->data.reg.contents);
     break;
   case kSDItemHistoryEntry:
-    tv_list_unref(entry->data.history_item.additional_elements);
     xfree(entry->data.history_item.string);
     break;
   case kSDItemVariable:
-    tv_list_unref(entry->data.global_var.additional_elements);
     xfree(entry->data.global_var.name);
     tv_clear(&entry->data.global_var.value);
     break;
   case kSDItemSubString:
-    tv_list_unref(entry->data.sub_string.additional_elements);
     xfree(entry->data.sub_string.sub);
     break;
   case kSDItemBufferList:
     for (size_t i = 0; i < entry->data.buffer_list.size; i++) {
       xfree(entry->data.buffer_list.buffers[i].fname);
-      tv_dict_unref(entry->data.buffer_list.buffers[i].additional_data);
+      xfree(entry->data.buffer_list.buffers[i].additional_data);
     }
     xfree(entry->data.buffer_list.buffers);
     break;
   }
+  XFREE_CLEAR(entry->additional_data);
 }
 
 #ifndef HAVE_BE64TOH
@@ -3203,128 +3080,6 @@ static ShaDaReadResult msgpack_read_uint64(FileDescriptor *const sd_reader, bool
   RERR "Error while reading ShaDa file: " \
   entry_name " entry at position %" PRIu64 " " \
   error_desc
-#define CHECK_KEY(key, \
-                  expected) ((key).via.str.size == (sizeof(expected) - 1) \
-                             && strncmp((key).via.str.ptr, expected, (sizeof(expected) - 1)) == 0)
-#define CLEAR_GA_AND_ERROR_OUT(ga) \
-  do { \
-    ga_clear(&(ga)); \
-    goto shada_read_next_item_error; \
-  } while (0)
-#define ID(s) s
-#define BINDUP(b) xmemdupz((b).ptr, (b).size)
-#define TOINT(s) ((int)(s))
-#define TOCHAR(s) ((char)(s))
-#define TOU8(s) ((uint8_t)(s))
-#define TOSIZE(s) ((size_t)(s))
-#define CHECKED_ENTRY(condition, error_desc, entry_name, obj, tgt, attr, \
-                      proc) \
-  do { \
-    if (!(condition)) { \
-      semsg(_(READERR(entry_name, error_desc)), initial_fpos); \
-      CLEAR_GA_AND_ERROR_OUT(ad_ga); \
-    } \
-    (tgt) = proc((obj).via.attr); \
-  } while (0)
-#define CHECK_KEY_IS_STR(un, entry_name) \
-  if ((un).data.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR) { \
-    semsg(_(READERR(entry_name, "has key which is not a string")), \
-          initial_fpos); \
-    CLEAR_GA_AND_ERROR_OUT(ad_ga); \
-  } else if ((un).data.via.map.ptr[i].key.via.str.size == 0) { \
-    semsg(_(READERR(entry_name, "has empty key")), initial_fpos); \
-    CLEAR_GA_AND_ERROR_OUT(ad_ga); \
-  }
-#define CHECKED_KEY(un, entry_name, name, error_desc, tgt, condition, attr, proc) \
-  else if (CHECK_KEY((un).data.via.map.ptr[i].key, name)) \
-  { \
-    CHECKED_ENTRY(condition, \
-                  "has " name " key value " error_desc, \
-                  entry_name, \
-                  (un).data.via.map.ptr[i].val, \
-                  tgt, \
-                  attr, \
-                  proc); \
-  }
-#define TYPED_KEY(un, entry_name, name, type_name, tgt, objtype, attr, proc) \
-  CHECKED_KEY(un, entry_name, name, "which is not " type_name, tgt, \
-              (un).data.via.map.ptr[i].val.type == MSGPACK_OBJECT_##objtype, \
-              attr, proc)
-#define BOOLEAN_KEY(un, entry_name, name, tgt) \
-  TYPED_KEY(un, entry_name, name, "a boolean", tgt, BOOLEAN, boolean, ID)
-#define STRING_KEY(un, entry_name, name, tgt) \
-  TYPED_KEY(un, entry_name, name, "a binary", tgt, BIN, bin, BINDUP)
-#define CONVERTED_STRING_KEY(un, entry_name, name, tgt) \
-  TYPED_KEY(un, entry_name, name, "a binary", tgt, BIN, bin, \
-            BIN_CONVERTED)
-#define INT_KEY(un, entry_name, name, tgt, proc) \
-  CHECKED_KEY(un, entry_name, name, "which is not an integer", tgt, \
-              (((un).data.via.map.ptr[i].val.type \
-                == MSGPACK_OBJECT_POSITIVE_INTEGER) \
-               || ((un).data.via.map.ptr[i].val.type \
-                   == MSGPACK_OBJECT_NEGATIVE_INTEGER)), \
-              i64, proc)
-#define INTEGER_KEY(un, entry_name, name, tgt) \
-  INT_KEY(un, entry_name, name, tgt, TOINT)
-#define ADDITIONAL_KEY(un) \
-  else { \
-    ga_grow(&ad_ga, 1); \
-    memcpy(((char *)ad_ga.ga_data) + ((size_t)ad_ga.ga_len \
-                                      * sizeof(*(un).data.via.map.ptr)), \
-           (un).data.via.map.ptr + i, \
-           sizeof(*(un).data.via.map.ptr)); \
-    ad_ga.ga_len++; \
-  }
-#define BIN_CONVERTED(b) (xmemdupz(((b).ptr), ((b).size)))
-#define SET_ADDITIONAL_DATA(tgt, name) \
-  do { \
-    if (ad_ga.ga_len) { \
-      msgpack_object obj = { \
-        .type = MSGPACK_OBJECT_MAP, \
-        .via = { \
-          .map = { \
-            .size = (uint32_t)ad_ga.ga_len, \
-            .ptr = ad_ga.ga_data, \
-          } \
-        } \
-      }; \
-      typval_T adtv; \
-      if (msgpack_to_vim(obj, &adtv) == FAIL \
-          || adtv.v_type != VAR_DICT) { \
-        semsg(_(READERR(name, \
-                        "cannot be converted to a Vimscript dictionary")), \
-              initial_fpos); \
-        ga_clear(&ad_ga); \
-        tv_clear(&adtv); \
-        goto shada_read_next_item_error; \
-      } \
-      (tgt) = adtv.vval.v_dict; \
-    } \
-    ga_clear(&ad_ga); \
-  } while (0)
-#define SET_ADDITIONAL_ELEMENTS(src, src_maxsize, tgt, name) \
-  do { \
-    if ((src).size > (size_t)(src_maxsize)) { \
-      msgpack_object obj = { \
-        .type = MSGPACK_OBJECT_ARRAY, \
-        .via = { \
-          .array = { \
-            .size = ((src).size - (uint32_t)(src_maxsize)), \
-            .ptr = (src).ptr + (src_maxsize), \
-          } \
-        } \
-      }; \
-      typval_T aetv; \
-      if (msgpack_to_vim(obj, &aetv) == FAIL) { \
-        semsg(_(READERR(name, "cannot be converted to a Vimscript list")), \
-              initial_fpos); \
-        tv_clear(&aetv); \
-        goto shada_read_next_item_error; \
-      } \
-      assert(aetv.v_type == VAR_LIST); \
-      (tgt) = aetv.vval.v_list; \
-    } \
-  } while (0)
 
 /// Iterate over shada file contents
 ///
@@ -3352,6 +3107,8 @@ shada_read_next_item_start:
     return kSDReadStatusFinished;
   }
 
+  bool verify_but_ignore = false;
+
   // First: manually unpack type, timestamp and length.
   // This is needed to avoid both seeking and having to maintain a buffer.
   uint64_t type_u64 = (uint64_t)kSDItemMissing;
@@ -3359,6 +3116,9 @@ shada_read_next_item_start:
   uint64_t length_u64;
 
   const uint64_t initial_fpos = sd_reader->bytes_read;
+  AdditionalDataBuilder ad = KV_INITIAL_VALUE;
+  uint32_t read_additional_array_elements = 0;
+  char *error_alloc = NULL;
 
   ShaDaReadResult mru_ret;
   if (((mru_ret = msgpack_read_uint64(sd_reader, true, &type_u64))
@@ -3406,16 +3166,42 @@ shada_read_next_item_start:
     // in incomplete MessagePack string.
     if (initial_fpos == 0
         && (type_u64 == '\n' || type_u64 > SHADA_LAST_ENTRY)) {
-      const ShaDaReadResult spm_ret = shada_parse_msgpack(sd_reader, length,
-                                                          NULL, NULL);
-      if (spm_ret != kSDReadStatusSuccess) {
-        return spm_ret;
-      }
+      verify_but_ignore = true;
     } else {
       const ShaDaReadResult srs_ret = sd_reader_skip(sd_reader, length);
       if (srs_ret != kSDReadStatusSuccess) {
         return srs_ret;
       }
+      goto shada_read_next_item_start;
+    }
+  }
+
+  const uint64_t parse_pos = sd_reader->bytes_read;
+  bool buf_allocated = false;
+  // try to avoid allocation for small items which fits entirely
+  // in the internal buffer of sd_reader
+  char *buf = file_try_read_buffered(sd_reader, length);
+  if (!buf) {
+    buf_allocated = true;
+    buf = xmalloc(length);
+    const ShaDaReadResult fl_ret = fread_len(sd_reader, buf, length);
+    if (fl_ret != kSDReadStatusSuccess) {
+      ret = fl_ret;
+      goto shada_read_next_item_error;
+    }
+  }
+
+  const char *read_ptr = buf;
+  size_t read_size = length;
+
+  if (verify_but_ignore) {
+    int status = unpack_skip(&read_ptr, &read_size);
+    ShaDaReadResult spm_ret = shada_check_status(parse_pos, status, read_size);
+    if (buf_allocated) {
+      xfree(buf);
+    }
+    if (spm_ret != kSDReadStatusSuccess) {
+      return spm_ret;
     }
     goto shada_read_next_item_start;
   }
@@ -3425,32 +3211,20 @@ shada_read_next_item_start:
     entry->data.unknown_item.size = length;
     entry->data.unknown_item.type = type_u64;
     if (initial_fpos == 0) {
-      const ShaDaReadResult spm_ret = shada_parse_msgpack(sd_reader, length, NULL,
-                                                          &entry->data.unknown_item.contents);
+      int status = unpack_skip(&read_ptr, &read_size);
+      ShaDaReadResult spm_ret = shada_check_status(parse_pos, status, read_size);
       if (spm_ret != kSDReadStatusSuccess) {
+        if (buf_allocated) {
+          xfree(buf);
+        }
         entry->type = kSDItemMissing;
+        return spm_ret;
       }
-      return spm_ret;
-    }
-    entry->data.unknown_item.contents = xmalloc(length);
-    const ShaDaReadResult fl_ret =
-      fread_len(sd_reader, entry->data.unknown_item.contents, length);
-    if (fl_ret != kSDReadStatusSuccess) {
-      shada_free_shada_entry(entry);
-      entry->type = kSDItemMissing;
     }
-    return fl_ret;
+    entry->data.unknown_item.contents = buf_allocated ? buf : xmemdup(buf, length);
+    return kSDReadStatusSuccess;
   }
 
-  msgpack_unpacked unpacked;
-  char *buf = NULL;
-
-  const ShaDaReadResult spm_ret = shada_parse_msgpack(sd_reader, length,
-                                                      &unpacked, &buf);
-  if (spm_ret != kSDReadStatusSuccess) {
-    ret = spm_ret;
-    goto shada_read_next_item_error;
-  }
   entry->data = sd_default_values[type_u64].data;
   switch ((ShadaEntryType)type_u64) {
   case kSDItemHeader:
@@ -3459,363 +3233,286 @@ shada_read_next_item_start:
     // Dictionary, but that value was never used)
     break;
   case kSDItemSearchPattern: {
-    if (unpacked.data.type != MSGPACK_OBJECT_MAP) {
-      semsg(_(READERR("search pattern", "is not a dictionary")),
-            initial_fpos);
+    Dict(_shada_search_pat) *it = &entry->data.search_pattern;
+    if (!unpack_keydict(it, DictHash(_shada_search_pat), &ad, &read_ptr, &read_size,
+                        &error_alloc)) {
+      semsg(_(READERR("search pattern", "%s")), initial_fpos, error_alloc);
+      it->pat = NULL_STRING;
       goto shada_read_next_item_error;
     }
-    garray_T ad_ga;
-    ga_init(&ad_ga, sizeof(*(unpacked.data.via.map.ptr)), 1);
-    for (size_t i = 0; i < unpacked.data.via.map.size; i++) {
-      CHECK_KEY_IS_STR(unpacked, "search pattern")
-      BOOLEAN_KEY(unpacked, "search pattern", SEARCH_KEY_MAGIC,
-                  entry->data.search_pattern.magic)
-      BOOLEAN_KEY(unpacked, "search pattern", SEARCH_KEY_SMARTCASE,
-                  entry->data.search_pattern.smartcase)
-      BOOLEAN_KEY(unpacked, "search pattern", SEARCH_KEY_HAS_LINE_OFFSET,
-                  entry->data.search_pattern.has_line_offset)
-      BOOLEAN_KEY(unpacked, "search pattern", SEARCH_KEY_PLACE_CURSOR_AT_END,
-                  entry->data.search_pattern.place_cursor_at_end)
-      BOOLEAN_KEY(unpacked, "search pattern", SEARCH_KEY_IS_LAST_USED,
-                  entry->data.search_pattern.is_last_used)
-      BOOLEAN_KEY(unpacked, "search pattern",
-                  SEARCH_KEY_IS_SUBSTITUTE_PATTERN,
-                  entry->data.search_pattern.is_substitute_pattern)
-      BOOLEAN_KEY(unpacked, "search pattern", SEARCH_KEY_HIGHLIGHTED,
-                  entry->data.search_pattern.highlighted)
-      BOOLEAN_KEY(unpacked, "search pattern", SEARCH_KEY_BACKWARD,
-                  entry->data.search_pattern.search_backward)
-      INTEGER_KEY(unpacked, "search pattern", SEARCH_KEY_OFFSET,
-                  entry->data.search_pattern.offset)
-      CONVERTED_STRING_KEY(unpacked, "search pattern", SEARCH_KEY_PAT,
-                           entry->data.search_pattern.pat)
-      ADDITIONAL_KEY(unpacked)
-    }
-    if (entry->data.search_pattern.pat == NULL) {
+
+    if (!HAS_KEY(it, _shada_search_pat, sp)) {  // SEARCH_KEY_PAT
       semsg(_(READERR("search pattern", "has no pattern")), initial_fpos);
-      CLEAR_GA_AND_ERROR_OUT(ad_ga);
+      goto shada_read_next_item_error;
     }
-    SET_ADDITIONAL_DATA(entry->data.search_pattern.additional_data,
-                        "search pattern");
+    entry->data.search_pattern.pat = copy_string(entry->data.search_pattern.pat, NULL);
+
     break;
   }
   case kSDItemChange:
   case kSDItemJump:
   case kSDItemGlobalMark:
   case kSDItemLocalMark: {
-    if (unpacked.data.type != MSGPACK_OBJECT_MAP) {
-      semsg(_(READERR("mark", "is not a dictionary")), initial_fpos);
+    Dict(_shada_mark) it = { 0 };
+    if (!unpack_keydict(&it, DictHash(_shada_mark), &ad, &read_ptr, &read_size, &error_alloc)) {
+      semsg(_(READERR("mark", "%s")), initial_fpos, error_alloc);
       goto shada_read_next_item_error;
     }
-    garray_T ad_ga;
-    ga_init(&ad_ga, sizeof(*(unpacked.data.via.map.ptr)), 1);
-    for (size_t i = 0; i < unpacked.data.via.map.size; i++) {
-      CHECK_KEY_IS_STR(unpacked, "mark")
-      if (CHECK_KEY(unpacked.data.via.map.ptr[i].key, KEY_NAME_CHAR)) {
-        if (type_u64 == kSDItemJump || type_u64 == kSDItemChange) {
-          semsg(_(READERR("mark", "has n key which is only valid for "
-                          "local and global mark entries")), initial_fpos);
-          CLEAR_GA_AND_ERROR_OUT(ad_ga);
-        }
-        CHECKED_ENTRY((unpacked.data.via.map.ptr[i].val.type
-                       == MSGPACK_OBJECT_POSITIVE_INTEGER),
-                      "has n key value which is not an unsigned integer",
-                      "mark", unpacked.data.via.map.ptr[i].val,
-                      entry->data.filemark.name, u64, TOCHAR);
+
+    if (HAS_KEY(&it, _shada_mark, n)) {
+      if (type_u64 == kSDItemJump || type_u64 == kSDItemChange) {
+        semsg(_(READERR("mark", "has n key which is only valid for "
+                        "local and global mark entries")), initial_fpos);
+        goto shada_read_next_item_error;
       }
-      INTEGER_KEY(unpacked, "mark", KEY_LNUM, entry->data.filemark.mark.lnum)
-      INTEGER_KEY(unpacked, "mark", KEY_COL, entry->data.filemark.mark.col)
-      STRING_KEY(unpacked, "mark", KEY_FILE, entry->data.filemark.fname)
-      ADDITIONAL_KEY(unpacked)
+      entry->data.filemark.name = (char)it.n;
+    }
+
+    if (HAS_KEY(&it, _shada_mark, l)) {
+      entry->data.filemark.mark.lnum = (linenr_T)it.l;
     }
+    if (HAS_KEY(&it, _shada_mark, c)) {
+      entry->data.filemark.mark.col = (colnr_T)it.c;
+    }
+    if (HAS_KEY(&it, _shada_mark, f)) {
+      entry->data.filemark.fname = xmemdupz(it.f.data, it.f.size);
+    }
+
     if (entry->data.filemark.fname == NULL) {
       semsg(_(READERR("mark", "is missing file name")), initial_fpos);
-      CLEAR_GA_AND_ERROR_OUT(ad_ga);
+      goto shada_read_next_item_error;
     }
     if (entry->data.filemark.mark.lnum <= 0) {
       semsg(_(READERR("mark", "has invalid line number")), initial_fpos);
-      CLEAR_GA_AND_ERROR_OUT(ad_ga);
+      goto shada_read_next_item_error;
     }
     if (entry->data.filemark.mark.col < 0) {
       semsg(_(READERR("mark", "has invalid column number")), initial_fpos);
-      CLEAR_GA_AND_ERROR_OUT(ad_ga);
+      goto shada_read_next_item_error;
     }
-    SET_ADDITIONAL_DATA(entry->data.filemark.additional_data, "mark");
     break;
   }
   case kSDItemRegister: {
-    if (unpacked.data.type != MSGPACK_OBJECT_MAP) {
-      semsg(_(READERR("register", "is not a dictionary")), initial_fpos);
+    Dict(_shada_register) it = { 0 };
+    if (!unpack_keydict(&it, DictHash(_shada_register), &ad, &read_ptr, &read_size, &error_alloc)) {
+      semsg(_(READERR("register", "%s")), initial_fpos, error_alloc);
+      kv_destroy(it.rc);
       goto shada_read_next_item_error;
     }
-    garray_T ad_ga;
-    ga_init(&ad_ga, sizeof(*(unpacked.data.via.map.ptr)), 1);
-    for (size_t i = 0; i < unpacked.data.via.map.size; i++) {
-      CHECK_KEY_IS_STR(unpacked, "register")
-      if (CHECK_KEY(unpacked.data.via.map.ptr[i].key,
-                    REG_KEY_CONTENTS)) {
-        if (unpacked.data.via.map.ptr[i].val.type != MSGPACK_OBJECT_ARRAY) {
-          semsg(_(READERR("register",
-                          "has " REG_KEY_CONTENTS
-                          " key with non-array value")),
-                initial_fpos);
-          CLEAR_GA_AND_ERROR_OUT(ad_ga);
-        }
-        if (unpacked.data.via.map.ptr[i].val.via.array.size == 0) {
-          semsg(_(READERR("register",
-                          "has " REG_KEY_CONTENTS " key with empty array")),
-                initial_fpos);
-          CLEAR_GA_AND_ERROR_OUT(ad_ga);
-        }
-        const msgpack_object_array arr =
-          unpacked.data.via.map.ptr[i].val.via.array;
-        for (size_t j = 0; j < arr.size; j++) {
-          if (arr.ptr[j].type != MSGPACK_OBJECT_BIN) {
-            semsg(_(READERR("register", "has " REG_KEY_CONTENTS " array "
-                            "with non-binary value")), initial_fpos);
-            CLEAR_GA_AND_ERROR_OUT(ad_ga);
-          }
-        }
-        entry->data.reg.contents_size = arr.size;
-        entry->data.reg.contents = xmalloc(arr.size * sizeof(char *));
-        for (size_t j = 0; j < arr.size; j++) {
-          entry->data.reg.contents[j] = BIN_CONVERTED(arr.ptr[j].via.bin);
-        }
-      }
-      BOOLEAN_KEY(unpacked, "register", REG_KEY_UNNAMED,
-                  entry->data.reg.is_unnamed)
-      TYPED_KEY(unpacked, "register", REG_KEY_TYPE, "an unsigned integer",
-                entry->data.reg.type, POSITIVE_INTEGER, u64, TOU8)
-      TYPED_KEY(unpacked, "register", KEY_NAME_CHAR, "an unsigned integer",
-                entry->data.reg.name, POSITIVE_INTEGER, u64, TOCHAR)
-      TYPED_KEY(unpacked, "register", REG_KEY_WIDTH, "an unsigned integer",
-                entry->data.reg.width, POSITIVE_INTEGER, u64, TOSIZE)
-      ADDITIONAL_KEY(unpacked)
-    }
-    if (entry->data.reg.contents == NULL) {
-      semsg(_(READERR("register", "has missing " REG_KEY_CONTENTS " array")),
+    if (it.rc.size == 0) {
+      semsg(_(READERR("register",
+                      "has " KEY_NAME2(REG_KEY_CONTENTS) " key with missing or empty array")),
             initial_fpos);
-      CLEAR_GA_AND_ERROR_OUT(ad_ga);
+      goto shada_read_next_item_error;
     }
-    SET_ADDITIONAL_DATA(entry->data.reg.additional_data, "register");
+    entry->data.reg.contents_size = it.rc.size;
+    entry->data.reg.contents = xmalloc(it.rc.size * sizeof(char *));
+    for (size_t j = 0; j < it.rc.size; j++) {
+      entry->data.reg.contents[j] = xmemdupz(it.rc.items[j].data, it.rc.items[j].size);
+    }
+    kv_destroy(it.rc);
+
+#define REGISTER_VAL(name, loc, type) \
+  if (HAS_KEY(&it, _shada_register, name)) { \
+    loc = (type)it.name; \
+  }
+    REGISTER_VAL(REG_KEY_UNNAMED, entry->data.reg.is_unnamed, bool)
+    REGISTER_VAL(REG_KEY_TYPE, entry->data.reg.type, uint8_t)
+    REGISTER_VAL(KEY_NAME_CHAR, entry->data.reg.name, char)
+    REGISTER_VAL(REG_KEY_WIDTH, entry->data.reg.width, size_t)
     break;
   }
   case kSDItemHistoryEntry: {
-    if (unpacked.data.type != MSGPACK_OBJECT_ARRAY) {
-      semsg(_(READERR("history", "is not an array")), initial_fpos);
-      goto shada_read_next_item_error;
-    }
-    if (unpacked.data.via.array.size < 2) {
-      semsg(_(READERR("history", "does not have enough elements")),
-            initial_fpos);
+    ssize_t len = unpack_array(&read_ptr, &read_size);
+
+    if (len < 2) {
+      semsg(_(READERR("history", "is not an array with enough elements")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    if (unpacked.data.via.array.ptr[0].type
-        != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-      semsg(_(READERR("history", "has wrong history type type")),
-            initial_fpos);
+    Integer hist_type;
+    if (!unpack_integer(&read_ptr, &read_size, &hist_type)) {
+      semsg(_(READERR("history", "has wrong history type type")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    if (unpacked.data.via.array.ptr[1].type
-        != MSGPACK_OBJECT_BIN) {
-      semsg(_(READERR("history", "has wrong history string type")),
-            initial_fpos);
+    const String item = unpack_string(&read_ptr, &read_size);
+    if (!item.data) {
+      semsg(_(READERR("history", "has wrong history string type")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    if (memchr(unpacked.data.via.array.ptr[1].via.bin.ptr, 0,
-               unpacked.data.via.array.ptr[1].via.bin.size) != NULL) {
-      semsg(_(READERR("history", "contains string with zero byte inside")),
-            initial_fpos);
+    if (memchr(item.data, 0, item.size) != NULL) {
+      semsg(_(READERR("history", "contains string with zero byte inside")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    entry->data.history_item.histtype =
-      (uint8_t)unpacked.data.via.array.ptr[0].via.u64;
-    const bool is_hist_search =
-      entry->data.history_item.histtype == HIST_SEARCH;
+    entry->data.history_item.histtype = (uint8_t)hist_type;
+    const bool is_hist_search = entry->data.history_item.histtype == HIST_SEARCH;
     if (is_hist_search) {
-      if (unpacked.data.via.array.size < 3) {
+      if (len < 3) {
         semsg(_(READERR("search history",
                         "does not have separator character")), initial_fpos);
         goto shada_read_next_item_error;
       }
-      if (unpacked.data.via.array.ptr[2].type
-          != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-        semsg(_(READERR("search history",
-                        "has wrong history separator type")), initial_fpos);
+      Integer sep_type;
+      if (!unpack_integer(&read_ptr, &read_size, &sep_type)) {
+        semsg(_(READERR("search history", "has wrong history separator type")), initial_fpos);
         goto shada_read_next_item_error;
       }
-      entry->data.history_item.sep =
-        (char)unpacked.data.via.array.ptr[2].via.u64;
+      entry->data.history_item.sep = (char)sep_type;
     }
-    size_t strsize;
-    strsize = (
-               unpacked.data.via.array.ptr[1].via.bin.size
-               + 1  // Zero byte
-               + 1);  // Separator character
+    size_t strsize = (item.size
+                      + 1  // Zero byte
+                      + 1);  // Separator character
     entry->data.history_item.string = xmalloc(strsize);
-    memcpy(entry->data.history_item.string,
-           unpacked.data.via.array.ptr[1].via.bin.ptr,
-           unpacked.data.via.array.ptr[1].via.bin.size);
+    memcpy(entry->data.history_item.string, item.data, item.size);
     entry->data.history_item.string[strsize - 2] = 0;
-    entry->data.history_item.string[strsize - 1] =
-      entry->data.history_item.sep;
-    SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, (2 + is_hist_search),
-                            entry->data.history_item.additional_elements,
-                            "history");
+    entry->data.history_item.string[strsize - 1] = entry->data.history_item.sep;
+    read_additional_array_elements = (uint32_t)(len - (2 + is_hist_search));
     break;
   }
   case kSDItemVariable: {
-    if (unpacked.data.type != MSGPACK_OBJECT_ARRAY) {
-      semsg(_(READERR("variable", "is not an array")), initial_fpos);
-      goto shada_read_next_item_error;
-    }
-    if (unpacked.data.via.array.size < 2) {
-      semsg(_(READERR("variable", "does not have enough elements")),
-            initial_fpos);
+    ssize_t len = unpack_array(&read_ptr, &read_size);
+
+    if (len < 2) {
+      semsg(_(READERR("variable", "is not an array with enough elements")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    if (unpacked.data.via.array.ptr[0].type != MSGPACK_OBJECT_BIN) {
-      semsg(_(READERR("variable", "has wrong variable name type")),
-            initial_fpos);
+
+    String name = unpack_string(&read_ptr, &read_size);
+
+    if (!name.data) {
+      semsg(_(READERR("variable", "has wrong variable name type")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    entry->data.global_var.name =
-      xmemdupz(unpacked.data.via.array.ptr[0].via.bin.ptr,
-               unpacked.data.via.array.ptr[0].via.bin.size);
-    SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, 2,
-                            entry->data.global_var.additional_elements,
-                            "variable");
+    entry->data.global_var.name = xmemdupz(name.data, name.size);
+
+    String binval = unpack_string(&read_ptr, &read_size);
+
     bool is_blob = false;
-    // A msgpack BIN could be a String or Blob; an additional VAR_TYPE_BLOB
-    // element is stored with Blobs which can be used to differentiate them
-    if (unpacked.data.via.array.ptr[1].type == MSGPACK_OBJECT_BIN) {
-      const listitem_T *type_item
-        = tv_list_first(entry->data.global_var.additional_elements);
-      if (type_item != NULL) {
-        const typval_T *type_tv = TV_LIST_ITEM_TV(type_item);
-        if (type_tv->v_type != VAR_NUMBER
-            || type_tv->vval.v_number != VAR_TYPE_BLOB) {
+    if (binval.data) {
+      if (len > 2) {
+        // A msgpack BIN could be a String or Blob; an additional VAR_TYPE_BLOB
+        // element is stored with Blobs which can be used to differentiate them
+        Integer type;
+        if (!unpack_integer(&read_ptr, &read_size, &type) || type != VAR_TYPE_BLOB) {
           semsg(_(READERR("variable", "has wrong variable type")),
                 initial_fpos);
           goto shada_read_next_item_error;
         }
         is_blob = true;
       }
+      entry->data.global_var.value = decode_string(binval.data, binval.size, is_blob, false);
+    } else {
+      int status = unpack_typval(&read_ptr, &read_size, &entry->data.global_var.value);
+      if (status != MPACK_OK) {
+        semsg(_(READERR("variable", "has value that cannot "
+                        "be converted to the Vimscript value")), initial_fpos);
+        goto shada_read_next_item_error;
+      }
     }
-    if (is_blob) {
-      const msgpack_object_bin *const bin
-        = &unpacked.data.via.array.ptr[1].via.bin;
-      blob_T *const blob = tv_blob_alloc();
-      ga_concat_len(&blob->bv_ga, bin->ptr, (size_t)bin->size);
-      tv_blob_set_ret(&entry->data.global_var.value, blob);
-    } else if (msgpack_to_vim(unpacked.data.via.array.ptr[1],
-                              &(entry->data.global_var.value)) == FAIL) {
-      semsg(_(READERR("variable", "has value that cannot "
-                      "be converted to the Vimscript value")), initial_fpos);
-      goto shada_read_next_item_error;
-    }
+    read_additional_array_elements = (uint32_t)(len - 2 - (is_blob ? 1 : 0));
     break;
   }
-  case kSDItemSubString:
-    if (unpacked.data.type != MSGPACK_OBJECT_ARRAY) {
-      semsg(_(READERR("sub string", "is not an array")), initial_fpos);
-      goto shada_read_next_item_error;
-    }
-    if (unpacked.data.via.array.size < 1) {
-      semsg(_(READERR("sub string", "does not have enough elements")),
-            initial_fpos);
+  case kSDItemSubString: {
+    ssize_t len = unpack_array(&read_ptr, &read_size);
+
+    if (len < 1) {
+      semsg(_(READERR("sub string", "is not an array with enough elements")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    if (unpacked.data.via.array.ptr[0].type != MSGPACK_OBJECT_BIN) {
-      semsg(_(READERR("sub string", "has wrong sub string type")),
-            initial_fpos);
+
+    String sub = unpack_string(&read_ptr, &read_size);
+    if (!sub.data) {
+      semsg(_(READERR("sub string", "has wrong sub string type")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    entry->data.sub_string.sub =
-      BIN_CONVERTED(unpacked.data.via.array.ptr[0].via.bin);
-    SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, 1,
-                            entry->data.sub_string.additional_elements,
-                            "sub string");
+    entry->data.sub_string.sub = xmemdupz(sub.data, sub.size);
+    read_additional_array_elements = (uint32_t)(len - 1);
     break;
-  case kSDItemBufferList:
-    if (unpacked.data.type != MSGPACK_OBJECT_ARRAY) {
+  }
+  case kSDItemBufferList: {
+    ssize_t len = unpack_array(&read_ptr, &read_size);
+    if (len < 0) {
       semsg(_(READERR("buffer list", "is not an array")), initial_fpos);
       goto shada_read_next_item_error;
     }
-    if (unpacked.data.via.array.size == 0) {
+    if (len == 0) {
       break;
     }
-    entry->data.buffer_list.buffers =
-      xcalloc(unpacked.data.via.array.size,
-              sizeof(*entry->data.buffer_list.buffers));
-    for (size_t i = 0; i < unpacked.data.via.array.size; i++) {
+    entry->data.buffer_list.buffers = xcalloc((size_t)len,
+                                              sizeof(*entry->data.buffer_list.buffers));
+    for (size_t i = 0; i < (size_t)len; i++) {
       entry->data.buffer_list.size++;
-      msgpack_unpacked unpacked_2 = (msgpack_unpacked) {
-        .data = unpacked.data.via.array.ptr[i],
-      };
-      {
-        if (unpacked_2.data.type != MSGPACK_OBJECT_MAP) {
-          semsg(_(RERR "Error while reading ShaDa file: "
-                  "buffer list at position %" PRIu64 " "
-                  "contains entry that is not a dictionary"),
-                initial_fpos);
-          goto shada_read_next_item_error;
-        }
-        entry->data.buffer_list.buffers[i].pos = default_pos;
-        garray_T ad_ga;
-        ga_init(&ad_ga, sizeof(*(unpacked_2.data.via.map.ptr)), 1);
-        {
-          // XXX: Temporarily reassign `i` because the macros depend on it.
-          const size_t j = i;
-          {
-            for (i = 0; i < unpacked_2.data.via.map.size; i++) {
-              CHECK_KEY_IS_STR(unpacked_2, "buffer list entry")
-              INTEGER_KEY(unpacked_2, "buffer list entry", KEY_LNUM,
-                          entry->data.buffer_list.buffers[j].pos.lnum)
-              INTEGER_KEY(unpacked_2, "buffer list entry", KEY_COL,
-                          entry->data.buffer_list.buffers[j].pos.col)
-              STRING_KEY(unpacked_2, "buffer list entry", KEY_FILE,
-                         entry->data.buffer_list.buffers[j].fname)
-              ADDITIONAL_KEY(unpacked_2)
-            }
-          }
-          i = j;  // XXX: Restore `i`.
-        }
-        if (entry->data.buffer_list.buffers[i].pos.lnum <= 0) {
-          semsg(_(RERR "Error while reading ShaDa file: "
-                  "buffer list at position %" PRIu64 " "
-                  "contains entry with invalid line number"),
-                initial_fpos);
-          CLEAR_GA_AND_ERROR_OUT(ad_ga);
-        }
-        if (entry->data.buffer_list.buffers[i].pos.col < 0) {
-          semsg(_(RERR "Error while reading ShaDa file: "
-                  "buffer list at position %" PRIu64 " "
-                  "contains entry with invalid column number"),
-                initial_fpos);
-          CLEAR_GA_AND_ERROR_OUT(ad_ga);
-        }
-        if (entry->data.buffer_list.buffers[i].fname == NULL) {
-          semsg(_(RERR "Error while reading ShaDa file: "
-                  "buffer list at position %" PRIu64 " "
-                  "contains entry that does not have a file name"),
-                initial_fpos);
-          CLEAR_GA_AND_ERROR_OUT(ad_ga);
-        }
-        SET_ADDITIONAL_DATA(entry->data.buffer_list.buffers[i].additional_data,
-                            "buffer list entry");
+      Dict(_shada_buflist_item) it = { 0 };
+      AdditionalDataBuilder it_ad = KV_INITIAL_VALUE;
+      if (!unpack_keydict(&it, DictHash(_shada_buflist_item), &it_ad, &read_ptr, &read_size,
+                          &error_alloc)) {
+        semsg(_(RERR "Error while reading ShaDa file: "
+                "buffer list at position %" PRIu64 " contains entry that %s"),
+              initial_fpos, error_alloc);
+        kv_destroy(it_ad);
+        goto shada_read_next_item_error;
+      }
+      struct buffer_list_buffer *e = &entry->data.buffer_list.buffers[i];
+      e->additional_data = (AdditionalData *)it_ad.items;
+      e->pos = default_pos;
+      if (HAS_KEY(&it, _shada_buflist_item, l)) {
+        e->pos.lnum = (linenr_T)it.l;
+      }
+      if (HAS_KEY(&it, _shada_buflist_item, c)) {
+        e->pos.col = (colnr_T)it.c;
+      }
+      if (HAS_KEY(&it, _shada_buflist_item, f)) {
+        e->fname = xmemdupz(it.f.data, it.f.size);
+      }
+
+      if (e->pos.lnum <= 0) {
+        semsg(_(RERR "Error while reading ShaDa file: "
+                "buffer list at position %" PRIu64 " "
+                "contains entry with invalid line number"),
+              initial_fpos);
+        goto shada_read_next_item_error;
+      }
+      if (e->pos.col < 0) {
+        semsg(_(RERR "Error while reading ShaDa file: "
+                "buffer list at position %" PRIu64 " "
+                "contains entry with invalid column number"),
+              initial_fpos);
+        goto shada_read_next_item_error;
+      }
+      if (e->fname == NULL) {
+        semsg(_(RERR "Error while reading ShaDa file: "
+                "buffer list at position %" PRIu64 " "
+                "contains entry that does not have a file name"),
+              initial_fpos);
+        goto shada_read_next_item_error;
       }
     }
     break;
+  }
   case kSDItemMissing:
   case kSDItemUnknown:
     abort();
   }
+
+  for (uint32_t i = 0; i < read_additional_array_elements; i++) {
+    const char *item_start = read_ptr;
+    int status = unpack_skip(&read_ptr, &read_size);
+    if (status) {
+      goto shada_read_next_item_error;
+    }
+
+    push_additional_data(&ad, item_start, (size_t)(read_ptr - item_start));
+  }
+
+  if (read_size) {
+    semsg(_(READERR("item", "additional bytes")), initial_fpos);
+    goto shada_read_next_item_error;
+  }
+
   entry->type = (ShadaEntryType)type_u64;
+  entry->additional_data = (AdditionalData *)ad.items;
   ret = kSDReadStatusSuccess;
 shada_read_next_item_end:
-  if (buf != NULL) {
-    msgpack_unpacked_destroy(&unpacked);
+  if (buf_allocated) {
     xfree(buf);
   }
   return ret;
@@ -3823,26 +3520,10 @@ shada_read_next_item_error:
   entry->type = (ShadaEntryType)type_u64;
   shada_free_shada_entry(entry);
   entry->type = kSDItemMissing;
+  xfree(error_alloc);
+  kv_destroy(ad);
   goto shada_read_next_item_end;
 }
-#undef BIN_CONVERTED
-#undef CHECK_KEY
-#undef BOOLEAN_KEY
-#undef CONVERTED_STRING_KEY
-#undef STRING_KEY
-#undef ADDITIONAL_KEY
-#undef ID
-#undef BINDUP
-#undef TOCHAR
-#undef TOINT
-#undef TYPED_KEY
-#undef INT_KEY
-#undef INTEGER_KEY
-#undef TOU8
-#undef TOSIZE
-#undef SET_ADDITIONAL_DATA
-#undef SET_ADDITIONAL_ELEMENTS
-#undef CLEAR_GA_AND_ERROR_OUT
 
 /// Check whether "name" is on removable media (according to 'shada')
 ///
@@ -3919,9 +3600,9 @@ static inline size_t shada_init_jumps(PossiblyFreedShadaEntry *jumps,
             .name = NUL,
             .mark = fm.fmark.mark,
             .fname = (char *)fname,
-            .additional_data = fm.fmark.additional_data,
           }
-        }
+        },
+        .additional_data = fm.fmark.additional_data,
       }
     };
   } while (jump_iter != NULL);
@@ -4013,9 +3694,9 @@ String shada_encode_gvars(void)
           .global_var = {
             .name = (char *)name,
             .value = tgttv,
-            .additional_elements = NULL,
           }
-        }
+        },
+        .additional_data = NULL,
       }, 0);
       if (kSDWriteFailed == r) {
         abort();
diff --git a/src/nvim/shada.h b/src/nvim/shada.h
index 58689a5bd7..7d736dadc7 100644
--- a/src/nvim/shada.h
+++ b/src/nvim/shada.h
@@ -1,7 +1,5 @@
 #pragma once
 
-#include <msgpack.h>  // IWYU pragma: keep
-
 #include "nvim/api/private/defs.h"
 
 /// Flags for shada_read_file and children
diff --git a/src/nvim/types_defs.h b/src/nvim/types_defs.h
index 934159b9d9..2dd2b01adf 100644
--- a/src/nvim/types_defs.h
+++ b/src/nvim/types_defs.h
@@ -56,3 +56,9 @@ typedef struct regprog regprog_T;
 typedef struct syn_state synstate_T;
 typedef struct terminal Terminal;
 typedef struct window_S win_T;
+
+typedef struct {
+  uint32_t nitems;
+  uint32_t nbytes;
+  char data[];
+} AdditionalData;
diff --git a/src/nvim/ui_client.c b/src/nvim/ui_client.c
index 4f36cae4b2..839cf965b0 100644
--- a/src/nvim/ui_client.c
+++ b/src/nvim/ui_client.c
@@ -174,7 +174,7 @@ static HlAttrs ui_client_dict2hlattrs(Dictionary d, bool rgb)
 {
   Error err = ERROR_INIT;
   Dict(highlight) dict = KEYDICT_INIT;
-  if (!api_dict_to_keydict(&dict, KeyDict_highlight_get_field, d, &err)) {
+  if (!api_dict_to_keydict(&dict, DictHash(highlight), d, &err)) {
     // TODO(bfredl): log "err"
     return HLATTRS_INIT;
   }
diff --git a/src/nvim/undo.c b/src/nvim/undo.c
index c403e94184..15c8e0b283 100644
--- a/src/nvim/undo.c
+++ b/src/nvim/undo.c
@@ -343,8 +343,7 @@ static OptInt get_undolevel(buf_T *buf)
 static inline void zero_fmark_additional_data(fmark_T *fmarks)
 {
   for (size_t i = 0; i < NMARKS; i++) {
-    tv_dict_unref(fmarks[i].additional_data);
-    fmarks[i].additional_data = NULL;
+    XFREE_CLEAR(fmarks[i].additional_data);
   }
 }
 
diff --git a/test/functional/shada/errors_spec.lua b/test/functional/shada/errors_spec.lua
index e000d0988b..321744f7dd 100644
--- a/test/functional/shada/errors_spec.lua
+++ b/test/functional/shada/errors_spec.lua
@@ -58,7 +58,7 @@ describe('ShaDa error handling', function()
   it('fails on search pattern item with zero length', function()
     wshada('\002\000\000')
     eq(
-      'Vim(rshada):E576: Failed to parse ShaDa file: incomplete msgpack string at position 3',
+      'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 is not a dictionary',
       exc_exec(sdrcmd())
     )
   end)
@@ -89,18 +89,10 @@ describe('ShaDa error handling', function()
 
   it('fails on search pattern item with invalid byte', function()
     -- 195 (== 0xC1) cannot start any valid messagepack entry (the only byte
-    -- that cannot do this). Specifically unpack_template.h contains
-    --
-    --     //case 0xc1:  // string
-    --     //  again_terminal_trail(NEXT_CS(p), p+1);
-    --
-    -- (literally: commented out code) which means that in place of this code
-    -- `goto _failed` is used from default: case. I do not know any other way to
-    -- get MSGPACK_UNPACK_PARSE_ERROR and not MSGPACK_UNPACK_CONTINUE or
-    -- MSGPACK_UNPACK_EXTRA_BYTES.
+    -- that cannot do this)
     wshada('\002\000\001\193')
     eq(
-      'Vim(rshada):E576: Failed to parse ShaDa file due to a msgpack parser error at position 3',
+      'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 is not a dictionary',
       exc_exec(sdrcmd())
     )
   end)
@@ -108,7 +100,7 @@ describe('ShaDa error handling', function()
   it('fails on search pattern item with incomplete map', function()
     wshada('\002\000\001\129')
     eq(
-      'Vim(rshada):E576: Failed to parse ShaDa file: incomplete msgpack string at position 3',
+      'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has key value which is not a string',
       exc_exec(sdrcmd())
     )
   end)
@@ -124,7 +116,7 @@ describe('ShaDa error handling', function()
   it('fails on search pattern with extra bytes', function()
     wshada('\002\000\002\128\000')
     eq(
-      'Vim(rshada):E576: Failed to parse ShaDa file: extra bytes in msgpack string at position 3',
+      'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has no pattern',
       exc_exec(sdrcmd())
     )
   end)
@@ -138,15 +130,6 @@ describe('ShaDa error handling', function()
   end)
 
   -- sp entry is here because it causes an allocation.
-  it('fails on search pattern item with BIN key', function()
-    wshada('\002\000\014\131\162sp\196\001a\162sX\192\196\000\000')
-    eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has key which is not a string',
-      exc_exec(sdrcmd())
-    )
-  end)
-
-  -- sp entry is here because it causes an allocation.
   it('fails on search pattern item with empty key', function()
     wshada('\002\000\013\131\162sp\196\001a\162sX\192\160\000')
     eq(
@@ -235,22 +218,12 @@ describe('ShaDa error handling', function()
     )
   end)
 
-  it('fails on search pattern item with STR pat key value', function()
-    wshada('\002\000\011\130\162sX\192\162sp\162sp')
-    eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: search pattern entry at position 0 has sp key value which is not a binary',
-      exc_exec(sdrcmd())
-    )
-  end)
-
   for _, v in ipairs({
     { name = 'global mark', mpack = '\007' },
     { name = 'jump', mpack = '\008' },
     { name = 'local mark', mpack = '\010' },
     { name = 'change', mpack = '\011' },
   }) do
-    local is_mark_test = ({ ['global mark'] = true, ['local mark'] = true })[v.name]
-
     it('fails on ' .. v.name .. ' item with NIL value', function()
       wshada(v.mpack .. '\000\001\192')
       eq(
@@ -260,15 +233,6 @@ describe('ShaDa error handling', function()
     end)
 
     -- f entry is here because it causes an allocation.
-    it('fails on ' .. v.name .. ' item with BIN key', function()
-      wshada(v.mpack .. '\000\013\131\161f\196\001/\162mX\192\196\000\000')
-      eq(
-        'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has key which is not a string',
-        exc_exec(sdrcmd())
-      )
-    end)
-
-    -- f entry is here because it causes an allocation.
     it('fails on ' .. v.name .. ' item with empty key', function()
       wshada(v.mpack .. '\000\012\131\161f\196\001/\162mX\192\160\000')
       eq(
@@ -312,9 +276,7 @@ describe('ShaDa error handling', function()
     it('fails on ' .. v.name .. ' item with STR n key value', function()
       wshada(v.mpack .. '\000\011\130\162mX\192\161n\163spa')
       eq(
-        is_mark_test
-            and 'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has n key value which is not an unsigned integer'
-          or 'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has n key which is only valid for local and global mark entries',
+        'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has n key value which is not an integer',
         exc_exec(sdrcmd())
       )
     end)
@@ -334,14 +296,6 @@ describe('ShaDa error handling', function()
         exc_exec(sdrcmd())
       )
     end)
-
-    it('fails on ' .. v.name .. ' item with STR f key value', function()
-      wshada(v.mpack .. '\000\010\130\162mX\192\161f\162sp')
-      eq(
-        'Vim(rshada):E575: Error while reading ShaDa file: mark entry at position 0 has f key value which is not a binary',
-        exc_exec(sdrcmd())
-      )
-    end)
   end
 
   it('fails on register item with NIL value', function()
@@ -354,15 +308,6 @@ describe('ShaDa error handling', function()
 
   -- rc entry is here because it causes an allocation
   it('fails on register item with BIN key', function()
-    wshada('\005\000\015\131\162rc\145\196\001a\162rX\192\196\000\000')
-    eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has key which is not a string',
-      exc_exec(sdrcmd())
-    )
-  end)
-
-  -- rc entry is here because it causes an allocation
-  it('fails on register item with BIN key', function()
     wshada('\005\000\014\131\162rc\145\196\001a\162rX\192\160\000')
     eq(
       'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has empty key',
@@ -373,7 +318,7 @@ describe('ShaDa error handling', function()
   it('fails on register item with NIL rt key value', function()
     wshada('\005\000\009\130\162rX\192\162rt\192')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rt key value which is not an unsigned integer',
+      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rt key value which is not an integer',
       exc_exec(sdrcmd())
     )
   end)
@@ -381,7 +326,7 @@ describe('ShaDa error handling', function()
   it('fails on register item with NIL rw key value', function()
     wshada('\005\000\009\130\162rX\192\162rw\192')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rw key value which is not an unsigned integer',
+      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rw key value which is not an integer',
       exc_exec(sdrcmd())
     )
   end)
@@ -397,7 +342,7 @@ describe('ShaDa error handling', function()
   it('fails on register item with empty rc key value', function()
     wshada('\005\000\009\130\162rX\192\162rc\144')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc key with empty array',
+      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc key with missing or empty array',
       exc_exec(sdrcmd())
     )
   end)
@@ -413,7 +358,7 @@ describe('ShaDa error handling', function()
   it('fails on register item without rc array', function()
     wshada('\005\000\009\129\162rX\146\196\001a\192')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has missing rc array',
+      'Vim(rshada):E575: Error while reading ShaDa file: register entry at position 0 has rc key with missing or empty array',
       exc_exec(sdrcmd())
     )
   end)
@@ -421,7 +366,7 @@ describe('ShaDa error handling', function()
   it('fails on history item with NIL value', function()
     wshada('\004\000\001\192')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array',
+      'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array with enough elements',
       exc_exec(sdrcmd())
     )
   end)
@@ -429,7 +374,7 @@ describe('ShaDa error handling', function()
   it('fails on history item with empty value', function()
     wshada('\004\000\001\144')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 does not have enough elements',
+      'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array with enough elements',
       exc_exec(sdrcmd())
     )
   end)
@@ -437,7 +382,7 @@ describe('ShaDa error handling', function()
   it('fails on history item with single element value', function()
     wshada('\004\000\002\145\000')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 does not have enough elements',
+      'Vim(rshada):E575: Error while reading ShaDa file: history entry at position 0 is not an array with enough elements',
       exc_exec(sdrcmd())
     )
   end)
@@ -485,7 +430,7 @@ describe('ShaDa error handling', function()
   it('fails on variable item with NIL value', function()
     wshada('\006\000\001\192')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array',
+      'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array with enough elements',
       exc_exec(sdrcmd())
     )
   end)
@@ -493,7 +438,7 @@ describe('ShaDa error handling', function()
   it('fails on variable item with empty value', function()
     wshada('\006\000\001\144')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 does not have enough elements',
+      'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array with enough elements',
       exc_exec(sdrcmd())
     )
   end)
@@ -501,7 +446,7 @@ describe('ShaDa error handling', function()
   it('fails on variable item with single element value', function()
     wshada('\006\000\002\145\000')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 does not have enough elements',
+      'Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 is not an array with enough elements',
       exc_exec(sdrcmd())
     )
   end)
@@ -525,7 +470,7 @@ describe('ShaDa error handling', function()
   it('fails on replacement item with NIL value', function()
     wshada('\003\000\001\192')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 is not an array',
+      'Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 is not an array with enough elements',
       exc_exec(sdrcmd())
     )
   end)
@@ -533,7 +478,7 @@ describe('ShaDa error handling', function()
   it('fails on replacement item with empty value', function()
     wshada('\003\000\001\144')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 does not have enough elements',
+      'Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 is not an array with enough elements',
       exc_exec(sdrcmd())
     )
   end)
@@ -577,7 +522,7 @@ describe('ShaDa error handling', function()
     nvim_command('set shada+=%')
     wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161l\192')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: buffer list entry entry at position 0 has l key value which is not an integer',
+      'Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry that has l key value which is not an integer',
       exc_exec(sdrcmd())
     )
   end)
@@ -613,7 +558,7 @@ describe('ShaDa error handling', function()
     nvim_command('set shada+=%')
     wshada('\009\000\017\146\129\161f\196\001/\130\161f\196\002/a\161c\192')
     eq(
-      'Vim(rshada):E575: Error while reading ShaDa file: buffer list entry entry at position 0 has c key value which is not an integer',
+      'Vim(rshada):E575: Error while reading ShaDa file: buffer list at position 0 contains entry that has c key value which is not an integer',
       exc_exec(sdrcmd())
     )
   end)
diff --git a/test/unit/eval/encode_spec.lua b/test/unit/eval/encode_spec.lua
index 5b9188163e..9f193bc2f9 100644
--- a/test/unit/eval/encode_spec.lua
+++ b/test/unit/eval/encode_spec.lua
@@ -21,81 +21,81 @@ describe('encode_list_write()', function()
 
   itp('writes empty string', function()
     local l = list()
-    eq(0, encode_list_write(l, ''))
+    encode_list_write(l, '')
     eq({ [type_key] = list_type }, lst2tbl(l))
   end)
 
   itp('writes ASCII string literal with printable characters', function()
     local l = list()
-    eq(0, encode_list_write(l, 'abc'))
+    encode_list_write(l, 'abc')
     eq({ 'abc' }, lst2tbl(l))
   end)
 
   itp('writes string starting with NL', function()
     local l = list()
-    eq(0, encode_list_write(l, '\nabc'))
+    encode_list_write(l, '\nabc')
     eq({ null_string, 'abc' }, lst2tbl(l))
   end)
 
   itp('writes string starting with NL twice', function()
     local l = list()
-    eq(0, encode_list_write(l, '\nabc'))
+    encode_list_write(l, '\nabc')
     eq({ null_string, 'abc' }, lst2tbl(l))
-    eq(0, encode_list_write(l, '\nabc'))
+    encode_list_write(l, '\nabc')
     eq({ null_string, 'abc', 'abc' }, lst2tbl(l))
   end)
 
   itp('writes string ending with NL', function()
     local l = list()
-    eq(0, encode_list_write(l, 'abc\n'))
+    encode_list_write(l, 'abc\n')
     eq({ 'abc', null_string }, lst2tbl(l))
   end)
 
   itp('writes string ending with NL twice', function()
     local l = list()
-    eq(0, encode_list_write(l, 'abc\n'))
+    encode_list_write(l, 'abc\n')
     eq({ 'abc', null_string }, lst2tbl(l))
-    eq(0, encode_list_write(l, 'abc\n'))
+    encode_list_write(l, 'abc\n')
     eq({ 'abc', 'abc', null_string }, lst2tbl(l))
   end)
 
   itp('writes string starting, ending and containing NL twice', function()
     local l = list()
-    eq(0, encode_list_write(l, '\na\nb\n'))
+    encode_list_write(l, '\na\nb\n')
     eq({ null_string, 'a', 'b', null_string }, lst2tbl(l))
-    eq(0, encode_list_write(l, '\na\nb\n'))
+    encode_list_write(l, '\na\nb\n')
     eq({ null_string, 'a', 'b', null_string, 'a', 'b', null_string }, lst2tbl(l))
   end)
 
   itp('writes string starting, ending and containing NUL with NL between twice', function()
     local l = list()
-    eq(0, encode_list_write(l, '\0\n\0\n\0'))
+    encode_list_write(l, '\0\n\0\n\0')
     eq({ '\n', '\n', '\n' }, lst2tbl(l))
-    eq(0, encode_list_write(l, '\0\n\0\n\0'))
+    encode_list_write(l, '\0\n\0\n\0')
     eq({ '\n', '\n', '\n\n', '\n', '\n' }, lst2tbl(l))
   end)
 
   itp('writes string starting, ending and containing NL with NUL between twice', function()
     local l = list()
-    eq(0, encode_list_write(l, '\n\0\n\0\n'))
+    encode_list_write(l, '\n\0\n\0\n')
     eq({ null_string, '\n', '\n', null_string }, lst2tbl(l))
-    eq(0, encode_list_write(l, '\n\0\n\0\n'))
+    encode_list_write(l, '\n\0\n\0\n')
     eq({ null_string, '\n', '\n', null_string, '\n', '\n', null_string }, lst2tbl(l))
   end)
 
   itp('writes string containing a single NL twice', function()
     local l = list()
-    eq(0, encode_list_write(l, '\n'))
+    encode_list_write(l, '\n')
     eq({ null_string, null_string }, lst2tbl(l))
-    eq(0, encode_list_write(l, '\n'))
+    encode_list_write(l, '\n')
     eq({ null_string, null_string, null_string }, lst2tbl(l))
   end)
 
   itp('writes string containing a few NLs twice', function()
     local l = list()
-    eq(0, encode_list_write(l, '\n\n\n'))
+    encode_list_write(l, '\n\n\n')
     eq({ null_string, null_string, null_string, null_string }, lst2tbl(l))
-    eq(0, encode_list_write(l, '\n\n\n'))
+    encode_list_write(l, '\n\n\n')
     eq(
       { null_string, null_string, null_string, null_string, null_string, null_string, null_string },
       lst2tbl(l)