refactor(shada): rework msgpack decoding without msgpack-c

This also makes shada reading slightly faster due to avoiding
some copying and allocation.

Use keysets to drive decoding of msgpack maps for shada entries.

2 files changed, 15 insertions, 6 deletions

diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index 94e2d76cbe..ac3404cf86 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -19,6 +19,8 @@
 # define ArrayOf(...) Array
 # define DictionaryOf(...) Dictionary
 # define Dict(name) KeyDict_##name
+# define DictHash(name) KeyDict_##name##_get_field
+# define DictKey(name)
 # include "api/private/defs.h.inline.generated.h"
 #endif
 
@@ -88,6 +90,8 @@ typedef kvec_t(Object) Array;
 typedef struct key_value_pair KeyValuePair;
 typedef kvec_t(KeyValuePair) Dictionary;
 
+typedef kvec_t(String) StringArray;
+
 typedef enum {
   kObjectTypeNil = 0,
   kObjectTypeBoolean,
@@ -103,6 +107,10 @@ typedef enum {
   kObjectTypeTabpage,
 } ObjectType;
 
+typedef enum {
+  kUnpackTypeStringArray = -1,
+} UnpackType;
+
 /// Value by which objects represented as EXT type are shifted
 ///
 /// Subtracted when packing, added when unpacking. Used to allow moving
@@ -140,7 +148,7 @@ typedef struct {
 typedef struct {
   char *str;
   size_t ptr_off;
-  ObjectType type;  // kObjectTypeNil == untyped
+  int type;  // ObjectType or UnpackType. kObjectTypeNil == untyped
   int opt_index;
   bool is_hlgroup;
 } KeySetLink;
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index ac621b1486..017265e0d3 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -1,6 +1,5 @@
 #include <assert.h>
 #include <limits.h>
-#include <msgpack/unpack.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stddef.h>
@@ -922,15 +921,17 @@ bool api_dict_to_keydict(void *retval, FieldHashfn hashy, Dictionary dict, Error
       } else if (value->type == kObjectTypeDictionary) {
         *val = value->data.dictionary;
       } else {
-        api_err_exp(err, field->str, api_typename(field->type), api_typename(value->type));
+        api_err_exp(err, field->str, api_typename((ObjectType)field->type),
+                    api_typename(value->type));
         return false;
       }
     } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
                || field->type == kObjectTypeTabpage) {
-      if (value->type == kObjectTypeInteger || value->type == field->type) {
+      if (value->type == kObjectTypeInteger || value->type == (ObjectType)field->type) {
         *(handle_T *)mem = (handle_T)value->data.integer;
       } else {
-        api_err_exp(err, field->str, api_typename(field->type), api_typename(value->type));
+        api_err_exp(err, field->str, api_typename((ObjectType)field->type),
+                    api_typename(value->type));
         return false;
       }
     } else if (field->type == kObjectTypeLuaRef) {
@@ -980,7 +981,7 @@ Dictionary api_keydict_to_dict(void *value, KeySetLink *table, size_t max_size,
     } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
                || field->type == kObjectTypeTabpage) {
       val.data.integer = *(handle_T *)mem;
-      val.type = field->type;
+      val.type = (ObjectType)field->type;
     } else if (field->type == kObjectTypeLuaRef) {
       // do nothing
     } else {