Merge remote-tracking branch 'upstream/master' into mix_20240309

9 files changed, 381 insertions, 77 deletions

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 5737a0440f..626312b666 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -1,8 +1,5 @@
 #include <assert.h>
 #include <inttypes.h>
-#include <msgpack/object.h>
-#include <msgpack/sbuffer.h>
-#include <msgpack/unpack.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -17,7 +14,7 @@
 #include "nvim/event/defs.h"
 #include "nvim/event/loop.h"
 #include "nvim/event/multiqueue.h"
-#include "nvim/event/process.h"
+#include "nvim/event/proc.h"
 #include "nvim/event/rstream.h"
 #include "nvim/event/wstream.h"
 #include "nvim/globals.h"
@@ -31,8 +28,6 @@
 #include "nvim/msgpack_rpc/packer.h"
 #include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/os/input.h"
-#include "nvim/rbuffer.h"
-#include "nvim/rbuffer_defs.h"
 #include "nvim/types_defs.h"
 #include "nvim/ui.h"
 #include "nvim/ui_client.h"
@@ -85,11 +80,11 @@ void rpc_start(Channel *channel)
   rpc->unpacker = xcalloc(1, sizeof *rpc->unpacker);
   unpacker_init(rpc->unpacker);
   rpc->next_request_id = 1;
-  rpc->info = (Dictionary)ARRAY_DICT_INIT;
+  rpc->info = (Dict)ARRAY_DICT_INIT;
   kv_init(rpc->call_stack);
 
   if (channel->streamtype != kChannelStreamInternal) {
-    Stream *out = channel_outstream(channel);
+    RStream *out = channel_outstream(channel);
 #ifdef NVIM_LOG_DEBUG
     Stream *in = channel_instream(channel);
     DLOG("rpc ch %" PRIu64 " in-stream=%p out-stream=%p", channel->id,
@@ -202,10 +197,25 @@ Object rpc_send_call(uint64_t id, const char *method_name, Array args, ArenaMem
   return frame.errored ? NIL : frame.result;
 }
 
-static void receive_msgpack(Stream *stream, RBuffer *rbuf, size_t c, void *data, bool eof)
+static size_t receive_msgpack(RStream *stream, const char *rbuf, size_t c, void *data, bool eof)
 {
   Channel *channel = data;
   channel_incref(channel);
+  size_t consumed = 0;
+
+  DLOG("ch %" PRIu64 ": parsing %zu bytes from msgpack Stream: %p",
+       channel->id, c, (void *)stream);
+
+  if (c > 0) {
+    Unpacker *p = channel->rpc.unpacker;
+    p->read_ptr = rbuf;
+    p->read_size = c;
+    parse_msgpack(channel);
+
+    if (!unpacker_closed(p)) {
+      consumed = c - p->read_size;
+    }
+  }
 
   if (eof) {
     channel_close(channel->id, kChannelPartRpc, NULL);
@@ -213,25 +223,10 @@ static void receive_msgpack(Stream *stream, RBuffer *rbuf, size_t c, void *data,
     snprintf(buf, sizeof(buf), "ch %" PRIu64 " was closed by the client",
              channel->id);
     chan_close_with_error(channel, buf, LOGLVL_INF);
-    goto end;
-  }
-
-  DLOG("ch %" PRIu64 ": parsing %zu bytes from msgpack Stream: %p",
-       channel->id, rbuffer_size(rbuf), (void *)stream);
-
-  Unpacker *p = channel->rpc.unpacker;
-  size_t size = 0;
-  p->read_ptr = rbuffer_read_ptr(rbuf, &size);
-  p->read_size = size;
-  parse_msgpack(channel);
-
-  if (!unpacker_closed(p)) {
-    size_t consumed = size - p->read_size;
-    rbuffer_consumed_compact(rbuf, consumed);
   }
 
-end:
   channel_decref(channel);
+  return consumed;
 }
 
 static ChannelCallFrame *find_call_frame(RpcState *rpc, uint32_t request_id)
@@ -505,7 +500,7 @@ void rpc_free(Channel *channel)
   xfree(channel->rpc.unpacker);
 
   kv_destroy(channel->rpc.call_stack);
-  api_free_dictionary(channel->rpc.info);
+  api_free_dict(channel->rpc.info);
 }
 
 static void chan_close_with_error(Channel *channel, char *msg, int loglevel)
@@ -592,16 +587,16 @@ static void packer_buffer_init_channels(Channel **chans, size_t nchans, PackerBu
   packer->endptr = packer->startptr + ARENA_BLOCK_SIZE;
   packer->packer_flush = channel_flush_callback;
   packer->anydata = chans;
-  packer->anylen = nchans;
+  packer->anyint = (int64_t)nchans;
 }
 
 static void packer_buffer_finish_channels(PackerBuffer *packer)
 {
   size_t len = (size_t)(packer->ptr - packer->startptr);
   if (len > 0) {
-    WBuffer *buf = wstream_new_buffer(packer->startptr, len, packer->anylen, free_block);
+    WBuffer *buf = wstream_new_buffer(packer->startptr, len, (size_t)packer->anyint, free_block);
     Channel **chans = packer->anydata;
-    for (size_t i = 0; i < packer->anylen; i++) {
+    for (int64_t i = 0; i < packer->anyint; i++) {
       channel_write(chans[i], buf);
     }
   } else {
@@ -612,17 +607,17 @@ static void packer_buffer_finish_channels(PackerBuffer *packer)
 static void channel_flush_callback(PackerBuffer *packer)
 {
   packer_buffer_finish_channels(packer);
-  packer_buffer_init_channels(packer->anydata, packer->anylen, packer);
+  packer_buffer_init_channels(packer->anydata, (size_t)packer->anyint, packer);
 }
 
-void rpc_set_client_info(uint64_t id, Dictionary info)
+void rpc_set_client_info(uint64_t id, Dict info)
 {
   Channel *chan = find_rpc_channel(id);
   if (!chan) {
     abort();
   }
 
-  api_free_dictionary(chan->rpc.info);
+  api_free_dict(chan->rpc.info);
   chan->rpc.info = info;
 
   // Parse "type" on "info" and set "client_type"
@@ -646,9 +641,9 @@ void rpc_set_client_info(uint64_t id, Dictionary info)
   channel_info_changed(chan, false);
 }
 
-Dictionary rpc_client_info(Channel *chan)
+Dict rpc_client_info(Channel *chan)
 {
-  return copy_dictionary(chan->rpc.info, NULL);
+  return copy_dict(chan->rpc.info, NULL);
 }
 
 const char *get_client_info(Channel *chan, const char *key)
@@ -657,7 +652,7 @@ const char *get_client_info(Channel *chan, const char *key)
   if (!chan->is_rpc) {
     return NULL;
   }
-  Dictionary info = chan->rpc.info;
+  Dict info = chan->rpc.info;
   for (size_t i = 0; i < info.size; i++) {
     if (strequal(key, info.items[i].key.data)
         && info.items[i].value.type == kObjectTypeString) {
diff --git a/src/nvim/msgpack_rpc/channel.h b/src/nvim/msgpack_rpc/channel.h
index ff73a2fc53..abf1ce7bf6 100644
--- a/src/nvim/msgpack_rpc/channel.h
+++ b/src/nvim/msgpack_rpc/channel.h
@@ -12,7 +12,7 @@
 
 /// HACK: os/input.c drains this queue immediately before blocking for input.
 ///       Events on this queue are async-safe, but they need the resolved state
-///       of os_inchar(), so they are processed "just-in-time".
+///       of input_get(), so they are processed "just-in-time".
 EXTERN MultiQueue *ch_before_blocking_events INIT( = NULL);
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
diff --git a/src/nvim/msgpack_rpc/channel_defs.h b/src/nvim/msgpack_rpc/channel_defs.h
index 7dc1374964..871d4e615f 100644
--- a/src/nvim/msgpack_rpc/channel_defs.h
+++ b/src/nvim/msgpack_rpc/channel_defs.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <msgpack.h>
 #include <stdbool.h>
 #include <uv.h>
 
@@ -41,6 +40,6 @@ typedef struct {
   Unpacker *unpacker;
   uint32_t next_request_id;
   kvec_t(ChannelCallFrame *) call_stack;
-  Dictionary info;
+  Dict info;
   ClientType client_type;
 } RpcState;
diff --git a/src/nvim/msgpack_rpc/packer.c b/src/nvim/msgpack_rpc/packer.c
index cac68f76f0..b739f7ba28 100644
--- a/src/nvim/msgpack_rpc/packer.c
+++ b/src/nvim/msgpack_rpc/packer.c
@@ -8,10 +8,9 @@
 # include "msgpack_rpc/packer.c.generated.h"
 #endif
 
-static void check_buffer(PackerBuffer *packer)
+void mpack_check_buffer(PackerBuffer *packer)
 {
-  ptrdiff_t remaining = packer->endptr - packer->ptr;
-  if (remaining < MPACK_ITEM_SIZE) {
+  if (mpack_remaining(packer) < 2 * MPACK_ITEM_SIZE) {
     packer->packer_flush(packer);
   }
 }
@@ -28,15 +27,20 @@ static void mpack_w8(char **b, const char *data)
 #endif
 }
 
+void mpack_uint64(char **ptr, uint64_t i)
+{
+  if (i > 0xfffffff) {
+    mpack_w(ptr, 0xcf);
+    mpack_w8(ptr, (char *)&i);
+  } else {
+    mpack_uint(ptr, (uint32_t)i);
+  }
+}
+
 void mpack_integer(char **ptr, Integer i)
 {
   if (i >= 0) {
-    if (i > 0xfffffff) {
-      mpack_w(ptr, 0xcf);
-      mpack_w8(ptr, (char *)&i);
-    } else {
-      mpack_uint(ptr, (uint32_t)i);
-    }
+    mpack_uint64(ptr, (uint64_t)i);
   } else {
     if (i < -0x80000000LL) {
       mpack_w(ptr, 0xd3);
@@ -80,11 +84,35 @@ void mpack_str(String str, PackerBuffer *packer)
     abort();
   }
 
+  mpack_raw(str.data, len, packer);
+}
+
+void mpack_bin(String str, PackerBuffer *packer)
+{
+  const size_t len = str.size;
+  if (len < 0xff) {
+    mpack_w(&packer->ptr, 0xc4);
+    mpack_w(&packer->ptr, len);
+  } else if (len < 0xffff) {
+    mpack_w(&packer->ptr, 0xc5);
+    mpack_w2(&packer->ptr, (uint32_t)len);
+  } else if (len < 0xffffffff) {
+    mpack_w(&packer->ptr, 0xc6);
+    mpack_w4(&packer->ptr, (uint32_t)len);
+  } else {
+    abort();
+  }
+
+  mpack_raw(str.data, len, packer);
+}
+
+void mpack_raw(const char *data, size_t len, PackerBuffer *packer)
+{
   size_t pos = 0;
   while (pos < len) {
     ptrdiff_t remaining = packer->endptr - packer->ptr;
     size_t to_copy = MIN(len - pos, (size_t)remaining);
-    memcpy(packer->ptr, str.data + pos, to_copy);
+    memcpy(packer->ptr, data + pos, to_copy);
     packer->ptr += to_copy;
     pos += to_copy;
 
@@ -92,6 +120,28 @@ void mpack_str(String str, PackerBuffer *packer)
       packer->packer_flush(packer);
     }
   }
+  mpack_check_buffer(packer);
+}
+
+void mpack_ext(char *buf, size_t len, int8_t type, PackerBuffer *packer)
+{
+  if (len == 1) {
+    mpack_w(&packer->ptr, 0xd4);
+  } else if (len == 2) {
+    mpack_w(&packer->ptr, 0xd5);
+  } else if (len <= 0xff) {
+    mpack_w(&packer->ptr, 0xc7);
+  } else if (len < 0xffff) {
+    mpack_w(&packer->ptr, 0xc8);
+    mpack_w2(&packer->ptr, (uint32_t)len);
+  } else if (len < 0xffffffff) {
+    mpack_w(&packer->ptr, 0xc9);
+    mpack_w4(&packer->ptr, (uint32_t)len);
+  } else {
+    abort();
+  }
+  mpack_w(&packer->ptr, type);
+  mpack_raw(buf, len, packer);
 }
 
 void mpack_handle(ObjectType type, handle_T handle, PackerBuffer *packer)
@@ -113,7 +163,6 @@ void mpack_handle(ObjectType type, handle_T handle, PackerBuffer *packer)
     mpack_w(&packer->ptr, 0xc7);
     mpack_w(&packer->ptr, packsize);
     mpack_w(&packer->ptr, exttype);
-    // check_buffer(packer);
     memcpy(packer->ptr, buf, (size_t)packsize);
     packer->ptr += packsize;
   }
@@ -148,7 +197,7 @@ void mpack_object_inner(Object *current, Object *container, size_t container_idx
   kvi_init(stack);
 
   while (true) {
-    check_buffer(packer);
+    mpack_check_buffer(packer);
     switch (current->type) {
     case kObjectTypeLuaRef:
       // TODO(bfredl): could also be an error. Though kObjectTypeLuaRef
@@ -177,14 +226,14 @@ void mpack_object_inner(Object *current, Object *container, size_t container_idx
     case kObjectTypeTabpage:
       mpack_handle(current->type, (handle_T)current->data.integer, packer);
       break;
-    case kObjectTypeDictionary:
+    case kObjectTypeDict:
     case kObjectTypeArray: {}
       size_t current_size;
       if (current->type == kObjectTypeArray) {
         current_size = current->data.array.size;
         mpack_array(&packer->ptr, (uint32_t)current_size);
       } else {
-        current_size = current->data.dictionary.size;
+        current_size = current->data.dict.size;
         mpack_map(&packer->ptr, (uint32_t)current_size);
       }
       if (current_size > 0) {
@@ -221,9 +270,9 @@ void mpack_object_inner(Object *current, Object *container, size_t container_idx
         container = NULL;
       }
     } else {
-      Dictionary dict = container->data.dictionary;
+      Dict dict = container->data.dict;
       KeyValuePair *it = &dict.items[container_idx++];
-      check_buffer(packer);
+      mpack_check_buffer(packer);
       mpack_str(it->key, packer);
       current = &it->value;
       if (container_idx >= dict.size) {
@@ -233,3 +282,32 @@ void mpack_object_inner(Object *current, Object *container, size_t container_idx
   }
   kvi_destroy(stack);
 }
+
+PackerBuffer packer_string_buffer(void)
+{
+  const size_t initial_size = 64;  // must be larger than SHADA_MPACK_FREE_SPACE
+  char *alloc = xmalloc(initial_size);
+  return (PackerBuffer) {
+    .startptr = alloc,
+    .ptr = alloc,
+    .endptr = alloc + initial_size,
+    .packer_flush = flush_string_buffer,
+  };
+}
+
+static void flush_string_buffer(PackerBuffer *buffer)
+{
+  size_t current_capacity = (size_t)(buffer->endptr - buffer->startptr);
+  size_t new_capacity = 2 * current_capacity;
+  size_t len = (size_t)(buffer->ptr - buffer->startptr);
+
+  buffer->startptr = xrealloc(buffer->startptr, new_capacity);
+  buffer->ptr = buffer->startptr + len;
+  buffer->endptr = buffer->startptr + new_capacity;
+}
+
+/// can only be used with a PackerBuffer from `packer_string_buffer`
+String packer_take_string(PackerBuffer *buffer)
+{
+  return (String){ .data = buffer->startptr, .size = (size_t)(buffer->ptr - buffer->startptr) };
+}
diff --git a/src/nvim/msgpack_rpc/packer.h b/src/nvim/msgpack_rpc/packer.h
index 8117bd09bd..299962bab4 100644
--- a/src/nvim/msgpack_rpc/packer.h
+++ b/src/nvim/msgpack_rpc/packer.h
@@ -71,6 +71,11 @@ static inline void mpack_map(char **buf, uint32_t len)
   }
 }
 
+static inline size_t mpack_remaining(PackerBuffer *packer)
+{
+  return (size_t)(packer->endptr - packer->ptr);
+}
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "msgpack_rpc/packer.h.generated.h"
 #endif
diff --git a/src/nvim/msgpack_rpc/packer_defs.h b/src/nvim/msgpack_rpc/packer_defs.h
index 420f3dc424..95d86caaab 100644
--- a/src/nvim/msgpack_rpc/packer_defs.h
+++ b/src/nvim/msgpack_rpc/packer_defs.h
@@ -19,6 +19,6 @@ struct packer_buffer_t {
 
   // these are free to be used by packer_flush for any purpose, if want
   void *anydata;
-  size_t anylen;
+  int64_t anyint;
   PackerBufferFlush packer_flush;
 };
diff --git a/src/nvim/msgpack_rpc/server.c b/src/nvim/msgpack_rpc/server.c
index 56b03d67d0..462f8397f4 100644
--- a/src/nvim/msgpack_rpc/server.c
+++ b/src/nvim/msgpack_rpc/server.c
@@ -4,18 +4,21 @@
 #include <string.h>
 #include <uv.h>
 
+#include "nvim/ascii_defs.h"
 #include "nvim/channel.h"
 #include "nvim/eval.h"
 #include "nvim/event/defs.h"
 #include "nvim/event/socket.h"
 #include "nvim/garray.h"
 #include "nvim/garray_defs.h"
+#include "nvim/globals.h"
 #include "nvim/log.h"
 #include "nvim/main.h"
 #include "nvim/memory.h"
 #include "nvim/msgpack_rpc/server.h"
 #include "nvim/os/os.h"
 #include "nvim/os/stdpaths_defs.h"
+#include "nvim/types_defs.h"
 
 #define MAX_CONNECTIONS 32
 #define ENV_LISTEN "NVIM_LISTEN_ADDRESS"  // deprecated
@@ -26,37 +29,61 @@ static garray_T watchers = GA_EMPTY_INIT_VALUE;
 # include "msgpack_rpc/server.c.generated.h"
 #endif
 
-/// Initializes the module
+/// Initializes resources, handles `--listen`, starts the primary server at v:servername.
+///
+/// @returns true on success, false on fatal error (message stored in IObuff)
 bool server_init(const char *listen_addr)
 {
+  bool ok = true;
+  bool must_free = false;
+  TriState user_arg = kTrue;  // User-provided --listen arg.
   ga_init(&watchers, sizeof(SocketWatcher *), 1);
 
   // $NVIM_LISTEN_ADDRESS (deprecated)
-  if (!listen_addr && os_env_exists(ENV_LISTEN)) {
+  if ((!listen_addr || listen_addr[0] == '\0') && os_env_exists(ENV_LISTEN)) {
+    user_arg = kFalse;  // User-provided env var.
     listen_addr = os_getenv(ENV_LISTEN);
   }
 
-  int rv = listen_addr ? server_start(listen_addr) : 1;
-  if (0 != rv) {
+  if (!listen_addr || listen_addr[0] == '\0') {
+    user_arg = kNone;  // Autogenerated server address.
     listen_addr = server_address_new(NULL);
-    if (!listen_addr) {
-      return false;
-    }
-    rv = server_start(listen_addr);
-    xfree((char *)listen_addr);
+    must_free = true;
   }
 
-  if (os_env_exists(ENV_LISTEN)) {
-    // Unset $NVIM_LISTEN_ADDRESS, it's a liability hereafter.
-    os_unsetenv(ENV_LISTEN);
-  }
+  int rv = server_start(listen_addr);
 
-  // TODO(justinmk): this is for logging_spec. Can remove this after nvim_log #7062 is merged.
+  // TODO(justinmk): this is for log_spec. Can remove this after nvim_log #7062 is merged.
   if (os_env_exists("__NVIM_TEST_LOG")) {
     ELOG("test log message");
   }
 
-  return rv == 0;
+  if (must_free) {
+    xfree((char *)listen_addr);
+  }
+
+  if (rv == 0 || user_arg == kNone) {
+    // The autogenerated servername can fail if the user has a broken $XDG_RUNTIME_DIR. #30282
+    // But that is not fatal (startup will continue, logged in $NVIM_LOGFILE, empty v:servername).
+    goto end;
+  }
+
+  (void)snprintf(IObuff, IOSIZE,
+                 user_arg ==
+                 kTrue ? "Failed to --listen: %s: \"%s\""
+                       : "Failed $NVIM_LISTEN_ADDRESS: %s: \"%s\"",
+                 rv < 0 ? os_strerror(rv) : (rv == 1 ? "empty address" : "?"),
+                 listen_addr);
+  ok = false;
+
+end:
+  if (os_env_exists(ENV_LISTEN)) {
+    // Unset $NVIM_LISTEN_ADDRESS, it's a liability hereafter. It is "input only", it must not be
+    // leaked to child jobs or :terminal.
+    os_unsetenv(ENV_LISTEN);
+  }
+
+  return ok;
 }
 
 /// Teardown a single server
@@ -87,17 +114,19 @@ void server_teardown(void)
 /// - Windows: "\\.\pipe\<name>.<pid>.<counter>"
 /// - Other: "/tmp/nvim.user/xxx/<name>.<pid>.<counter>"
 char *server_address_new(const char *name)
+  FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_RET
 {
   static uint32_t count = 0;
   char fmt[ADDRESS_MAX_SIZE];
-  const char *appname = get_appname();
 #ifdef MSWIN
+  (void)get_appname(true);
   int r = snprintf(fmt, sizeof(fmt), "\\\\.\\pipe\\%s.%" PRIu64 ".%" PRIu32,
-                   name ? name : appname, os_get_pid(), count++);
+                   name ? name : NameBuff, os_get_pid(), count++);
 #else
   char *dir = stdpaths_get_xdg_var(kXDGRuntimeDir);
+  (void)get_appname(true);
   int r = snprintf(fmt, sizeof(fmt), "%s/%s.%" PRIu64 ".%" PRIu32,
-                   dir, name ? name : appname, os_get_pid(), count++);
+                   dir, name ? name : NameBuff, os_get_pid(), count++);
   xfree(dir);
 #endif
   if ((size_t)r >= sizeof(fmt)) {
@@ -131,7 +160,7 @@ bool server_owns_pipe_address(const char *path)
 /// @returns 0: success, 1: validation error, 2: already listening, -errno: failed to bind/listen.
 int server_start(const char *addr)
 {
-  if (addr == NULL || addr[0] == '\0') {
+  if (addr == NULL || addr[0] == NUL) {
     WLOG("Empty or NULL address");
     return 1;
   }
diff --git a/src/nvim/msgpack_rpc/unpacker.c b/src/nvim/msgpack_rpc/unpacker.c
index 28d27e8268..4ddc41e596 100644
--- a/src/nvim/msgpack_rpc/unpacker.c
+++ b/src/nvim/msgpack_rpc/unpacker.c
@@ -11,6 +11,7 @@
 #include "nvim/memory.h"
 #include "nvim/msgpack_rpc/channel_defs.h"
 #include "nvim/msgpack_rpc/unpacker.h"
+#include "nvim/strings.h"
 #include "nvim/ui_client.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
@@ -58,7 +59,7 @@ static void api_parse_enter(mpack_parser_t *parser, mpack_node_t *node)
     }
     case MPACK_TOKEN_MAP: {
       Object *obj = parent->data[0].p;
-      KeyValuePair *kv = &kv_A(obj->data.dictionary, parent->pos);
+      KeyValuePair *kv = &kv_A(obj->data.dict, parent->pos);
       if (!parent->key_visited) {
         // TODO(bfredl): when implementing interrupt parse on error,
         // stop parsing here when node is not a STR/BIN
@@ -165,10 +166,10 @@ static void api_parse_enter(mpack_parser_t *parser, mpack_node_t *node)
     break;
   }
   case MPACK_TOKEN_MAP: {
-    Dictionary dict = KV_INITIAL_VALUE;
+    Dict dict = KV_INITIAL_VALUE;
     kv_fixsize_arena(&p->arena, dict, node->tok.length);
     kv_size(dict) = node->tok.length;
-    *result = DICTIONARY_OBJ(dict);
+    *result = DICT_OBJ(dict);
     node->data[0].p = result;
     break;
   }
@@ -206,6 +207,7 @@ bool unpacker_parse_header(Unpacker *p)
 
   assert(!ERROR_SET(&p->unpack_error));
 
+  // TODO(bfredl): eliminate p->reader, we can use mpack_rtoken directly
 #define NEXT(tok) \
   result = mpack_read(&p->reader, &data, &size, &tok); \
   if (result) { goto error; }
@@ -522,3 +524,197 @@ bool unpacker_parse_redraw(Unpacker *p)
     abort();
   }
 }
+
+/// Requires a complete string. safe to use e.g. in shada as we have loaded a
+/// complete shada item into a linear buffer.
+///
+/// Data and size are preserved in cause of failure.
+///
+/// @return "data" is NULL only when failure (non-null data and size=0 for
+/// valid empty string)
+String unpack_string(const char **data, size_t *size)
+{
+  const char *data2 = *data;
+  size_t size2 = *size;
+  mpack_token_t tok;
+
+  // TODO(bfredl): this code is hot a f, specialize!
+  int result = mpack_rtoken(&data2, &size2, &tok);
+  if (result || (tok.type != MPACK_TOKEN_STR && tok.type != MPACK_TOKEN_BIN)) {
+    return (String)STRING_INIT;
+  }
+  if (*size < tok.length) {
+    // result = MPACK_EOF;
+    return (String)STRING_INIT;
+  }
+  (*data) = data2 + tok.length;
+  (*size) = size2 - tok.length;
+  return cbuf_as_string((char *)data2, tok.length);
+}
+
+/// @return -1 if not an array or EOF. otherwise size of valid array
+ssize_t unpack_array(const char **data, size_t *size)
+{
+  // TODO(bfredl): this code is hot, specialize!
+  mpack_token_t tok;
+  int result = mpack_rtoken(data, size, &tok);
+  if (result || tok.type != MPACK_TOKEN_ARRAY) {
+    return -1;
+  }
+  return tok.length;
+}
+
+/// does not keep "data" untouched on failure
+bool unpack_integer(const char **data, size_t *size, Integer *res)
+{
+  mpack_token_t tok;
+  int result = mpack_rtoken(data, size, &tok);
+  if (result) {
+    return false;
+  }
+  return unpack_uint_or_sint(tok, res);
+}
+
+bool unpack_uint_or_sint(mpack_token_t tok, Integer *res)
+{
+  if (tok.type == MPACK_TOKEN_UINT) {
+    *res = (Integer)mpack_unpack_uint(tok);
+    return true;
+  } else if (tok.type == MPACK_TOKEN_SINT) {
+    *res = (Integer)mpack_unpack_sint(tok);
+    return true;
+  }
+  return false;
+}
+
+static void parse_nop(mpack_parser_t *parser, mpack_node_t *node)
+{
+}
+
+int unpack_skip(const char **data, size_t *size)
+{
+  mpack_parser_t parser;
+  mpack_parser_init(&parser, 0);
+
+  return mpack_parse(&parser, data, size, parse_nop, parse_nop);
+}
+
+void push_additional_data(AdditionalDataBuilder *ad, const char *data, size_t size)
+{
+  if (kv_size(*ad) == 0) {
+    AdditionalData init = { 0 };
+    kv_concat_len(*ad, &init, sizeof(init));
+  }
+  AdditionalData *a = (AdditionalData *)ad->items;
+  a->nitems++;
+  a->nbytes += (uint32_t)size;
+  kv_concat_len(*ad, data, size);
+}
+
+// currently only used for shada, so not re-entrant like unpacker_parse_redraw
+bool unpack_keydict(void *retval, FieldHashfn hashy, AdditionalDataBuilder *ad, const char **data,
+                    size_t *restrict size, char **error)
+{
+  OptKeySet *ks = (OptKeySet *)retval;
+  mpack_token_t tok;
+
+  int result = mpack_rtoken(data, size, &tok);
+  if (result || tok.type != MPACK_TOKEN_MAP) {
+    *error = xstrdup("is not a dict");
+    return false;
+  }
+
+  size_t map_size = tok.length;
+
+  for (size_t i = 0; i < map_size; i++) {
+    const char *item_start = *data;
+    // TODO(bfredl): we could specialize a hot path for FIXSTR here
+    String key = unpack_string(data, size);
+    if (!key.data) {
+      *error = arena_printf(NULL, "has key value which is not a string").data;
+      return false;
+    } else if (key.size == 0) {
+      *error = arena_printf(NULL, "has empty key").data;
+      return false;
+    }
+    KeySetLink *field = hashy(key.data, key.size);
+
+    if (!field) {
+      int status = unpack_skip(data, size);
+      if (status) {
+        return false;
+      }
+
+      if (ad) {
+        push_additional_data(ad, item_start, (size_t)(*data - item_start));
+      }
+      continue;
+    }
+
+    assert(field->opt_index >= 0);
+    uint64_t flag = (1ULL << field->opt_index);
+    if (ks->is_set_ & flag) {
+      *error = xstrdup("duplicate key");
+      return false;
+    }
+    ks->is_set_ |= flag;
+
+    char *mem = ((char *)retval + field->ptr_off);
+    switch (field->type) {
+    case kObjectTypeBoolean:
+      if (*size == 0 || (**data & 0xfe) != 0xc2) {
+        *error = arena_printf(NULL, "has %.*s key value which is not a boolean", (int)key.size,
+                              key.data).data;
+        return false;
+      }
+      *(Boolean *)mem = **data & 0x01;
+      (*data)++; (*size)--;
+      break;
+
+    case kObjectTypeInteger:
+      if (!unpack_integer(data, size, (Integer *)mem)) {
+        *error = arena_printf(NULL, "has %.*s key value which is not an integer", (int)key.size,
+                              key.data).data;
+        return false;
+      }
+      break;
+
+    case kObjectTypeString: {
+      String val = unpack_string(data, size);
+      if (!val.data) {
+        *error = arena_printf(NULL, "has %.*s key value which is not a binary", (int)key.size,
+                              key.data).data;
+        return false;
+      }
+      *(String *)mem = val;
+      break;
+    }
+
+    case kUnpackTypeStringArray: {
+      ssize_t len = unpack_array(data, size);
+      if (len < 0) {
+        *error = arena_printf(NULL, "has %.*s key with non-array value", (int)key.size,
+                              key.data).data;
+        return false;
+      }
+      StringArray *a = (StringArray *)mem;
+      kv_ensure_space(*a, (size_t)len);
+      for (size_t j = 0; j < (size_t)len; j++) {
+        String item = unpack_string(data, size);
+        if (!item.data) {
+          *error = arena_printf(NULL, "has %.*s array with non-binary value", (int)key.size,
+                                key.data).data;
+          return false;
+        }
+        kv_push(*a, item);
+      }
+      break;
+    }
+
+    default:
+      abort();  // not supported
+    }
+  }
+
+  return true;
+}
diff --git a/src/nvim/msgpack_rpc/unpacker.h b/src/nvim/msgpack_rpc/unpacker.h
index ed55fdd4af..c29462292f 100644
--- a/src/nvim/msgpack_rpc/unpacker.h
+++ b/src/nvim/msgpack_rpc/unpacker.h
@@ -41,6 +41,8 @@ struct Unpacker {
   bool has_grid_line_event;
 };
 
+typedef kvec_t(char) AdditionalDataBuilder;
+
 // unrecovareble error. unpack_error should be set!
 #define unpacker_closed(p) ((p)->state < 0)