11 files changed, 470 insertions, 743 deletions

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 0fb1ebf931..a8fde5a652 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -1,7 +1,6 @@
 #include <assert.h>
 #include <inttypes.h>
 #include <msgpack/object.h>
-#include <msgpack/pack.h>
 #include <msgpack/sbuffer.h>
 #include <msgpack/unpack.h>
 #include <stdbool.h>
@@ -14,13 +13,13 @@
 #include "nvim/api/private/helpers.h"
 #include "nvim/api/ui.h"
 #include "nvim/channel.h"
+#include "nvim/channel_defs.h"
 #include "nvim/event/defs.h"
 #include "nvim/event/loop.h"
+#include "nvim/event/multiqueue.h"
 #include "nvim/event/process.h"
 #include "nvim/event/rstream.h"
-#include "nvim/event/stream.h"
 #include "nvim/event/wstream.h"
-#include "nvim/func_attr.h"
 #include "nvim/globals.h"
 #include "nvim/log.h"
 #include "nvim/main.h"
@@ -29,10 +28,11 @@
 #include "nvim/message.h"
 #include "nvim/msgpack_rpc/channel.h"
 #include "nvim/msgpack_rpc/channel_defs.h"
-#include "nvim/msgpack_rpc/helpers.h"
+#include "nvim/msgpack_rpc/packer.h"
 #include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/os/input.h"
 #include "nvim/rbuffer.h"
+#include "nvim/rbuffer_defs.h"
 #include "nvim/types_defs.h"
 #include "nvim/ui.h"
 #include "nvim/ui_client.h"
@@ -43,73 +43,31 @@
 # define NOT "[notify]   "
 # define ERR "[error]    "
 
-// Cannot define array with negative offsets, so this one is needed to be added
-// to MSGPACK_UNPACK_\* values.
-# define MUR_OFF 2
+# define SEND "->"
+# define RECV "<-"
 
-static const char *const msgpack_error_messages[] = {
-  [MSGPACK_UNPACK_EXTRA_BYTES + MUR_OFF] = "extra bytes found",
-  [MSGPACK_UNPACK_CONTINUE + MUR_OFF] = "incomplete string",
-  [MSGPACK_UNPACK_PARSE_ERROR + MUR_OFF] = "parse error",
-  [MSGPACK_UNPACK_NOMEM_ERROR + MUR_OFF] = "not enough memory",
-};
-
-static void log_close(FILE *f)
+static void log_request(char *dir, uint64_t channel_id, uint32_t req_id, const char *name)
 {
-  fputc('\n', f);
-  fflush(f);
-  fclose(f);
-  log_unlock();
+  DLOGN("RPC %s %" PRIu64 ": %s id=%u: %s\n", dir, channel_id, REQ, req_id, name);
 }
 
-static void log_server_msg(uint64_t channel_id, msgpack_sbuffer *packed)
+static void log_response(char *dir, uint64_t channel_id, char *kind, uint32_t req_id)
 {
-  msgpack_unpacked unpacked;
-  msgpack_unpacked_init(&unpacked);
-  DLOGN("RPC ->ch %" PRIu64 ": ", channel_id);
-  const msgpack_unpack_return result =
-    msgpack_unpack_next(&unpacked, packed->data, packed->size, NULL);
-  switch (result) {
-  case MSGPACK_UNPACK_SUCCESS: {
-    uint64_t type = unpacked.data.via.array.ptr[0].via.u64;
-    log_lock();
-    FILE *f = open_log_file();
-    fprintf(f, type ? (type == 1 ? RES : NOT) : REQ);
-    msgpack_object_print(f, unpacked.data);
-    log_close(f);
-    msgpack_unpacked_destroy(&unpacked);
-    break;
-  }
-  case MSGPACK_UNPACK_EXTRA_BYTES:
-  case MSGPACK_UNPACK_CONTINUE:
-  case MSGPACK_UNPACK_PARSE_ERROR:
-  case MSGPACK_UNPACK_NOMEM_ERROR: {
-    log_lock();
-    FILE *f = open_log_file();
-    fprintf(f, ERR);
-    fprintf(f, "%s", msgpack_error_messages[result + MUR_OFF]);
-    log_close(f);
-    break;
-  }
-  }
+  DLOGN("RPC %s %" PRIu64 ": %s id=%u\n", dir, channel_id, kind, req_id);
 }
 
-static void log_client_msg(uint64_t channel_id, bool is_request, const char *name)
+static void log_notify(char *dir, uint64_t channel_id, const char *name)
 {
-  DLOGN("RPC <-ch %" PRIu64 ": ", channel_id);
-  log_lock();
-  FILE *f = open_log_file();
-  fprintf(f, "%s: %s", is_request ? REQ : RES, name);
-  log_close(f);
+  DLOGN("RPC %s %" PRIu64 ": %s %s\n", dir, channel_id, NOT, name);
 }
 
 #else
-# define log_client_msg(...)
-# define log_server_msg(...)
+# define log_request(...)
+# define log_response(...)
+# define log_notify(...)
 #endif
 
 static Set(cstr_t) event_strings = SET_INIT;
-static msgpack_sbuffer out_buffer;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "msgpack_rpc/channel.c.generated.h"
@@ -118,7 +76,6 @@ static msgpack_sbuffer out_buffer;
 void rpc_init(void)
 {
   ch_before_blocking_events = multiqueue_new_child(main_loop.events);
-  msgpack_sbuffer_init(&out_buffer);
 }
 
 void rpc_start(Channel *channel)
@@ -168,8 +125,9 @@ bool rpc_send_event(uint64_t id, const char *name, Array args)
     return false;
   }
 
+  log_notify(SEND, channel ? channel->id : 0, name);
   if (channel) {
-    send_event(channel, name, args);
+    serialize_request(&channel, 1, 0, name, args);
   } else {
     broadcast_event(name, args);
   }
@@ -191,7 +149,6 @@ Object rpc_send_call(uint64_t id, const char *method_name, Array args, ArenaMem
 
   if (!(channel = find_rpc_channel(id))) {
     api_set_error(err, kErrorTypeException, "Invalid channel: %" PRIu64, id);
-    api_free_array(args);
     return NIL;
   }
 
@@ -199,8 +156,9 @@ Object rpc_send_call(uint64_t id, const char *method_name, Array args, ArenaMem
   RpcState *rpc = &channel->rpc;
   uint32_t request_id = rpc->next_request_id++;
   // Send the msgpack-rpc request
-  send_request(channel, request_id, method_name, args);
-  api_free_array(args);
+  serialize_request(&channel, 1, request_id, method_name, args);
+
+  log_request(SEND, channel->id, request_id, method_name);
 
   // Push the frame
   ChannelCallFrame frame = { request_id, false, false, NIL, NULL };
@@ -303,8 +261,11 @@ static void receive_msgpack(Stream *stream, RBuffer *rbuf, size_t c, void *data,
   p->read_ptr = rbuffer_read_ptr(rbuf, &size);
   p->read_size = size;
   parse_msgpack(channel);
-  size_t consumed = size - p->read_size;
-  rbuffer_consumed_compact(rbuf, consumed);
+
+  if (!unpacker_closed(p)) {
+    size_t consumed = size - p->read_size;
+    rbuffer_consumed_compact(rbuf, consumed);
+  }
 
 end:
   channel_decref(channel);
@@ -359,8 +320,13 @@ static void parse_msgpack(Channel *channel)
         frame->result = p->result;
       }
       frame->result_mem = arena_finish(&p->arena);
+      log_response(RECV, channel->id, frame->errored ? ERR : RES, p->request_id);
     } else {
-      log_client_msg(channel->id, p->type == kMessageTypeRequest, p->handler.name);
+      if (p->type == kMessageTypeNotification) {
+        log_notify(RECV, channel->id, p->handler.name);
+      } else {
+        log_request(RECV, channel->id, p->request_id, p->handler.name);
+      }
 
       Object res = p->result;
       if (p->result.type != kObjectTypeArray) {
@@ -405,7 +371,7 @@ static void handle_request(Channel *channel, Unpacker *p, Array args)
 
     if (is_get_mode && !input_blocking()) {
       // Defer the event to a special queue used by os/input.c. #6247
-      multiqueue_put(ch_before_blocking_events, request_event, 1, evdata);
+      multiqueue_put(ch_before_blocking_events, request_event, evdata);
     } else {
       // Invoke immediately.
       request_event((void **)&evdata);
@@ -413,12 +379,11 @@ static void handle_request(Channel *channel, Unpacker *p, Array args)
   } else {
     bool is_resize = p->handler.fn == handle_nvim_ui_try_resize;
     if (is_resize) {
-      Event ev = event_create_oneshot(event_create(request_event, 1, evdata),
-                                      2);
+      Event ev = event_create_oneshot(event_create(request_event, evdata), 2);
       multiqueue_put_event(channel->events, ev);
       multiqueue_put_event(resize_events, ev);
     } else {
-      multiqueue_put(channel->events, request_event, 1, evdata);
+      multiqueue_put(channel->events, request_event, evdata);
       DLOG("RPC: scheduled %.*s", (int)p->method_name_len, p->handler.name);
     }
   }
@@ -441,17 +406,9 @@ static void request_event(void **argv)
   Object result = handler.fn(channel->id, e->args, &e->used_mem, &error);
   if (e->type == kMessageTypeRequest || ERROR_SET(&error)) {
     // Send the response.
-    msgpack_packer response;
-    msgpack_packer_init(&response, &out_buffer, msgpack_sbuffer_write);
-    channel_write(channel, serialize_response(channel->id,
-                                              e->handler,
-                                              e->type,
-                                              e->request_id,
-                                              &error,
-                                              result,
-                                              &out_buffer));
-  }
-  if (!handler.arena_return) {
+    serialize_response(channel, e->handler, e->type, e->request_id, &error, &result);
+  }
+  if (handler.ret_alloc) {
     api_free_object(result);
   }
 
@@ -485,7 +442,7 @@ static bool channel_write(Channel *channel, WBuffer *buffer)
 
   if (channel->streamtype == kChannelStreamInternal) {
     channel_incref(channel);
-    CREATE_EVENT(channel->events, internal_read_event, 2, channel, buffer);
+    CREATE_EVENT(channel->events, internal_read_event, channel, buffer);
     success = true;
   } else {
     Stream *in = channel_instream(channel);
@@ -532,41 +489,14 @@ static void send_error(Channel *chan, MsgpackRpcRequestHandler handler, MessageT
 {
   Error e = ERROR_INIT;
   api_set_error(&e, kErrorTypeException, "%s", err);
-  channel_write(chan, serialize_response(chan->id,
-                                         handler,
-                                         type,
-                                         id,
-                                         &e,
-                                         NIL,
-                                         &out_buffer));
+  serialize_response(chan, handler, type, id, &e, &NIL);
   api_clear_error(&e);
 }
 
-static void send_request(Channel *channel, uint32_t id, const char *name, Array args)
-{
-  const String method = cstr_as_string((char *)name);
-  channel_write(channel, serialize_request(channel->id,
-                                           id,
-                                           method,
-                                           args,
-                                           &out_buffer,
-                                           1));
-}
-
-static void send_event(Channel *channel, const char *name, Array args)
-{
-  const String method = cstr_as_string((char *)name);
-  channel_write(channel, serialize_request(channel->id,
-                                           0,
-                                           method,
-                                           args,
-                                           &out_buffer,
-                                           1));
-}
-
 static void broadcast_event(const char *name, Array args)
 {
-  kvec_t(Channel *) subscribed = KV_INITIAL_VALUE;
+  kvec_withinit_t(Channel *, 4) subscribed = KV_INITIAL_VALUE;
+  kvi_init(subscribed);
   Channel *channel;
 
   map_foreach_value(&channels, channel, {
@@ -576,25 +506,11 @@ static void broadcast_event(const char *name, Array args)
     }
   });
 
-  if (!kv_size(subscribed)) {
-    goto end;
-  }
-
-  const String method = cstr_as_string((char *)name);
-  WBuffer *buffer = serialize_request(0,
-                                      0,
-                                      method,
-                                      args,
-                                      &out_buffer,
-                                      kv_size(subscribed));
-
-  for (size_t i = 0; i < kv_size(subscribed); i++) {
-    Channel *c = kv_A(subscribed, i);
-    channel_write(c, buffer);
+  if (kv_size(subscribed)) {
+    serialize_request(subscribed.items, kv_size(subscribed), 0, name, args);
   }
 
-end:
-  kv_destroy(subscribed);
+  kvi_destroy(subscribed);
 }
 
 static void unsubscribe(Channel *channel, char *event)
@@ -652,27 +568,28 @@ static void chan_close_with_error(Channel *channel, char *msg, int loglevel)
   channel_close(channel->id, kChannelPartRpc, NULL);
 }
 
-static WBuffer *serialize_request(uint64_t channel_id, uint32_t request_id, const String method,
-                                  Array args, msgpack_sbuffer *sbuffer, size_t refcount)
+static void serialize_request(Channel **chans, size_t nchans, uint32_t request_id,
+                              const char *method, Array args)
 {
-  msgpack_packer pac;
-  msgpack_packer_init(&pac, sbuffer, msgpack_sbuffer_write);
-  msgpack_rpc_serialize_request(request_id, method, args, &pac);
-  log_server_msg(channel_id, sbuffer);
-  WBuffer *rv = wstream_new_buffer(xmemdup(sbuffer->data, sbuffer->size),
-                                   sbuffer->size,
-                                   refcount,
-                                   xfree);
-  msgpack_sbuffer_clear(sbuffer);
-  return rv;
+  PackerBuffer packer;
+  packer_buffer_init_channels(chans, nchans, &packer);
+
+  mpack_array(&packer.ptr, request_id ? 4 : 3);
+  mpack_w(&packer.ptr, request_id ? 0 : 2);
+
+  if (request_id) {
+    mpack_uint(&packer.ptr, request_id);
+  }
+
+  mpack_str(cstr_as_string(method), &packer);
+  mpack_object_array(args, &packer);
+
+  packer_buffer_finish_channels(&packer);
 }
 
-static WBuffer *serialize_response(uint64_t channel_id, MsgpackRpcRequestHandler handler,
-                                   MessageType type, uint32_t response_id, Error *err, Object arg,
-                                   msgpack_sbuffer *sbuffer)
+void serialize_response(Channel *channel, MsgpackRpcRequestHandler handler, MessageType type,
+                        uint32_t response_id, Error *err, Object *arg)
 {
-  msgpack_packer pac;
-  msgpack_packer_init(&pac, sbuffer, msgpack_sbuffer_write);
   if (ERROR_SET(err) && type == kMessageTypeNotification) {
     if (handler.fn == handle_nvim_paste) {
       // TODO(bfredl): this is pretty much ad-hoc. maybe TUI and UI:s should be
@@ -681,23 +598,68 @@ static WBuffer *serialize_response(uint64_t channel_id, MsgpackRpcRequestHandler
       semsg("paste: %s", err->msg);
       api_clear_error(err);
     } else {
-      Array args = ARRAY_DICT_INIT;
-      ADD(args, INTEGER_OBJ(err->type));
-      ADD(args, CSTR_TO_OBJ(err->msg));
-      msgpack_rpc_serialize_request(0, cstr_as_string("nvim_error_event"),
-                                    args, &pac);
-      api_free_array(args);
+      MAXSIZE_TEMP_ARRAY(args, 2);
+      ADD_C(args, INTEGER_OBJ(err->type));
+      ADD_C(args, CSTR_AS_OBJ(err->msg));
+      serialize_request(&channel, 1, 0, "nvim_error_event", args);
     }
+    return;
+  }
+
+  PackerBuffer packer;
+  packer_buffer_init_channels(&channel, 1, &packer);
+
+  mpack_array(&packer.ptr, 4);
+  mpack_w(&packer.ptr, 1);
+  mpack_uint(&packer.ptr, response_id);
+
+  if (ERROR_SET(err)) {
+    // error represented by a [type, message] array
+    mpack_array(&packer.ptr, 2);
+    mpack_integer(&packer.ptr, err->type);
+    mpack_str(cstr_as_string(err->msg), &packer);
+    // Nil result
+    mpack_nil(&packer.ptr);
   } else {
-    msgpack_rpc_serialize_response(response_id, err, arg, &pac);
+    // Nil error
+    mpack_nil(&packer.ptr);
+    // Return value
+    mpack_object(arg, &packer);
   }
-  log_server_msg(channel_id, sbuffer);
-  WBuffer *rv = wstream_new_buffer(xmemdup(sbuffer->data, sbuffer->size),
-                                   sbuffer->size,
-                                   1,  // responses only go though 1 channel
-                                   xfree);
-  msgpack_sbuffer_clear(sbuffer);
-  return rv;
+
+  packer_buffer_finish_channels(&packer);
+
+  log_response(SEND, channel->id, ERROR_SET(err) ? ERR : RES, response_id);
+}
+
+static void packer_buffer_init_channels(Channel **chans, size_t nchans, PackerBuffer *packer)
+{
+  packer->startptr = alloc_block();
+  packer->ptr = packer->startptr;
+  packer->endptr = packer->startptr + ARENA_BLOCK_SIZE;
+  packer->packer_flush = channel_flush_callback;
+  packer->anydata = chans;
+  packer->anylen = nchans;
+}
+
+static void packer_buffer_finish_channels(PackerBuffer *packer)
+{
+  size_t len = (size_t)(packer->ptr - packer->startptr);
+  if (len > 0) {
+    WBuffer *buf = wstream_new_buffer(packer->startptr, len, packer->anylen, free_block);
+    Channel **chans = packer->anydata;
+    for (size_t i = 0; i < packer->anylen; i++) {
+      channel_write(chans[i], buf);
+    }
+  } else {
+    free_block(packer->startptr);
+  }
+}
+
+static void channel_flush_callback(PackerBuffer *packer)
+{
+  packer_buffer_finish_channels(packer);
+  packer_buffer_init_channels(packer->anydata, packer->anylen, packer);
 }
 
 void rpc_set_client_info(uint64_t id, Dictionary info)
@@ -753,6 +715,7 @@ const char *get_client_info(Channel *chan, const char *key)
   return NULL;
 }
 
+#ifdef EXITFREE
 void rpc_free_all_mem(void)
 {
   cstr_t key;
@@ -760,4 +723,7 @@ void rpc_free_all_mem(void)
     xfree((void *)key);
   });
   set_destroy(cstr_t, &event_strings);
+
+  multiqueue_free(ch_before_blocking_events);
 }
+#endif
diff --git a/src/nvim/msgpack_rpc/channel.h b/src/nvim/msgpack_rpc/channel.h
index 818bee8318..ff73a2fc53 100644
--- a/src/nvim/msgpack_rpc/channel.h
+++ b/src/nvim/msgpack_rpc/channel.h
@@ -3,14 +3,10 @@
 #include <stdint.h>  // IWYU pragma: keep
 
 #include "nvim/api/private/defs.h"  // IWYU pragma: keep
-#include "nvim/channel.h"
-#include "nvim/event/multiqueue.h"
-#include "nvim/event/process.h"
-#include "nvim/event/socket.h"
-#include "nvim/event/wstream.h"
+#include "nvim/event/defs.h"
 #include "nvim/macros_defs.h"
 #include "nvim/memory_defs.h"  // IWYU pragma: keep
-#include "nvim/msgpack_rpc/channel_defs.h"  // IWYU pragma: export
+#include "nvim/msgpack_rpc/channel_defs.h"  // IWYU pragma: keep
 
 #define METHOD_MAXLEN 512
 
diff --git a/src/nvim/msgpack_rpc/channel_defs.h b/src/nvim/msgpack_rpc/channel_defs.h
index 20b8a89afb..56dbb332e0 100644
--- a/src/nvim/msgpack_rpc/channel_defs.h
+++ b/src/nvim/msgpack_rpc/channel_defs.h
@@ -4,11 +4,7 @@
 #include <stdbool.h>
 #include <uv.h>
 
-#include "klib/kvec.h"
-#include "nvim/api/private/defs.h"
 #include "nvim/api/private/dispatch.h"
-#include "nvim/event/process.h"
-#include "nvim/event/socket.h"
 #include "nvim/map_defs.h"
 
 typedef struct Channel Channel;
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
deleted file mode 100644
index 1fdfc9e536..0000000000
--- a/src/nvim/msgpack_rpc/helpers.c
+++ /dev/null
@@ -1,547 +0,0 @@
-#include <msgpack/object.h>
-#include <msgpack/sbuffer.h>
-#include <msgpack/unpack.h>
-#include <msgpack/zone.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-
-#include "klib/kvec.h"
-#include "msgpack/pack.h"
-#include "nvim/api/private/helpers.h"
-#include "nvim/assert_defs.h"
-#include "nvim/func_attr.h"
-#include "nvim/memory.h"
-#include "nvim/msgpack_rpc/helpers.h"
-#include "nvim/types_defs.h"
-
-#ifdef INCLUDE_GENERATED_DECLARATIONS
-# include "msgpack_rpc/helpers.c.generated.h"
-#endif
-
-static msgpack_zone zone;
-static msgpack_sbuffer sbuffer;
-
-void msgpack_rpc_helpers_init(void)
-{
-  msgpack_zone_init(&zone, 0xfff);
-  msgpack_sbuffer_init(&sbuffer);
-}
-
-typedef struct {
-  const msgpack_object *mobj;
-  Object *aobj;
-  bool container;
-  size_t idx;
-} MPToAPIObjectStackItem;
-
-// uncrustify:off
-
-/// Convert type used by msgpack parser to Nvim API type.
-///
-/// @param[in]  obj  Msgpack value to convert.
-/// @param[out]  arg  Location where result of conversion will be saved.
-///
-/// @return true in case of success, false otherwise.
-bool msgpack_rpc_to_object(const msgpack_object *const obj, Object *const arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  bool ret = true;
-  kvec_withinit_t(MPToAPIObjectStackItem, 2) stack = KV_INITIAL_VALUE;
-  kvi_init(stack);
-  kvi_push(stack, ((MPToAPIObjectStackItem) {
-    .mobj = obj,
-    .aobj = arg,
-    .container = false,
-    .idx = 0,
-  }));
-  while (ret && kv_size(stack)) {
-    MPToAPIObjectStackItem cur = kv_last(stack);
-    if (!cur.container) {
-      *cur.aobj = NIL;
-    }
-    switch (cur.mobj->type) {
-    case MSGPACK_OBJECT_NIL:
-      break;
-    case MSGPACK_OBJECT_BOOLEAN:
-      *cur.aobj = BOOLEAN_OBJ(cur.mobj->via.boolean);
-      break;
-    case MSGPACK_OBJECT_NEGATIVE_INTEGER:
-      STATIC_ASSERT(sizeof(Integer) == sizeof(cur.mobj->via.i64),
-                    "Msgpack integer size does not match API integer");
-      *cur.aobj = INTEGER_OBJ(cur.mobj->via.i64);
-      break;
-    case MSGPACK_OBJECT_POSITIVE_INTEGER:
-      STATIC_ASSERT(sizeof(Integer) == sizeof(cur.mobj->via.u64),
-                    "Msgpack integer size does not match API integer");
-      if (cur.mobj->via.u64 > API_INTEGER_MAX) {
-        ret = false;
-      } else {
-        *cur.aobj = INTEGER_OBJ((Integer)cur.mobj->via.u64);
-      }
-      break;
-    case MSGPACK_OBJECT_FLOAT32:
-    case MSGPACK_OBJECT_FLOAT64:
-    {
-      STATIC_ASSERT(sizeof(Float) == sizeof(cur.mobj->via.f64),
-                    "Msgpack floating-point size does not match API integer");
-      *cur.aobj = FLOAT_OBJ(cur.mobj->via.f64);
-      break;
-    }
-#define STR_CASE(type, attr, obj, dest, conv) \
-  case type: { \
-      dest = conv(((String) { \
-      .size = obj->via.attr.size, \
-      .data = (obj->via.attr.ptr == NULL || obj->via.attr.size == 0 \
-               ? xmemdupz("", 0) \
-               : xmemdupz(obj->via.attr.ptr, obj->via.attr.size)), })); \
-      break; \
-  }
-      STR_CASE(MSGPACK_OBJECT_STR, str, cur.mobj, *cur.aobj, STRING_OBJ)
-      STR_CASE(MSGPACK_OBJECT_BIN, bin, cur.mobj, *cur.aobj, STRING_OBJ)
-    case MSGPACK_OBJECT_ARRAY: {
-      const size_t size = cur.mobj->via.array.size;
-      if (cur.container) {
-        if (cur.idx >= size) {
-          (void)kv_pop(stack);
-        } else {
-          const size_t idx = cur.idx;
-          cur.idx++;
-          kv_last(stack) = cur;
-          kvi_push(stack, ((MPToAPIObjectStackItem) {
-              .mobj = &cur.mobj->via.array.ptr[idx],
-              .aobj = &cur.aobj->data.array.items[idx],
-              .container = false,
-          }));
-        }
-      } else {
-        *cur.aobj = ARRAY_OBJ(((Array) {
-            .size = size,
-            .capacity = size,
-            .items = (size > 0
-                      ? xcalloc(size, sizeof(*cur.aobj->data.array.items))
-                      : NULL),
-        }));
-        cur.container = true;
-        kv_last(stack) = cur;
-      }
-      break;
-    }
-    case MSGPACK_OBJECT_MAP: {
-      const size_t size = cur.mobj->via.map.size;
-      if (cur.container) {
-        if (cur.idx >= size) {
-          (void)kv_pop(stack);
-        } else {
-          const size_t idx = cur.idx;
-          cur.idx++;
-          kv_last(stack) = cur;
-          const msgpack_object *const key = &cur.mobj->via.map.ptr[idx].key;
-          switch (key->type) {
-#define ID(x) x
-          STR_CASE(MSGPACK_OBJECT_STR, str, key,
-                   cur.aobj->data.dictionary.items[idx].key, ID)
-          STR_CASE(MSGPACK_OBJECT_BIN, bin, key,
-                   cur.aobj->data.dictionary.items[idx].key, ID)
-#undef ID
-          case MSGPACK_OBJECT_NIL:
-          case MSGPACK_OBJECT_BOOLEAN:
-          case MSGPACK_OBJECT_POSITIVE_INTEGER:
-          case MSGPACK_OBJECT_NEGATIVE_INTEGER:
-          case MSGPACK_OBJECT_FLOAT32:
-          case MSGPACK_OBJECT_FLOAT64:
-          case MSGPACK_OBJECT_EXT:
-          case MSGPACK_OBJECT_MAP:
-          case MSGPACK_OBJECT_ARRAY:
-            ret = false;
-            break;
-          }
-          if (ret) {
-            kvi_push(stack, ((MPToAPIObjectStackItem) {
-                .mobj = &cur.mobj->via.map.ptr[idx].val,
-                .aobj = &cur.aobj->data.dictionary.items[idx].value,
-                .container = false,
-            }));
-          }
-        }
-      } else {
-        *cur.aobj = DICTIONARY_OBJ(((Dictionary) {
-            .size = size,
-            .capacity = size,
-            .items = (size > 0
-                      ? xcalloc(size, sizeof(*cur.aobj->data.dictionary.items))
-                      : NULL),
-        }));
-        cur.container = true;
-        kv_last(stack) = cur;
-      }
-      break;
-    }
-    case MSGPACK_OBJECT_EXT:
-      if (0 <= cur.mobj->via.ext.type && cur.mobj->via.ext.type <= EXT_OBJECT_TYPE_MAX) {
-        cur.aobj->type = (ObjectType)(cur.mobj->via.ext.type + EXT_OBJECT_TYPE_SHIFT);
-        msgpack_object data;
-        msgpack_unpack_return status = msgpack_unpack(cur.mobj->via.ext.ptr, cur.mobj->via.ext.size,
-                                                      NULL, &zone, &data);
-
-        if (status != MSGPACK_UNPACK_SUCCESS || data.type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-          ret = false;
-          break;
-        }
-        cur.aobj->data.integer = (handle_T)data.via.i64;
-        ret = true;
-      }
-      break;
-#undef STR_CASE
-    }
-    if (!cur.container) {
-      (void)kv_pop(stack);
-    }
-  }
-  kvi_destroy(stack);
-  return ret;
-}
-
-static bool msgpack_rpc_to_string(const msgpack_object *const obj, String *const arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  if (obj->type == MSGPACK_OBJECT_BIN || obj->type == MSGPACK_OBJECT_STR) {
-    arg->data = obj->via.bin.ptr != NULL
-                    ? xmemdupz(obj->via.bin.ptr, obj->via.bin.size)
-                    : NULL;
-    arg->size = obj->via.bin.size;
-    return true;
-  }
-  return false;
-}
-
-bool msgpack_rpc_to_array(const msgpack_object *const obj, Array *const arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  if (obj->type != MSGPACK_OBJECT_ARRAY) {
-    return false;
-  }
-
-  arg->size = obj->via.array.size;
-  arg->items = xcalloc(obj->via.array.size, sizeof(Object));
-
-  for (uint32_t i = 0; i < obj->via.array.size; i++) {
-    if (!msgpack_rpc_to_object(obj->via.array.ptr + i, &arg->items[i])) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-bool msgpack_rpc_to_dictionary(const msgpack_object *const obj, Dictionary *const arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  if (obj->type != MSGPACK_OBJECT_MAP) {
-    return false;
-  }
-
-  arg->size = obj->via.array.size;
-  arg->items = xcalloc(obj->via.map.size, sizeof(KeyValuePair));
-
-  for (uint32_t i = 0; i < obj->via.map.size; i++) {
-    if (!msgpack_rpc_to_string(&obj->via.map.ptr[i].key,
-                               &arg->items[i].key)) {
-      return false;
-    }
-
-    if (!msgpack_rpc_to_object(&obj->via.map.ptr[i].val,
-                               &arg->items[i].value)) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-void msgpack_rpc_from_boolean(Boolean result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  if (result) {
-    msgpack_pack_true(res);
-  } else {
-    msgpack_pack_false(res);
-  }
-}
-
-void msgpack_rpc_from_integer(Integer result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_int64(res, result);
-}
-
-void msgpack_rpc_from_float(Float result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_double(res, result);
-}
-
-void msgpack_rpc_from_string(const String result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_str(res, result.size);
-  if (result.size > 0) {
-    msgpack_pack_str_body(res, result.data, result.size);
-  }
-}
-
-static void msgpack_rpc_from_handle(ObjectType type, Integer o, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(3)
-{
-  msgpack_packer pac;
-  msgpack_packer_init(&pac, &sbuffer, msgpack_sbuffer_write);
-  msgpack_pack_int64(&pac, (handle_T)o);
-  msgpack_pack_ext(res, sbuffer.size, (int8_t)(type - EXT_OBJECT_TYPE_SHIFT));
-  msgpack_pack_ext_body(res, sbuffer.data, sbuffer.size);
-  msgpack_sbuffer_clear(&sbuffer);
-}
-
-typedef struct {
-  const Object *aobj;
-  bool container;
-  size_t idx;
-} APIToMPObjectStackItem;
-
-/// Convert type used by Nvim API to msgpack type.
-///
-/// @param[in]  result  Object to convert.
-/// @param[out]  res  Structure that defines where conversion results are saved.
-///
-/// @return true in case of success, false otherwise.
-void msgpack_rpc_from_object(const Object result, msgpack_packer *const res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  kvec_withinit_t(APIToMPObjectStackItem, 2) stack = KV_INITIAL_VALUE;
-  kvi_init(stack);
-  kvi_push(stack, ((APIToMPObjectStackItem) { &result, false, 0 }));
-  while (kv_size(stack)) {
-    APIToMPObjectStackItem cur = kv_last(stack);
-    STATIC_ASSERT(kObjectTypeWindow == kObjectTypeBuffer + 1
-                  && kObjectTypeTabpage == kObjectTypeWindow + 1,
-                  "Buffer, window and tabpage enum items are in order");
-    switch (cur.aobj->type) {
-    case kObjectTypeNil:
-    case kObjectTypeLuaRef:
-      // TODO(bfredl): could also be an error. Though kObjectTypeLuaRef
-      // should only appear when the caller has opted in to handle references,
-      // see nlua_pop_Object.
-      msgpack_pack_nil(res);
-      break;
-    case kObjectTypeBoolean:
-      msgpack_rpc_from_boolean(cur.aobj->data.boolean, res);
-      break;
-    case kObjectTypeInteger:
-      msgpack_rpc_from_integer(cur.aobj->data.integer, res);
-      break;
-    case kObjectTypeFloat:
-      msgpack_rpc_from_float(cur.aobj->data.floating, res);
-      break;
-    case kObjectTypeString:
-      msgpack_rpc_from_string(cur.aobj->data.string, res);
-      break;
-    case kObjectTypeBuffer:
-    case kObjectTypeWindow:
-    case kObjectTypeTabpage:
-      msgpack_rpc_from_handle(cur.aobj->type, cur.aobj->data.integer, res);
-      break;
-    case kObjectTypeArray: {
-      const size_t size = cur.aobj->data.array.size;
-      if (cur.container) {
-        if (cur.idx >= size) {
-          (void)kv_pop(stack);
-        } else {
-          const size_t idx = cur.idx;
-          cur.idx++;
-          kv_last(stack) = cur;
-          kvi_push(stack, ((APIToMPObjectStackItem) {
-              .aobj = &cur.aobj->data.array.items[idx],
-              .container = false,
-          }));
-        }
-      } else {
-        msgpack_pack_array(res, size);
-        cur.container = true;
-        kv_last(stack) = cur;
-      }
-      break;
-    }
-    case kObjectTypeDictionary: {
-      const size_t size = cur.aobj->data.dictionary.size;
-      if (cur.container) {
-        if (cur.idx >= size) {
-          (void)kv_pop(stack);
-        } else {
-          const size_t idx = cur.idx;
-          cur.idx++;
-          kv_last(stack) = cur;
-          msgpack_rpc_from_string(cur.aobj->data.dictionary.items[idx].key, res);
-          kvi_push(stack, ((APIToMPObjectStackItem) {
-              .aobj = &cur.aobj->data.dictionary.items[idx].value,
-              .container = false,
-          }));
-        }
-      } else {
-        msgpack_pack_map(res, size);
-        cur.container = true;
-        kv_last(stack) = cur;
-      }
-      break;
-    }
-    }
-    if (!cur.container) {
-      (void)kv_pop(stack);
-    }
-  }
-  kvi_destroy(stack);
-}
-
-// uncrustify:on
-
-void msgpack_rpc_from_array(Array result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_array(res, result.size);
-
-  for (size_t i = 0; i < result.size; i++) {
-    msgpack_rpc_from_object(result.items[i], res);
-  }
-}
-
-void msgpack_rpc_from_dictionary(Dictionary result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_map(res, result.size);
-
-  for (size_t i = 0; i < result.size; i++) {
-    msgpack_rpc_from_string(result.items[i].key, res);
-    msgpack_rpc_from_object(result.items[i].value, res);
-  }
-}
-
-/// Serializes a msgpack-rpc request or notification(id == 0)
-void msgpack_rpc_serialize_request(uint32_t request_id, const String method, Array args,
-                                   msgpack_packer *pac)
-  FUNC_ATTR_NONNULL_ARG(4)
-{
-  msgpack_pack_array(pac, request_id ? 4 : 3);
-  msgpack_pack_int(pac, request_id ? 0 : 2);
-
-  if (request_id) {
-    msgpack_pack_uint32(pac, request_id);
-  }
-
-  msgpack_rpc_from_string(method, pac);
-  msgpack_rpc_from_array(args, pac);
-}
-
-/// Serializes a msgpack-rpc response
-void msgpack_rpc_serialize_response(uint32_t response_id, Error *err, Object arg,
-                                    msgpack_packer *pac)
-  FUNC_ATTR_NONNULL_ARG(2, 4)
-{
-  msgpack_pack_array(pac, 4);
-  msgpack_pack_int(pac, 1);
-  msgpack_pack_uint32(pac, response_id);
-
-  if (ERROR_SET(err)) {
-    // error represented by a [type, message] array
-    msgpack_pack_array(pac, 2);
-    msgpack_rpc_from_integer(err->type, pac);
-    msgpack_rpc_from_string(cstr_as_string(err->msg), pac);
-    // Nil result
-    msgpack_pack_nil(pac);
-  } else {
-    // Nil error
-    msgpack_pack_nil(pac);
-    // Return value
-    msgpack_rpc_from_object(arg, pac);
-  }
-}
-
-static bool msgpack_rpc_is_notification(msgpack_object *req)
-{
-  return req->via.array.ptr[0].via.u64 == 2;
-}
-
-msgpack_object *msgpack_rpc_method(msgpack_object *req)
-{
-  msgpack_object *obj = req->via.array.ptr
-                        + (msgpack_rpc_is_notification(req) ? 1 : 2);
-  return obj->type == MSGPACK_OBJECT_STR || obj->type == MSGPACK_OBJECT_BIN
-         ? obj : NULL;
-}
-
-msgpack_object *msgpack_rpc_args(msgpack_object *req)
-{
-  msgpack_object *obj = req->via.array.ptr
-                        + (msgpack_rpc_is_notification(req) ? 2 : 3);
-  return obj->type == MSGPACK_OBJECT_ARRAY ? obj : NULL;
-}
-
-static msgpack_object *msgpack_rpc_msg_id(msgpack_object *req)
-{
-  if (msgpack_rpc_is_notification(req)) {
-    return NULL;
-  }
-  msgpack_object *obj = &req->via.array.ptr[1];
-  return obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER ? obj : NULL;
-}
-
-MessageType msgpack_rpc_validate(uint32_t *response_id, msgpack_object *req, Error *err)
-{
-  *response_id = 0;
-  // Validate the basic structure of the msgpack-rpc payload
-  if (req->type != MSGPACK_OBJECT_ARRAY) {
-    api_set_error(err, kErrorTypeValidation, "Message is not an array");
-    return kMessageTypeUnknown;
-  }
-
-  if (req->via.array.size == 0) {
-    api_set_error(err, kErrorTypeValidation, "Message is empty");
-    return kMessageTypeUnknown;
-  }
-
-  if (req->via.array.ptr[0].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-    api_set_error(err, kErrorTypeValidation, "Message type must be an integer");
-    return kMessageTypeUnknown;
-  }
-
-  MessageType type = (MessageType)req->via.array.ptr[0].via.u64;
-  if (type != kMessageTypeRequest && type != kMessageTypeNotification) {
-    api_set_error(err, kErrorTypeValidation, "Unknown message type");
-    return kMessageTypeUnknown;
-  }
-
-  if ((type == kMessageTypeRequest && req->via.array.size != 4)
-      || (type == kMessageTypeNotification && req->via.array.size != 3)) {
-    api_set_error(err, kErrorTypeValidation,
-                  "Request array size must be 4 (request) or 3 (notification)");
-    return type;
-  }
-
-  if (type == kMessageTypeRequest) {
-    msgpack_object *id_obj = msgpack_rpc_msg_id(req);
-    if (!id_obj) {
-      api_set_error(err, kErrorTypeValidation, "ID must be a positive integer");
-      return type;
-    }
-    *response_id = (uint32_t)id_obj->via.u64;
-  }
-
-  if (!msgpack_rpc_method(req)) {
-    api_set_error(err, kErrorTypeValidation, "Method must be a string");
-    return type;
-  }
-
-  if (!msgpack_rpc_args(req)) {
-    api_set_error(err, kErrorTypeValidation, "Parameters must be an array");
-    return type;
-  }
-
-  return type;
-}
diff --git a/src/nvim/msgpack_rpc/helpers.h b/src/nvim/msgpack_rpc/helpers.h
deleted file mode 100644
index dd2096f305..0000000000
--- a/src/nvim/msgpack_rpc/helpers.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-#include <msgpack.h>
-#include <stdbool.h>
-#include <stdint.h>
-
-#include "nvim/api/private/defs.h"
-#include "nvim/event/wstream.h"
-
-/// Value by which objects represented as EXT type are shifted
-///
-/// Subtracted when packing, added when unpacking. Used to allow moving
-/// buffer/window/tabpage block inside ObjectType enum. This block yet cannot be
-/// split or reordered.
-#define EXT_OBJECT_TYPE_SHIFT kObjectTypeBuffer
-#define EXT_OBJECT_TYPE_MAX (kObjectTypeTabpage - EXT_OBJECT_TYPE_SHIFT)
-
-#ifdef INCLUDE_GENERATED_DECLARATIONS
-# include "msgpack_rpc/helpers.h.generated.h"
-#endif
diff --git a/src/nvim/msgpack_rpc/packer.c b/src/nvim/msgpack_rpc/packer.c
new file mode 100644
index 0000000000..cac68f76f0
--- /dev/null
+++ b/src/nvim/msgpack_rpc/packer.c
@@ -0,0 +1,235 @@
+#include <assert.h>
+
+#include "nvim/api/private/defs.h"
+#include "nvim/lua/executor.h"
+#include "nvim/msgpack_rpc/packer.h"
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "msgpack_rpc/packer.c.generated.h"
+#endif
+
+static void check_buffer(PackerBuffer *packer)
+{
+  ptrdiff_t remaining = packer->endptr - packer->ptr;
+  if (remaining < MPACK_ITEM_SIZE) {
+    packer->packer_flush(packer);
+  }
+}
+
+static void mpack_w8(char **b, const char *data)
+{
+#ifdef ORDER_BIG_ENDIAN
+  memcpy(*b, data, 8);
+  *b += 8;
+#else
+  for (int i = 7; i >= 0; i--) {
+    *(*b)++ = data[i];
+  }
+#endif
+}
+
+void mpack_integer(char **ptr, Integer i)
+{
+  if (i >= 0) {
+    if (i > 0xfffffff) {
+      mpack_w(ptr, 0xcf);
+      mpack_w8(ptr, (char *)&i);
+    } else {
+      mpack_uint(ptr, (uint32_t)i);
+    }
+  } else {
+    if (i < -0x80000000LL) {
+      mpack_w(ptr, 0xd3);
+      mpack_w8(ptr, (char *)&i);
+    } else if (i < -0x8000) {
+      mpack_w(ptr, 0xd2);
+      mpack_w4(ptr, (uint32_t)i);
+    } else if (i < -0x80) {
+      mpack_w(ptr, 0xd1);
+      mpack_w2(ptr, (uint32_t)i);
+    } else if (i < -0x20) {
+      mpack_w(ptr, 0xd0);
+      mpack_w(ptr, (char)i);
+    } else {
+      mpack_w(ptr, (char)i);
+    }
+  }
+}
+
+void mpack_float8(char **ptr, double i)
+{
+  mpack_w(ptr, 0xcb);
+  mpack_w8(ptr, (char *)&i);
+}
+
+void mpack_str(String str, PackerBuffer *packer)
+{
+  const size_t len = str.size;
+  if (len < 20) {
+    mpack_w(&packer->ptr, 0xa0 | len);
+  } else if (len < 0xff) {
+    mpack_w(&packer->ptr, 0xd9);
+    mpack_w(&packer->ptr, len);
+  } else if (len < 0xffff) {
+    mpack_w(&packer->ptr, 0xda);
+    mpack_w2(&packer->ptr, (uint32_t)len);
+  } else if (len < 0xffffffff) {
+    mpack_w(&packer->ptr, 0xdb);
+    mpack_w4(&packer->ptr, (uint32_t)len);
+  } else {
+    abort();
+  }
+
+  size_t pos = 0;
+  while (pos < len) {
+    ptrdiff_t remaining = packer->endptr - packer->ptr;
+    size_t to_copy = MIN(len - pos, (size_t)remaining);
+    memcpy(packer->ptr, str.data + pos, to_copy);
+    packer->ptr += to_copy;
+    pos += to_copy;
+
+    if (pos < len) {
+      packer->packer_flush(packer);
+    }
+  }
+}
+
+void mpack_handle(ObjectType type, handle_T handle, PackerBuffer *packer)
+{
+  char exttype = (char)(type - EXT_OBJECT_TYPE_SHIFT);
+  if (-0x1f <= handle && handle <= 0x7f) {
+    mpack_w(&packer->ptr, 0xd4);
+    mpack_w(&packer->ptr, exttype);
+    mpack_w(&packer->ptr, (char)handle);
+  } else {
+    // we want to encode some small negative sentinel like -1. This is handled above
+    assert(handle >= 0);
+    // FAIL: we cannot use fixext 4/8 due to a design error
+    // (in theory fixext 2 for handle<=0xff but we don't gain much from it)
+    char buf[MPACK_ITEM_SIZE];
+    char *pos = buf;
+    mpack_uint(&pos, (uint32_t)handle);
+    ptrdiff_t packsize = pos - buf;
+    mpack_w(&packer->ptr, 0xc7);
+    mpack_w(&packer->ptr, packsize);
+    mpack_w(&packer->ptr, exttype);
+    // check_buffer(packer);
+    memcpy(packer->ptr, buf, (size_t)packsize);
+    packer->ptr += packsize;
+  }
+}
+
+void mpack_object(Object *obj, PackerBuffer *packer)
+{
+  mpack_object_inner(obj, NULL, 0, packer);
+}
+
+void mpack_object_array(Array arr, PackerBuffer *packer)
+{
+  mpack_array(&packer->ptr, (uint32_t)arr.size);
+  if (arr.size > 0) {
+    Object container = ARRAY_OBJ(arr);
+    mpack_object_inner(&arr.items[0], arr.size > 1 ? &container : NULL, 1, packer);
+  }
+}
+
+typedef struct {
+  Object *container;
+  size_t idx;
+} ContainerStackItem;
+
+void mpack_object_inner(Object *current, Object *container, size_t container_idx,
+                        PackerBuffer *packer)
+  FUNC_ATTR_NONNULL_ARG(1, 4)
+{
+  // The inner loop of this function packs "current" and then fetches the next
+  // value from "container". "stack" is only used for nested containers.
+  kvec_withinit_t(ContainerStackItem, 2) stack = KV_INITIAL_VALUE;
+  kvi_init(stack);
+
+  while (true) {
+    check_buffer(packer);
+    switch (current->type) {
+    case kObjectTypeLuaRef:
+      // TODO(bfredl): could also be an error. Though kObjectTypeLuaRef
+      // should only appear when the caller has opted in to handle references,
+      // see nlua_pop_Object.
+      api_free_luaref(current->data.luaref);
+      current->data.luaref = LUA_NOREF;
+      FALLTHROUGH;
+    case kObjectTypeNil:
+      mpack_nil(&packer->ptr);
+      break;
+    case kObjectTypeBoolean:
+      mpack_bool(&packer->ptr, current->data.boolean);
+      break;
+    case kObjectTypeInteger:
+      mpack_integer(&packer->ptr, current->data.integer);
+      break;
+    case kObjectTypeFloat:
+      mpack_float8(&packer->ptr, current->data.floating);
+      break;
+    case kObjectTypeString:
+      mpack_str(current->data.string, packer);
+      break;
+    case kObjectTypeBuffer:
+    case kObjectTypeWindow:
+    case kObjectTypeTabpage:
+      mpack_handle(current->type, (handle_T)current->data.integer, packer);
+      break;
+    case kObjectTypeDictionary:
+    case kObjectTypeArray: {}
+      size_t current_size;
+      if (current->type == kObjectTypeArray) {
+        current_size = current->data.array.size;
+        mpack_array(&packer->ptr, (uint32_t)current_size);
+      } else {
+        current_size = current->data.dictionary.size;
+        mpack_map(&packer->ptr, (uint32_t)current_size);
+      }
+      if (current_size > 0) {
+        if (current->type == kObjectTypeArray && current_size == 1) {
+          current = &current->data.array.items[0];
+          continue;
+        }
+        if (container) {
+          kvi_push(stack, ((ContainerStackItem) {
+            .container = container,
+            .idx = container_idx,
+          }));
+        }
+        container = current;
+        container_idx = 0;
+      }
+      break;
+    }
+
+    if (!container) {
+      if (kv_size(stack)) {
+        ContainerStackItem it = kv_pop(stack);
+        container = it.container;
+        container_idx = it.idx;
+      } else {
+        break;
+      }
+    }
+
+    if (container->type == kObjectTypeArray) {
+      Array arr = container->data.array;
+      current = &arr.items[container_idx++];
+      if (container_idx >= arr.size) {
+        container = NULL;
+      }
+    } else {
+      Dictionary dict = container->data.dictionary;
+      KeyValuePair *it = &dict.items[container_idx++];
+      check_buffer(packer);
+      mpack_str(it->key, packer);
+      current = &it->value;
+      if (container_idx >= dict.size) {
+        container = NULL;
+      }
+    }
+  }
+  kvi_destroy(stack);
+}
diff --git a/src/nvim/msgpack_rpc/packer.h b/src/nvim/msgpack_rpc/packer.h
new file mode 100644
index 0000000000..8117bd09bd
--- /dev/null
+++ b/src/nvim/msgpack_rpc/packer.h
@@ -0,0 +1,76 @@
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "nvim/api/private/defs.h"
+#include "nvim/msgpack_rpc/packer_defs.h"
+
+#define mpack_w(b, byte) *(*(b))++ = (char)(byte);
+static inline void mpack_w2(char **b, uint32_t v)
+{
+  *(*b)++ = (char)((v >> 8) & 0xff);
+  *(*b)++ = (char)(v & 0xff);
+}
+
+static inline void mpack_w4(char **b, uint32_t v)
+{
+  *(*b)++ = (char)((v >> 24) & 0xff);
+  *(*b)++ = (char)((v >> 16) & 0xff);
+  *(*b)++ = (char)((v >> 8) & 0xff);
+  *(*b)++ = (char)(v & 0xff);
+}
+
+static inline void mpack_uint(char **buf, uint32_t val)
+{
+  if (val > 0xffff) {
+    mpack_w(buf, 0xce);
+    mpack_w4(buf, val);
+  } else if (val > 0xff) {
+    mpack_w(buf, 0xcd);
+    mpack_w2(buf, val);
+  } else if (val > 0x7f) {
+    mpack_w(buf, 0xcc);
+    mpack_w(buf, val);
+  } else {
+    mpack_w(buf, val);
+  }
+}
+
+#define mpack_nil(buf) mpack_w(buf, 0xc0)
+static inline void mpack_bool(char **buf, bool val)
+{
+  mpack_w(buf, 0xc2 | (val ? 1 : 0));
+}
+
+static inline void mpack_array(char **buf, uint32_t len)
+{
+  if (len < 0x10) {
+    mpack_w(buf, 0x90 | len);
+  } else if (len < 0x10000) {
+    mpack_w(buf, 0xdc);
+    mpack_w2(buf, len);
+  } else {
+    mpack_w(buf, 0xdd);
+    mpack_w4(buf, len);
+  }
+}
+
+static inline void mpack_map(char **buf, uint32_t len)
+{
+  if (len < 0x10) {
+    mpack_w(buf, 0x80 | len);
+  } else if (len < 0x10000) {
+    mpack_w(buf, 0xde);
+    mpack_w2(buf, len);
+  } else {
+    mpack_w(buf, 0xdf);
+    mpack_w4(buf, len);
+  }
+}
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "msgpack_rpc/packer.h.generated.h"
+#endif
diff --git a/src/nvim/msgpack_rpc/packer_defs.h b/src/nvim/msgpack_rpc/packer_defs.h
new file mode 100644
index 0000000000..420f3dc424
--- /dev/null
+++ b/src/nvim/msgpack_rpc/packer_defs.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+// Max possible length: bytecode + 8 int/float bytes
+// Ext objects are maximum 8=3+5 (nested uint32 payload)
+#define MPACK_ITEM_SIZE 9
+
+typedef struct packer_buffer_t PackerBuffer;
+
+// Must ensure at least MPACK_ITEM_SIZE of space.
+typedef void (*PackerBufferFlush)(PackerBuffer *self);
+
+struct packer_buffer_t {
+  char *startptr;
+  char *ptr;
+  char *endptr;
+
+  // these are free to be used by packer_flush for any purpose, if want
+  void *anydata;
+  size_t anylen;
+  PackerBufferFlush packer_flush;
+};
diff --git a/src/nvim/msgpack_rpc/server.c b/src/nvim/msgpack_rpc/server.c
index f3627eaa61..56b03d67d0 100644
--- a/src/nvim/msgpack_rpc/server.c
+++ b/src/nvim/msgpack_rpc/server.c
@@ -6,9 +6,10 @@
 
 #include "nvim/channel.h"
 #include "nvim/eval.h"
+#include "nvim/event/defs.h"
 #include "nvim/event/socket.h"
-#include "nvim/func_attr.h"
 #include "nvim/garray.h"
+#include "nvim/garray_defs.h"
 #include "nvim/log.h"
 #include "nvim/main.h"
 #include "nvim/memory.h"
diff --git a/src/nvim/msgpack_rpc/unpacker.c b/src/nvim/msgpack_rpc/unpacker.c
index 38263381bf..dbb30b0c9a 100644
--- a/src/nvim/msgpack_rpc/unpacker.c
+++ b/src/nvim/msgpack_rpc/unpacker.c
@@ -10,7 +10,6 @@
 #include "nvim/macros_defs.h"
 #include "nvim/memory.h"
 #include "nvim/msgpack_rpc/channel_defs.h"
-#include "nvim/msgpack_rpc/helpers.h"
 #include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/ui_client.h"
 
@@ -18,15 +17,18 @@
 # include "msgpack_rpc/unpacker.c.generated.h"
 #endif
 
-Object unpack(const char *data, size_t size, Error *err)
+Object unpack(const char *data, size_t size, Arena *arena, Error *err)
 {
   Unpacker unpacker;
   mpack_parser_init(&unpacker.parser, 0);
   unpacker.parser.data.p = &unpacker;
+  unpacker.arena = *arena;
 
   int result = mpack_parse(&unpacker.parser, &data, &size,
                            api_parse_enter, api_parse_exit);
 
+  *arena = unpacker.arena;
+
   if (result == MPACK_NOMEM) {
     api_set_error(err, kErrorTypeException, "object was too deep to unpack");
   } else if (result == MPACK_EOF) {
diff --git a/src/nvim/msgpack_rpc/unpacker.h b/src/nvim/msgpack_rpc/unpacker.h
index 53af29761e..022d778013 100644
--- a/src/nvim/msgpack_rpc/unpacker.h
+++ b/src/nvim/msgpack_rpc/unpacker.h
@@ -1,19 +1,17 @@
 #pragma once
 
 #include <inttypes.h>
-#include <stdbool.h>
 #include <string.h>
 
 #include "mpack/mpack_core.h"
 #include "mpack/object.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/dispatch.h"
-#include "nvim/api/private/helpers.h"
 #include "nvim/grid_defs.h"
 #include "nvim/memory_defs.h"
-#include "nvim/msgpack_rpc/channel_defs.h"
+#include "nvim/msgpack_rpc/channel_defs.h"  // IWYU pragma: keep
 #include "nvim/types_defs.h"
-#include "nvim/ui_client.h"
+#include "nvim/ui_defs.h"
 
 struct Unpacker {
   mpack_parser_t parser;