From 543e0256c19f397921a332e06b423215fd9aecb5 Mon Sep 17 00:00:00 2001
From: zeertzjq <zeertzjq@outlook.com>
Date: Thu, 30 Nov 2023 15:51:05 +0800
Subject: build: don't define FUNC_ATTR_* as empty in headers (#26317)

FUNC_ATTR_* should only be used in .c files with generated headers.
Defining FUNC_ATTR_* as empty in headers causes misuses of them to be
silently ignored. Instead don't define them by default, and only define
them as empty after a .c file has included its generated header.
---
 src/nvim/msgpack_rpc/channel.c | 1 -
 src/nvim/msgpack_rpc/helpers.c | 1 -
 src/nvim/msgpack_rpc/server.c  | 1 -
 3 files changed, 3 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 0fb1ebf931..50210e4936 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -20,7 +20,6 @@
 #include "nvim/event/rstream.h"
 #include "nvim/event/stream.h"
 #include "nvim/event/wstream.h"
-#include "nvim/func_attr.h"
 #include "nvim/globals.h"
 #include "nvim/log.h"
 #include "nvim/main.h"
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index 1fdfc9e536..d2be321e7a 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -10,7 +10,6 @@
 #include "msgpack/pack.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/assert_defs.h"
-#include "nvim/func_attr.h"
 #include "nvim/memory.h"
 #include "nvim/msgpack_rpc/helpers.h"
 #include "nvim/types_defs.h"
diff --git a/src/nvim/msgpack_rpc/server.c b/src/nvim/msgpack_rpc/server.c
index f3627eaa61..e60c1b88a5 100644
--- a/src/nvim/msgpack_rpc/server.c
+++ b/src/nvim/msgpack_rpc/server.c
@@ -7,7 +7,6 @@
 #include "nvim/channel.h"
 #include "nvim/eval.h"
 #include "nvim/event/socket.h"
-#include "nvim/func_attr.h"
 #include "nvim/garray.h"
 #include "nvim/log.h"
 #include "nvim/main.h"
-- 
cgit 


From 548f03c66c08d0235d62505e884e8088bfda1804 Mon Sep 17 00:00:00 2001
From: zeertzjq <zeertzjq@outlook.com>
Date: Fri, 1 Dec 2023 15:22:22 +0800
Subject: refactor: change event_create() to a macro (#26343)

A varargs functions can never be inlined, so a macro is faster.
---
 src/nvim/msgpack_rpc/channel.c | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 50210e4936..acc21bbf7e 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -404,7 +404,7 @@ static void handle_request(Channel *channel, Unpacker *p, Array args)
 
     if (is_get_mode && !input_blocking()) {
       // Defer the event to a special queue used by os/input.c. #6247
-      multiqueue_put(ch_before_blocking_events, request_event, 1, evdata);
+      multiqueue_put(ch_before_blocking_events, request_event, evdata);
     } else {
       // Invoke immediately.
       request_event((void **)&evdata);
@@ -412,12 +412,11 @@ static void handle_request(Channel *channel, Unpacker *p, Array args)
   } else {
     bool is_resize = p->handler.fn == handle_nvim_ui_try_resize;
     if (is_resize) {
-      Event ev = event_create_oneshot(event_create(request_event, 1, evdata),
-                                      2);
+      Event ev = event_create_oneshot(event_create(request_event, evdata), 2);
       multiqueue_put_event(channel->events, ev);
       multiqueue_put_event(resize_events, ev);
     } else {
-      multiqueue_put(channel->events, request_event, 1, evdata);
+      multiqueue_put(channel->events, request_event, evdata);
       DLOG("RPC: scheduled %.*s", (int)p->method_name_len, p->handler.name);
     }
   }
@@ -484,7 +483,7 @@ static bool channel_write(Channel *channel, WBuffer *buffer)
 
   if (channel->streamtype == kChannelStreamInternal) {
     channel_incref(channel);
-    CREATE_EVENT(channel->events, internal_read_event, 2, channel, buffer);
+    CREATE_EVENT(channel->events, internal_read_event, channel, buffer);
     success = true;
   } else {
     Stream *in = channel_instream(channel);
-- 
cgit 


From f6e5366d0077e9f171651f37282cb5c47629d1b6 Mon Sep 17 00:00:00 2001
From: zeertzjq <zeertzjq@outlook.com>
Date: Sat, 2 Dec 2023 07:55:44 +0800
Subject: refactor: free more reachable memory with EXITFREE (#26349)

Discovered using __sanitizer_print_memory_profile().
---
 src/nvim/msgpack_rpc/channel.c | 5 +++++
 src/nvim/msgpack_rpc/helpers.c | 8 ++++++++
 2 files changed, 13 insertions(+)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index acc21bbf7e..2336853609 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -751,6 +751,7 @@ const char *get_client_info(Channel *chan, const char *key)
   return NULL;
 }
 
+#ifdef EXITFREE
 void rpc_free_all_mem(void)
 {
   cstr_t key;
@@ -758,4 +759,8 @@ void rpc_free_all_mem(void)
     xfree((void *)key);
   });
   set_destroy(cstr_t, &event_strings);
+
+  msgpack_sbuffer_destroy(&out_buffer);
+  multiqueue_free(ch_before_blocking_events);
 }
+#endif
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index d2be321e7a..3162269df6 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -27,6 +27,14 @@ void msgpack_rpc_helpers_init(void)
   msgpack_sbuffer_init(&sbuffer);
 }
 
+#ifdef EXITFREE
+void msgpack_rpc_helpers_free_all_mem(void)
+{
+  msgpack_zone_destroy(&zone);
+  msgpack_sbuffer_destroy(&sbuffer);
+}
+#endif
+
 typedef struct {
   const msgpack_object *mobj;
   Object *aobj;
-- 
cgit 


From 69bc519b53ebf78fd95c8256468e7d538ebcb948 Mon Sep 17 00:00:00 2001
From: dundargoc <gocdundar@gmail.com>
Date: Tue, 12 Dec 2023 15:40:21 +0100
Subject: refactor: move non-symbols to defs.h headers

---
 src/nvim/msgpack_rpc/channel.c | 3 +--
 src/nvim/msgpack_rpc/channel.h | 6 +-----
 src/nvim/msgpack_rpc/server.c  | 1 +
 3 files changed, 3 insertions(+), 7 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 2336853609..c9139f5f93 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -15,10 +15,9 @@
 #include "nvim/api/ui.h"
 #include "nvim/channel.h"
 #include "nvim/event/defs.h"
-#include "nvim/event/loop.h"
+#include "nvim/event/multiqueue.h"
 #include "nvim/event/process.h"
 #include "nvim/event/rstream.h"
-#include "nvim/event/stream.h"
 #include "nvim/event/wstream.h"
 #include "nvim/globals.h"
 #include "nvim/log.h"
diff --git a/src/nvim/msgpack_rpc/channel.h b/src/nvim/msgpack_rpc/channel.h
index 818bee8318..a33081eadc 100644
--- a/src/nvim/msgpack_rpc/channel.h
+++ b/src/nvim/msgpack_rpc/channel.h
@@ -3,11 +3,7 @@
 #include <stdint.h>  // IWYU pragma: keep
 
 #include "nvim/api/private/defs.h"  // IWYU pragma: keep
-#include "nvim/channel.h"
-#include "nvim/event/multiqueue.h"
-#include "nvim/event/process.h"
-#include "nvim/event/socket.h"
-#include "nvim/event/wstream.h"
+#include "nvim/event/defs.h"
 #include "nvim/macros_defs.h"
 #include "nvim/memory_defs.h"  // IWYU pragma: keep
 #include "nvim/msgpack_rpc/channel_defs.h"  // IWYU pragma: export
diff --git a/src/nvim/msgpack_rpc/server.c b/src/nvim/msgpack_rpc/server.c
index e60c1b88a5..0cc249b2b8 100644
--- a/src/nvim/msgpack_rpc/server.c
+++ b/src/nvim/msgpack_rpc/server.c
@@ -6,6 +6,7 @@
 
 #include "nvim/channel.h"
 #include "nvim/eval.h"
+#include "nvim/event/defs.h"
 #include "nvim/event/socket.h"
 #include "nvim/garray.h"
 #include "nvim/log.h"
-- 
cgit 


From af93a74a0f4afa9a3a4f55ffdf28141eaf776d22 Mon Sep 17 00:00:00 2001
From: dundargoc <gocdundar@gmail.com>
Date: Mon, 18 Dec 2023 10:55:23 +0100
Subject: refactor: run IWYU on entire repo

Reference: https://github.com/neovim/neovim/issues/6371.
---
 src/nvim/msgpack_rpc/channel.c  | 1 +
 src/nvim/msgpack_rpc/helpers.h  | 5 +----
 src/nvim/msgpack_rpc/server.c   | 1 -
 src/nvim/msgpack_rpc/unpacker.h | 5 +----
 4 files changed, 3 insertions(+), 9 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index c9139f5f93..f1b7f8026e 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -15,6 +15,7 @@
 #include "nvim/api/ui.h"
 #include "nvim/channel.h"
 #include "nvim/event/defs.h"
+#include "nvim/event/loop.h"
 #include "nvim/event/multiqueue.h"
 #include "nvim/event/process.h"
 #include "nvim/event/rstream.h"
diff --git a/src/nvim/msgpack_rpc/helpers.h b/src/nvim/msgpack_rpc/helpers.h
index dd2096f305..6344d8c567 100644
--- a/src/nvim/msgpack_rpc/helpers.h
+++ b/src/nvim/msgpack_rpc/helpers.h
@@ -1,11 +1,8 @@
 #pragma once
 
-#include <msgpack.h>
-#include <stdbool.h>
-#include <stdint.h>
+#include <msgpack.h>  // IWYU pragma: keep
 
 #include "nvim/api/private/defs.h"
-#include "nvim/event/wstream.h"
 
 /// Value by which objects represented as EXT type are shifted
 ///
diff --git a/src/nvim/msgpack_rpc/server.c b/src/nvim/msgpack_rpc/server.c
index 0cc249b2b8..e60c1b88a5 100644
--- a/src/nvim/msgpack_rpc/server.c
+++ b/src/nvim/msgpack_rpc/server.c
@@ -6,7 +6,6 @@
 
 #include "nvim/channel.h"
 #include "nvim/eval.h"
-#include "nvim/event/defs.h"
 #include "nvim/event/socket.h"
 #include "nvim/garray.h"
 #include "nvim/log.h"
diff --git a/src/nvim/msgpack_rpc/unpacker.h b/src/nvim/msgpack_rpc/unpacker.h
index 53af29761e..d43dc2e997 100644
--- a/src/nvim/msgpack_rpc/unpacker.h
+++ b/src/nvim/msgpack_rpc/unpacker.h
@@ -1,17 +1,14 @@
 #pragma once
 
 #include <inttypes.h>
-#include <stdbool.h>
 #include <string.h>
 
 #include "mpack/mpack_core.h"
 #include "mpack/object.h"
-#include "nvim/api/private/defs.h"
 #include "nvim/api/private/dispatch.h"
-#include "nvim/api/private/helpers.h"
 #include "nvim/grid_defs.h"
 #include "nvim/memory_defs.h"
-#include "nvim/msgpack_rpc/channel_defs.h"
+#include "nvim/msgpack_rpc/channel_defs.h"  // IWYU pragma: keep
 #include "nvim/types_defs.h"
 #include "nvim/ui_client.h"
 
-- 
cgit 


From 1813661a6197c76ea6621284570aca1d56597099 Mon Sep 17 00:00:00 2001
From: dundargoc <gocdundar@gmail.com>
Date: Thu, 4 Jan 2024 15:38:16 +0100
Subject: refactor(IWYU): fix headers

Remove `export` pramgas from defs headers as it causes IWYU to believe
that the definitions from the defs headers comes from main header, which
is not what we really want.
---
 src/nvim/msgpack_rpc/channel.c      | 2 ++
 src/nvim/msgpack_rpc/channel.h      | 2 +-
 src/nvim/msgpack_rpc/channel_defs.h | 4 ----
 src/nvim/msgpack_rpc/server.c       | 2 ++
 src/nvim/msgpack_rpc/unpacker.h     | 3 ++-
 5 files changed, 7 insertions(+), 6 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index f1b7f8026e..286bb8098a 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -14,6 +14,7 @@
 #include "nvim/api/private/helpers.h"
 #include "nvim/api/ui.h"
 #include "nvim/channel.h"
+#include "nvim/channel_defs.h"
 #include "nvim/event/defs.h"
 #include "nvim/event/loop.h"
 #include "nvim/event/multiqueue.h"
@@ -32,6 +33,7 @@
 #include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/os/input.h"
 #include "nvim/rbuffer.h"
+#include "nvim/rbuffer_defs.h"
 #include "nvim/types_defs.h"
 #include "nvim/ui.h"
 #include "nvim/ui_client.h"
diff --git a/src/nvim/msgpack_rpc/channel.h b/src/nvim/msgpack_rpc/channel.h
index a33081eadc..ff73a2fc53 100644
--- a/src/nvim/msgpack_rpc/channel.h
+++ b/src/nvim/msgpack_rpc/channel.h
@@ -6,7 +6,7 @@
 #include "nvim/event/defs.h"
 #include "nvim/macros_defs.h"
 #include "nvim/memory_defs.h"  // IWYU pragma: keep
-#include "nvim/msgpack_rpc/channel_defs.h"  // IWYU pragma: export
+#include "nvim/msgpack_rpc/channel_defs.h"  // IWYU pragma: keep
 
 #define METHOD_MAXLEN 512
 
diff --git a/src/nvim/msgpack_rpc/channel_defs.h b/src/nvim/msgpack_rpc/channel_defs.h
index 20b8a89afb..56dbb332e0 100644
--- a/src/nvim/msgpack_rpc/channel_defs.h
+++ b/src/nvim/msgpack_rpc/channel_defs.h
@@ -4,11 +4,7 @@
 #include <stdbool.h>
 #include <uv.h>
 
-#include "klib/kvec.h"
-#include "nvim/api/private/defs.h"
 #include "nvim/api/private/dispatch.h"
-#include "nvim/event/process.h"
-#include "nvim/event/socket.h"
 #include "nvim/map_defs.h"
 
 typedef struct Channel Channel;
diff --git a/src/nvim/msgpack_rpc/server.c b/src/nvim/msgpack_rpc/server.c
index e60c1b88a5..56b03d67d0 100644
--- a/src/nvim/msgpack_rpc/server.c
+++ b/src/nvim/msgpack_rpc/server.c
@@ -6,8 +6,10 @@
 
 #include "nvim/channel.h"
 #include "nvim/eval.h"
+#include "nvim/event/defs.h"
 #include "nvim/event/socket.h"
 #include "nvim/garray.h"
+#include "nvim/garray_defs.h"
 #include "nvim/log.h"
 #include "nvim/main.h"
 #include "nvim/memory.h"
diff --git a/src/nvim/msgpack_rpc/unpacker.h b/src/nvim/msgpack_rpc/unpacker.h
index d43dc2e997..022d778013 100644
--- a/src/nvim/msgpack_rpc/unpacker.h
+++ b/src/nvim/msgpack_rpc/unpacker.h
@@ -5,12 +5,13 @@
 
 #include "mpack/mpack_core.h"
 #include "mpack/object.h"
+#include "nvim/api/private/defs.h"
 #include "nvim/api/private/dispatch.h"
 #include "nvim/grid_defs.h"
 #include "nvim/memory_defs.h"
 #include "nvim/msgpack_rpc/channel_defs.h"  // IWYU pragma: keep
 #include "nvim/types_defs.h"
-#include "nvim/ui_client.h"
+#include "nvim/ui_defs.h"
 
 struct Unpacker {
   mpack_parser_t parser;
-- 
cgit 


From 4d4092ac9e98f04ae949c605aa6e2b55ca605a1f Mon Sep 17 00:00:00 2001
From: nwounkn <nwounkn@gmail.com>
Date: Wed, 16 Aug 2023 04:33:39 +0500
Subject: fix(rpc): assertion failure due to invalid msgpack input

Problem:
  rbuffer_consumed assertion fails if Unpacker fails to parse msgpack,
  because it doesn't consume bytes on errors

Solution:
  Call rbuffer_consumed_compact only if Unpacker isn't closed
---
 src/nvim/msgpack_rpc/channel.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 286bb8098a..0178ef622b 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -304,8 +304,11 @@ static void receive_msgpack(Stream *stream, RBuffer *rbuf, size_t c, void *data,
   p->read_ptr = rbuffer_read_ptr(rbuf, &size);
   p->read_size = size;
   parse_msgpack(channel);
-  size_t consumed = size - p->read_size;
-  rbuffer_consumed_compact(rbuf, consumed);
+
+  if (!unpacker_closed(p)) {
+    size_t consumed = size - p->read_size;
+    rbuffer_consumed_compact(rbuf, consumed);
+  }
 
 end:
   channel_decref(channel);
-- 
cgit 


From e0e5b7f0ba1b0440bdc2b557e2b2cfae24706cbd Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Fri, 9 Feb 2024 11:42:40 +0100
Subject: refactor(api): make cstr_as_string accept "const char*"

In the context a String inside an Object/Dictionary etc is consumed,
it is considered to be read-only.
---
 src/nvim/msgpack_rpc/channel.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 0178ef622b..36fa7e77fc 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -547,7 +547,7 @@ static void send_error(Channel *chan, MsgpackRpcRequestHandler handler, MessageT
 
 static void send_request(Channel *channel, uint32_t id, const char *name, Array args)
 {
-  const String method = cstr_as_string((char *)name);
+  const String method = cstr_as_string(name);
   channel_write(channel, serialize_request(channel->id,
                                            id,
                                            method,
@@ -558,7 +558,7 @@ static void send_request(Channel *channel, uint32_t id, const char *name, Array
 
 static void send_event(Channel *channel, const char *name, Array args)
 {
-  const String method = cstr_as_string((char *)name);
+  const String method = cstr_as_string(name);
   channel_write(channel, serialize_request(channel->id,
                                            0,
                                            method,
@@ -583,7 +583,7 @@ static void broadcast_event(const char *name, Array args)
     goto end;
   }
 
-  const String method = cstr_as_string((char *)name);
+  const String method = cstr_as_string(name);
   WBuffer *buffer = serialize_request(0,
                                       0,
                                       method,
-- 
cgit 


From 0353dd3029f9ce31c3894530385443a90f6677ee Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Sun, 11 Feb 2024 15:46:14 +0100
Subject: refactor(lua): use Arena when converting from lua stack to API args

and for return value of nlua_exec/nlua_call_ref, as this uses
the same family of functions.

NB: the handling of luaref:s is a bit of a mess.
add api_luarefs_free_XX functions as a stop-gap as refactoring
luarefs is a can of worms for another PR:s.

as a minor feature/bug-fix, nvim_buf_call and nvim_win_call now preserves
arbitrary return values.
---
 src/nvim/msgpack_rpc/channel.c | 2 --
 1 file changed, 2 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 36fa7e77fc..2224403fa2 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -192,7 +192,6 @@ Object rpc_send_call(uint64_t id, const char *method_name, Array args, ArenaMem
 
   if (!(channel = find_rpc_channel(id))) {
     api_set_error(err, kErrorTypeException, "Invalid channel: %" PRIu64, id);
-    api_free_array(args);
     return NIL;
   }
 
@@ -201,7 +200,6 @@ Object rpc_send_call(uint64_t id, const char *method_name, Array args, ArenaMem
   uint32_t request_id = rpc->next_request_id++;
   // Send the msgpack-rpc request
   send_request(channel, request_id, method_name, args);
-  api_free_array(args);
 
   // Push the frame
   ChannelCallFrame frame = { request_id, false, false, NIL, NULL };
-- 
cgit 


From d60412b18e4e21f301baa2ac3f3fb7be89655e4b Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Mon, 12 Feb 2024 20:40:27 +0100
Subject: refactor(eval): use arena when converting typvals to Object

Note: this contains two _temporary_ changes which can be reverted
once the Arena vs no-Arena distinction in API wrappers has been removed.
Both nlua_push_Object and object_to_vim_take_luaref() has been changed
to take the object argument as a pointer. This is not going to be
necessary once these are only used with arena (or not at all) allocated
Objects.

The object_to_vim() variant which leaves luaref untouched might need to
stay for a little longer.
---
 src/nvim/msgpack_rpc/channel.c |  6 +++---
 src/nvim/msgpack_rpc/helpers.c | 23 +++++++++++++++--------
 2 files changed, 18 insertions(+), 11 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 2224403fa2..b411854387 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -449,7 +449,7 @@ static void request_event(void **argv)
                                               e->type,
                                               e->request_id,
                                               &error,
-                                              result,
+                                              &result,
                                               &out_buffer));
   }
   if (!handler.arena_return) {
@@ -538,7 +538,7 @@ static void send_error(Channel *chan, MsgpackRpcRequestHandler handler, MessageT
                                          type,
                                          id,
                                          &e,
-                                         NIL,
+                                         &NIL,
                                          &out_buffer));
   api_clear_error(&e);
 }
@@ -669,7 +669,7 @@ static WBuffer *serialize_request(uint64_t channel_id, uint32_t request_id, cons
 }
 
 static WBuffer *serialize_response(uint64_t channel_id, MsgpackRpcRequestHandler handler,
-                                   MessageType type, uint32_t response_id, Error *err, Object arg,
+                                   MessageType type, uint32_t response_id, Error *err, Object *arg,
                                    msgpack_sbuffer *sbuffer)
 {
   msgpack_packer pac;
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index 3162269df6..6caf50d319 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -10,6 +10,7 @@
 #include "msgpack/pack.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/assert_defs.h"
+#include "nvim/lua/executor.h"
 #include "nvim/memory.h"
 #include "nvim/msgpack_rpc/helpers.h"
 #include "nvim/types_defs.h"
@@ -309,34 +310,40 @@ static void msgpack_rpc_from_handle(ObjectType type, Integer o, msgpack_packer *
 }
 
 typedef struct {
-  const Object *aobj;
+  Object *aobj;
   bool container;
   size_t idx;
 } APIToMPObjectStackItem;
 
 /// Convert type used by Nvim API to msgpack type.
 ///
+/// consumes (frees) any luaref inside `result`, even though they are not used
+/// (just represented as NIL)
+///
 /// @param[in]  result  Object to convert.
 /// @param[out]  res  Structure that defines where conversion results are saved.
 ///
 /// @return true in case of success, false otherwise.
-void msgpack_rpc_from_object(const Object result, msgpack_packer *const res)
+void msgpack_rpc_from_object(Object *result, msgpack_packer *const res)
   FUNC_ATTR_NONNULL_ARG(2)
 {
   kvec_withinit_t(APIToMPObjectStackItem, 2) stack = KV_INITIAL_VALUE;
   kvi_init(stack);
-  kvi_push(stack, ((APIToMPObjectStackItem) { &result, false, 0 }));
+  kvi_push(stack, ((APIToMPObjectStackItem) { result, false, 0 }));
   while (kv_size(stack)) {
     APIToMPObjectStackItem cur = kv_last(stack);
     STATIC_ASSERT(kObjectTypeWindow == kObjectTypeBuffer + 1
                   && kObjectTypeTabpage == kObjectTypeWindow + 1,
                   "Buffer, window and tabpage enum items are in order");
     switch (cur.aobj->type) {
-    case kObjectTypeNil:
     case kObjectTypeLuaRef:
       // TODO(bfredl): could also be an error. Though kObjectTypeLuaRef
       // should only appear when the caller has opted in to handle references,
       // see nlua_pop_Object.
+      api_free_luaref(cur.aobj->data.luaref);
+      cur.aobj->data.luaref = LUA_NOREF;
+      FALLTHROUGH;
+    case kObjectTypeNil:
       msgpack_pack_nil(res);
       break;
     case kObjectTypeBoolean:
@@ -415,7 +422,7 @@ void msgpack_rpc_from_array(Array result, msgpack_packer *res)
   msgpack_pack_array(res, result.size);
 
   for (size_t i = 0; i < result.size; i++) {
-    msgpack_rpc_from_object(result.items[i], res);
+    msgpack_rpc_from_object(&result.items[i], res);
   }
 }
 
@@ -426,7 +433,7 @@ void msgpack_rpc_from_dictionary(Dictionary result, msgpack_packer *res)
 
   for (size_t i = 0; i < result.size; i++) {
     msgpack_rpc_from_string(result.items[i].key, res);
-    msgpack_rpc_from_object(result.items[i].value, res);
+    msgpack_rpc_from_object(&result.items[i].value, res);
   }
 }
 
@@ -447,9 +454,9 @@ void msgpack_rpc_serialize_request(uint32_t request_id, const String method, Arr
 }
 
 /// Serializes a msgpack-rpc response
-void msgpack_rpc_serialize_response(uint32_t response_id, Error *err, Object arg,
+void msgpack_rpc_serialize_response(uint32_t response_id, Error *err, Object *arg,
                                     msgpack_packer *pac)
-  FUNC_ATTR_NONNULL_ARG(2, 4)
+  FUNC_ATTR_NONNULL_ALL
 {
   msgpack_pack_array(pac, 4);
   msgpack_pack_int(pac, 1);
-- 
cgit 


From 146333ca123ab85397eb089345569cd9ed2d405a Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Sat, 17 Feb 2024 20:33:11 +0100
Subject: refactor(api): use arena for channel info and terminal info

---
 src/nvim/msgpack_rpc/channel.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index b411854387..1208c91760 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -682,12 +682,11 @@ static WBuffer *serialize_response(uint64_t channel_id, MsgpackRpcRequestHandler
       semsg("paste: %s", err->msg);
       api_clear_error(err);
     } else {
-      Array args = ARRAY_DICT_INIT;
-      ADD(args, INTEGER_OBJ(err->type));
-      ADD(args, CSTR_TO_OBJ(err->msg));
+      MAXSIZE_TEMP_ARRAY(args, 2);
+      ADD_C(args, INTEGER_OBJ(err->type));
+      ADD_C(args, CSTR_AS_OBJ(err->msg));
       msgpack_rpc_serialize_request(0, cstr_as_string("nvim_error_event"),
                                     args, &pac);
-      api_free_array(args);
     }
   } else {
     msgpack_rpc_serialize_response(response_id, err, arg, &pac);
-- 
cgit 


From bbf6d4a4bc44c8e61f807d23cc4ff2c2683e1ef4 Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Sun, 18 Feb 2024 12:51:27 +0100
Subject: refactor(api): use arena for metadata; msgpack_rpc_to_object delenda
 est

Note: kSDItemHeader is something is _written_ by nvim in the shada file
to identify it for debugging purposes outside of nvim. But this data wasn't ever used by
neovim after reading the file back, So I removed the parsing of it for now.
---
 src/nvim/msgpack_rpc/helpers.c  | 229 ----------------------------------------
 src/nvim/msgpack_rpc/unpacker.c |   5 +-
 2 files changed, 4 insertions(+), 230 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index 6caf50d319..c0c519848c 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -36,237 +36,8 @@ void msgpack_rpc_helpers_free_all_mem(void)
 }
 #endif
 
-typedef struct {
-  const msgpack_object *mobj;
-  Object *aobj;
-  bool container;
-  size_t idx;
-} MPToAPIObjectStackItem;
-
 // uncrustify:off
 
-/// Convert type used by msgpack parser to Nvim API type.
-///
-/// @param[in]  obj  Msgpack value to convert.
-/// @param[out]  arg  Location where result of conversion will be saved.
-///
-/// @return true in case of success, false otherwise.
-bool msgpack_rpc_to_object(const msgpack_object *const obj, Object *const arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  bool ret = true;
-  kvec_withinit_t(MPToAPIObjectStackItem, 2) stack = KV_INITIAL_VALUE;
-  kvi_init(stack);
-  kvi_push(stack, ((MPToAPIObjectStackItem) {
-    .mobj = obj,
-    .aobj = arg,
-    .container = false,
-    .idx = 0,
-  }));
-  while (ret && kv_size(stack)) {
-    MPToAPIObjectStackItem cur = kv_last(stack);
-    if (!cur.container) {
-      *cur.aobj = NIL;
-    }
-    switch (cur.mobj->type) {
-    case MSGPACK_OBJECT_NIL:
-      break;
-    case MSGPACK_OBJECT_BOOLEAN:
-      *cur.aobj = BOOLEAN_OBJ(cur.mobj->via.boolean);
-      break;
-    case MSGPACK_OBJECT_NEGATIVE_INTEGER:
-      STATIC_ASSERT(sizeof(Integer) == sizeof(cur.mobj->via.i64),
-                    "Msgpack integer size does not match API integer");
-      *cur.aobj = INTEGER_OBJ(cur.mobj->via.i64);
-      break;
-    case MSGPACK_OBJECT_POSITIVE_INTEGER:
-      STATIC_ASSERT(sizeof(Integer) == sizeof(cur.mobj->via.u64),
-                    "Msgpack integer size does not match API integer");
-      if (cur.mobj->via.u64 > API_INTEGER_MAX) {
-        ret = false;
-      } else {
-        *cur.aobj = INTEGER_OBJ((Integer)cur.mobj->via.u64);
-      }
-      break;
-    case MSGPACK_OBJECT_FLOAT32:
-    case MSGPACK_OBJECT_FLOAT64:
-    {
-      STATIC_ASSERT(sizeof(Float) == sizeof(cur.mobj->via.f64),
-                    "Msgpack floating-point size does not match API integer");
-      *cur.aobj = FLOAT_OBJ(cur.mobj->via.f64);
-      break;
-    }
-#define STR_CASE(type, attr, obj, dest, conv) \
-  case type: { \
-      dest = conv(((String) { \
-      .size = obj->via.attr.size, \
-      .data = (obj->via.attr.ptr == NULL || obj->via.attr.size == 0 \
-               ? xmemdupz("", 0) \
-               : xmemdupz(obj->via.attr.ptr, obj->via.attr.size)), })); \
-      break; \
-  }
-      STR_CASE(MSGPACK_OBJECT_STR, str, cur.mobj, *cur.aobj, STRING_OBJ)
-      STR_CASE(MSGPACK_OBJECT_BIN, bin, cur.mobj, *cur.aobj, STRING_OBJ)
-    case MSGPACK_OBJECT_ARRAY: {
-      const size_t size = cur.mobj->via.array.size;
-      if (cur.container) {
-        if (cur.idx >= size) {
-          (void)kv_pop(stack);
-        } else {
-          const size_t idx = cur.idx;
-          cur.idx++;
-          kv_last(stack) = cur;
-          kvi_push(stack, ((MPToAPIObjectStackItem) {
-              .mobj = &cur.mobj->via.array.ptr[idx],
-              .aobj = &cur.aobj->data.array.items[idx],
-              .container = false,
-          }));
-        }
-      } else {
-        *cur.aobj = ARRAY_OBJ(((Array) {
-            .size = size,
-            .capacity = size,
-            .items = (size > 0
-                      ? xcalloc(size, sizeof(*cur.aobj->data.array.items))
-                      : NULL),
-        }));
-        cur.container = true;
-        kv_last(stack) = cur;
-      }
-      break;
-    }
-    case MSGPACK_OBJECT_MAP: {
-      const size_t size = cur.mobj->via.map.size;
-      if (cur.container) {
-        if (cur.idx >= size) {
-          (void)kv_pop(stack);
-        } else {
-          const size_t idx = cur.idx;
-          cur.idx++;
-          kv_last(stack) = cur;
-          const msgpack_object *const key = &cur.mobj->via.map.ptr[idx].key;
-          switch (key->type) {
-#define ID(x) x
-          STR_CASE(MSGPACK_OBJECT_STR, str, key,
-                   cur.aobj->data.dictionary.items[idx].key, ID)
-          STR_CASE(MSGPACK_OBJECT_BIN, bin, key,
-                   cur.aobj->data.dictionary.items[idx].key, ID)
-#undef ID
-          case MSGPACK_OBJECT_NIL:
-          case MSGPACK_OBJECT_BOOLEAN:
-          case MSGPACK_OBJECT_POSITIVE_INTEGER:
-          case MSGPACK_OBJECT_NEGATIVE_INTEGER:
-          case MSGPACK_OBJECT_FLOAT32:
-          case MSGPACK_OBJECT_FLOAT64:
-          case MSGPACK_OBJECT_EXT:
-          case MSGPACK_OBJECT_MAP:
-          case MSGPACK_OBJECT_ARRAY:
-            ret = false;
-            break;
-          }
-          if (ret) {
-            kvi_push(stack, ((MPToAPIObjectStackItem) {
-                .mobj = &cur.mobj->via.map.ptr[idx].val,
-                .aobj = &cur.aobj->data.dictionary.items[idx].value,
-                .container = false,
-            }));
-          }
-        }
-      } else {
-        *cur.aobj = DICTIONARY_OBJ(((Dictionary) {
-            .size = size,
-            .capacity = size,
-            .items = (size > 0
-                      ? xcalloc(size, sizeof(*cur.aobj->data.dictionary.items))
-                      : NULL),
-        }));
-        cur.container = true;
-        kv_last(stack) = cur;
-      }
-      break;
-    }
-    case MSGPACK_OBJECT_EXT:
-      if (0 <= cur.mobj->via.ext.type && cur.mobj->via.ext.type <= EXT_OBJECT_TYPE_MAX) {
-        cur.aobj->type = (ObjectType)(cur.mobj->via.ext.type + EXT_OBJECT_TYPE_SHIFT);
-        msgpack_object data;
-        msgpack_unpack_return status = msgpack_unpack(cur.mobj->via.ext.ptr, cur.mobj->via.ext.size,
-                                                      NULL, &zone, &data);
-
-        if (status != MSGPACK_UNPACK_SUCCESS || data.type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-          ret = false;
-          break;
-        }
-        cur.aobj->data.integer = (handle_T)data.via.i64;
-        ret = true;
-      }
-      break;
-#undef STR_CASE
-    }
-    if (!cur.container) {
-      (void)kv_pop(stack);
-    }
-  }
-  kvi_destroy(stack);
-  return ret;
-}
-
-static bool msgpack_rpc_to_string(const msgpack_object *const obj, String *const arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  if (obj->type == MSGPACK_OBJECT_BIN || obj->type == MSGPACK_OBJECT_STR) {
-    arg->data = obj->via.bin.ptr != NULL
-                    ? xmemdupz(obj->via.bin.ptr, obj->via.bin.size)
-                    : NULL;
-    arg->size = obj->via.bin.size;
-    return true;
-  }
-  return false;
-}
-
-bool msgpack_rpc_to_array(const msgpack_object *const obj, Array *const arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  if (obj->type != MSGPACK_OBJECT_ARRAY) {
-    return false;
-  }
-
-  arg->size = obj->via.array.size;
-  arg->items = xcalloc(obj->via.array.size, sizeof(Object));
-
-  for (uint32_t i = 0; i < obj->via.array.size; i++) {
-    if (!msgpack_rpc_to_object(obj->via.array.ptr + i, &arg->items[i])) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-bool msgpack_rpc_to_dictionary(const msgpack_object *const obj, Dictionary *const arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  if (obj->type != MSGPACK_OBJECT_MAP) {
-    return false;
-  }
-
-  arg->size = obj->via.array.size;
-  arg->items = xcalloc(obj->via.map.size, sizeof(KeyValuePair));
-
-  for (uint32_t i = 0; i < obj->via.map.size; i++) {
-    if (!msgpack_rpc_to_string(&obj->via.map.ptr[i].key,
-                               &arg->items[i].key)) {
-      return false;
-    }
-
-    if (!msgpack_rpc_to_object(&obj->via.map.ptr[i].val,
-                               &arg->items[i].value)) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
 void msgpack_rpc_from_boolean(Boolean result, msgpack_packer *res)
   FUNC_ATTR_NONNULL_ARG(2)
 {
diff --git a/src/nvim/msgpack_rpc/unpacker.c b/src/nvim/msgpack_rpc/unpacker.c
index 38263381bf..6bc138f65a 100644
--- a/src/nvim/msgpack_rpc/unpacker.c
+++ b/src/nvim/msgpack_rpc/unpacker.c
@@ -18,15 +18,18 @@
 # include "msgpack_rpc/unpacker.c.generated.h"
 #endif
 
-Object unpack(const char *data, size_t size, Error *err)
+Object unpack(const char *data, size_t size, Arena *arena, Error *err)
 {
   Unpacker unpacker;
   mpack_parser_init(&unpacker.parser, 0);
   unpacker.parser.data.p = &unpacker;
+  unpacker.arena = *arena;
 
   int result = mpack_parse(&unpacker.parser, &data, &size,
                            api_parse_enter, api_parse_exit);
 
+  *arena = unpacker.arena;
+
   if (result == MPACK_NOMEM) {
     api_set_error(err, kErrorTypeException, "object was too deep to unpack");
   } else if (result == MPACK_EOF) {
-- 
cgit 


From 3cc54586be7760652e8bad88cae82ce74ef9432e Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Tue, 20 Feb 2024 13:44:50 +0100
Subject: refactor(api): make freeing of return-value opt-in instead of opt out

As only a few API functions make use of explicit freeing of the return
value, make it opt-in instead. The arena is always present under the
hood, so `Arena *arena` arg now doesn't mean anything other than getting
access to this arena. Also it is in principle possible to return an
allocated value while still using the arena as scratch space for other
stuff (unlikely, but there no reason to not allow it).
---
 src/nvim/msgpack_rpc/channel.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 1208c91760..ee7359c476 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -452,7 +452,7 @@ static void request_event(void **argv)
                                               &result,
                                               &out_buffer));
   }
-  if (!handler.arena_return) {
+  if (handler.ret_alloc) {
     api_free_object(result);
   }
 
-- 
cgit 


From cdcdc10411b460b6f69aeb01bcd8bb8dcd97c5c4 Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Mon, 26 Feb 2024 14:46:32 +0100
Subject: refactor(msgpack): remove dead unpacker code in helpers

Unpacker code now lives in unpacker.c
This was part of the old unpacker which I forgor to remove.
---
 src/nvim/msgpack_rpc/helpers.c | 87 ------------------------------------------
 1 file changed, 87 deletions(-)

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index c0c519848c..bc1eefb270 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -1,7 +1,6 @@
 #include <msgpack/object.h>
 #include <msgpack/sbuffer.h>
 #include <msgpack/unpack.h>
-#include <msgpack/zone.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -19,19 +18,16 @@
 # include "msgpack_rpc/helpers.c.generated.h"
 #endif
 
-static msgpack_zone zone;
 static msgpack_sbuffer sbuffer;
 
 void msgpack_rpc_helpers_init(void)
 {
-  msgpack_zone_init(&zone, 0xfff);
   msgpack_sbuffer_init(&sbuffer);
 }
 
 #ifdef EXITFREE
 void msgpack_rpc_helpers_free_all_mem(void)
 {
-  msgpack_zone_destroy(&zone);
   msgpack_sbuffer_destroy(&sbuffer);
 }
 #endif
@@ -247,86 +243,3 @@ void msgpack_rpc_serialize_response(uint32_t response_id, Error *err, Object *ar
     msgpack_rpc_from_object(arg, pac);
   }
 }
-
-static bool msgpack_rpc_is_notification(msgpack_object *req)
-{
-  return req->via.array.ptr[0].via.u64 == 2;
-}
-
-msgpack_object *msgpack_rpc_method(msgpack_object *req)
-{
-  msgpack_object *obj = req->via.array.ptr
-                        + (msgpack_rpc_is_notification(req) ? 1 : 2);
-  return obj->type == MSGPACK_OBJECT_STR || obj->type == MSGPACK_OBJECT_BIN
-         ? obj : NULL;
-}
-
-msgpack_object *msgpack_rpc_args(msgpack_object *req)
-{
-  msgpack_object *obj = req->via.array.ptr
-                        + (msgpack_rpc_is_notification(req) ? 2 : 3);
-  return obj->type == MSGPACK_OBJECT_ARRAY ? obj : NULL;
-}
-
-static msgpack_object *msgpack_rpc_msg_id(msgpack_object *req)
-{
-  if (msgpack_rpc_is_notification(req)) {
-    return NULL;
-  }
-  msgpack_object *obj = &req->via.array.ptr[1];
-  return obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER ? obj : NULL;
-}
-
-MessageType msgpack_rpc_validate(uint32_t *response_id, msgpack_object *req, Error *err)
-{
-  *response_id = 0;
-  // Validate the basic structure of the msgpack-rpc payload
-  if (req->type != MSGPACK_OBJECT_ARRAY) {
-    api_set_error(err, kErrorTypeValidation, "Message is not an array");
-    return kMessageTypeUnknown;
-  }
-
-  if (req->via.array.size == 0) {
-    api_set_error(err, kErrorTypeValidation, "Message is empty");
-    return kMessageTypeUnknown;
-  }
-
-  if (req->via.array.ptr[0].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-    api_set_error(err, kErrorTypeValidation, "Message type must be an integer");
-    return kMessageTypeUnknown;
-  }
-
-  MessageType type = (MessageType)req->via.array.ptr[0].via.u64;
-  if (type != kMessageTypeRequest && type != kMessageTypeNotification) {
-    api_set_error(err, kErrorTypeValidation, "Unknown message type");
-    return kMessageTypeUnknown;
-  }
-
-  if ((type == kMessageTypeRequest && req->via.array.size != 4)
-      || (type == kMessageTypeNotification && req->via.array.size != 3)) {
-    api_set_error(err, kErrorTypeValidation,
-                  "Request array size must be 4 (request) or 3 (notification)");
-    return type;
-  }
-
-  if (type == kMessageTypeRequest) {
-    msgpack_object *id_obj = msgpack_rpc_msg_id(req);
-    if (!id_obj) {
-      api_set_error(err, kErrorTypeValidation, "ID must be a positive integer");
-      return type;
-    }
-    *response_id = (uint32_t)id_obj->via.u64;
-  }
-
-  if (!msgpack_rpc_method(req)) {
-    api_set_error(err, kErrorTypeValidation, "Method must be a string");
-    return type;
-  }
-
-  if (!msgpack_rpc_args(req)) {
-    api_set_error(err, kErrorTypeValidation, "Parameters must be an array");
-    return type;
-  }
-
-  return type;
-}
-- 
cgit 


From dc37c1550bed46fffbb677d343cdc5bc94056219 Mon Sep 17 00:00:00 2001
From: bfredl <bjorn.linse@gmail.com>
Date: Sun, 25 Feb 2024 15:02:48 +0100
Subject: refactor(msgpack): allow flushing buffer while packing msgpack

Before, we needed to always pack an entire msgpack_rpc Object to
a continous memory buffer before sending it out to a channel.
But this is generally wasteful. it is better to just flush
whatever is in the buffer and then continue packing to a new buffer.

This is also done for the UI event packer where there are some extra logic
to "finish" of an existing batch of nevents/ncalls. This doesn't really
stop us from flushing the buffer, just that we need to update the state
machine accordingly so the next call to prepare_call() always will
start with a new event (even though the buffer might contain overflow
data from a large event).
---
 src/nvim/msgpack_rpc/channel.c     | 247 ++++++++++++++++---------------------
 src/nvim/msgpack_rpc/helpers.c     | 245 ------------------------------------
 src/nvim/msgpack_rpc/helpers.h     |  17 ---
 src/nvim/msgpack_rpc/packer.c      | 235 +++++++++++++++++++++++++++++++++++
 src/nvim/msgpack_rpc/packer.h      |  76 ++++++++++++
 src/nvim/msgpack_rpc/packer_defs.h |  24 ++++
 src/nvim/msgpack_rpc/unpacker.c    |   1 -
 7 files changed, 439 insertions(+), 406 deletions(-)
 delete mode 100644 src/nvim/msgpack_rpc/helpers.c
 delete mode 100644 src/nvim/msgpack_rpc/helpers.h
 create mode 100644 src/nvim/msgpack_rpc/packer.c
 create mode 100644 src/nvim/msgpack_rpc/packer.h
 create mode 100644 src/nvim/msgpack_rpc/packer_defs.h

(limited to 'src/nvim/msgpack_rpc')

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index ee7359c476..a8fde5a652 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -1,7 +1,6 @@
 #include <assert.h>
 #include <inttypes.h>
 #include <msgpack/object.h>
-#include <msgpack/pack.h>
 #include <msgpack/sbuffer.h>
 #include <msgpack/unpack.h>
 #include <stdbool.h>
@@ -29,7 +28,7 @@
 #include "nvim/message.h"
 #include "nvim/msgpack_rpc/channel.h"
 #include "nvim/msgpack_rpc/channel_defs.h"
-#include "nvim/msgpack_rpc/helpers.h"
+#include "nvim/msgpack_rpc/packer.h"
 #include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/os/input.h"
 #include "nvim/rbuffer.h"
@@ -44,73 +43,31 @@
 # define NOT "[notify]   "
 # define ERR "[error]    "
 
-// Cannot define array with negative offsets, so this one is needed to be added
-// to MSGPACK_UNPACK_\* values.
-# define MUR_OFF 2
+# define SEND "->"
+# define RECV "<-"
 
-static const char *const msgpack_error_messages[] = {
-  [MSGPACK_UNPACK_EXTRA_BYTES + MUR_OFF] = "extra bytes found",
-  [MSGPACK_UNPACK_CONTINUE + MUR_OFF] = "incomplete string",
-  [MSGPACK_UNPACK_PARSE_ERROR + MUR_OFF] = "parse error",
-  [MSGPACK_UNPACK_NOMEM_ERROR + MUR_OFF] = "not enough memory",
-};
-
-static void log_close(FILE *f)
+static void log_request(char *dir, uint64_t channel_id, uint32_t req_id, const char *name)
 {
-  fputc('\n', f);
-  fflush(f);
-  fclose(f);
-  log_unlock();
+  DLOGN("RPC %s %" PRIu64 ": %s id=%u: %s\n", dir, channel_id, REQ, req_id, name);
 }
 
-static void log_server_msg(uint64_t channel_id, msgpack_sbuffer *packed)
+static void log_response(char *dir, uint64_t channel_id, char *kind, uint32_t req_id)
 {
-  msgpack_unpacked unpacked;
-  msgpack_unpacked_init(&unpacked);
-  DLOGN("RPC ->ch %" PRIu64 ": ", channel_id);
-  const msgpack_unpack_return result =
-    msgpack_unpack_next(&unpacked, packed->data, packed->size, NULL);
-  switch (result) {
-  case MSGPACK_UNPACK_SUCCESS: {
-    uint64_t type = unpacked.data.via.array.ptr[0].via.u64;
-    log_lock();
-    FILE *f = open_log_file();
-    fprintf(f, type ? (type == 1 ? RES : NOT) : REQ);
-    msgpack_object_print(f, unpacked.data);
-    log_close(f);
-    msgpack_unpacked_destroy(&unpacked);
-    break;
-  }
-  case MSGPACK_UNPACK_EXTRA_BYTES:
-  case MSGPACK_UNPACK_CONTINUE:
-  case MSGPACK_UNPACK_PARSE_ERROR:
-  case MSGPACK_UNPACK_NOMEM_ERROR: {
-    log_lock();
-    FILE *f = open_log_file();
-    fprintf(f, ERR);
-    fprintf(f, "%s", msgpack_error_messages[result + MUR_OFF]);
-    log_close(f);
-    break;
-  }
-  }
+  DLOGN("RPC %s %" PRIu64 ": %s id=%u\n", dir, channel_id, kind, req_id);
 }
 
-static void log_client_msg(uint64_t channel_id, bool is_request, const char *name)
+static void log_notify(char *dir, uint64_t channel_id, const char *name)
 {
-  DLOGN("RPC <-ch %" PRIu64 ": ", channel_id);
-  log_lock();
-  FILE *f = open_log_file();
-  fprintf(f, "%s: %s", is_request ? REQ : RES, name);
-  log_close(f);
+  DLOGN("RPC %s %" PRIu64 ": %s %s\n", dir, channel_id, NOT, name);
 }
 
 #else
-# define log_client_msg(...)
-# define log_server_msg(...)
+# define log_request(...)
+# define log_response(...)
+# define log_notify(...)
 #endif
 
 static Set(cstr_t) event_strings = SET_INIT;
-static msgpack_sbuffer out_buffer;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "msgpack_rpc/channel.c.generated.h"
@@ -119,7 +76,6 @@ static msgpack_sbuffer out_buffer;
 void rpc_init(void)
 {
   ch_before_blocking_events = multiqueue_new_child(main_loop.events);
-  msgpack_sbuffer_init(&out_buffer);
 }
 
 void rpc_start(Channel *channel)
@@ -169,8 +125,9 @@ bool rpc_send_event(uint64_t id, const char *name, Array args)
     return false;
   }
 
+  log_notify(SEND, channel ? channel->id : 0, name);
   if (channel) {
-    send_event(channel, name, args);
+    serialize_request(&channel, 1, 0, name, args);
   } else {
     broadcast_event(name, args);
   }
@@ -199,7 +156,9 @@ Object rpc_send_call(uint64_t id, const char *method_name, Array args, ArenaMem
   RpcState *rpc = &channel->rpc;
   uint32_t request_id = rpc->next_request_id++;
   // Send the msgpack-rpc request
-  send_request(channel, request_id, method_name, args);
+  serialize_request(&channel, 1, request_id, method_name, args);
+
+  log_request(SEND, channel->id, request_id, method_name);
 
   // Push the frame
   ChannelCallFrame frame = { request_id, false, false, NIL, NULL };
@@ -361,8 +320,13 @@ static void parse_msgpack(Channel *channel)
         frame->result = p->result;
       }
       frame->result_mem = arena_finish(&p->arena);
+      log_response(RECV, channel->id, frame->errored ? ERR : RES, p->request_id);
     } else {
-      log_client_msg(channel->id, p->type == kMessageTypeRequest, p->handler.name);
+      if (p->type == kMessageTypeNotification) {
+        log_notify(RECV, channel->id, p->handler.name);
+      } else {
+        log_request(RECV, channel->id, p->request_id, p->handler.name);
+      }
 
       Object res = p->result;
       if (p->result.type != kObjectTypeArray) {
@@ -442,15 +406,7 @@ static void request_event(void **argv)
   Object result = handler.fn(channel->id, e->args, &e->used_mem, &error);
   if (e->type == kMessageTypeRequest || ERROR_SET(&error)) {
     // Send the response.
-    msgpack_packer response;
-    msgpack_packer_init(&response, &out_buffer, msgpack_sbuffer_write);
-    channel_write(channel, serialize_response(channel->id,
-                                              e->handler,
-                                              e->type,
-                                              e->request_id,
-                                              &error,
-                                              &result,
-                                              &out_buffer));
+    serialize_response(channel, e->handler, e->type, e->request_id, &error, &result);
   }
   if (handler.ret_alloc) {
     api_free_object(result);
@@ -533,41 +489,14 @@ static void send_error(Channel *chan, MsgpackRpcRequestHandler handler, MessageT
 {
   Error e = ERROR_INIT;
   api_set_error(&e, kErrorTypeException, "%s", err);
-  channel_write(chan, serialize_response(chan->id,
-                                         handler,
-                                         type,
-                                         id,
-                                         &e,
-                                         &NIL,
-                                         &out_buffer));
+  serialize_response(chan, handler, type, id, &e, &NIL);
   api_clear_error(&e);
 }
 
-static void send_request(Channel *channel, uint32_t id, const char *name, Array args)
-{
-  const String method = cstr_as_string(name);
-  channel_write(channel, serialize_request(channel->id,
-                                           id,
-                                           method,
-                                           args,
-                                           &out_buffer,
-                                           1));
-}
-
-static void send_event(Channel *channel, const char *name, Array args)
-{
-  const String method = cstr_as_string(name);
-  channel_write(channel, serialize_request(channel->id,
-                                           0,
-                                           method,
-                                           args,
-                                           &out_buffer,
-                                           1));
-}
-
 static void broadcast_event(const char *name, Array args)
 {
-  kvec_t(Channel *) subscribed = KV_INITIAL_VALUE;
+  kvec_withinit_t(Channel *, 4) subscribed = KV_INITIAL_VALUE;
+  kvi_init(subscribed);
   Channel *channel;
 
   map_foreach_value(&channels, channel, {
@@ -577,25 +506,11 @@ static void broadcast_event(const char *name, Array args)
     }
   });
 
-  if (!kv_size(subscribed)) {
-    goto end;
-  }
-
-  const String method = cstr_as_string(name);
-  WBuffer *buffer = serialize_request(0,
-                                      0,
-                                      method,
-                                      args,
-                                      &out_buffer,
-                                      kv_size(subscribed));
-
-  for (size_t i = 0; i < kv_size(subscribed); i++) {
-    Channel *c = kv_A(subscribed, i);
-    channel_write(c, buffer);
+  if (kv_size(subscribed)) {
+    serialize_request(subscribed.items, kv_size(subscribed), 0, name, args);
   }
 
-end:
-  kv_destroy(subscribed);
+  kvi_destroy(subscribed);
 }
 
 static void unsubscribe(Channel *channel, char *event)
@@ -653,27 +568,28 @@ static void chan_close_with_error(Channel *channel, char *msg, int loglevel)
   channel_close(channel->id, kChannelPartRpc, NULL);
 }
 
-static WBuffer *serialize_request(uint64_t channel_id, uint32_t request_id, const String method,
-                                  Array args, msgpack_sbuffer *sbuffer, size_t refcount)
+static void serialize_request(Channel **chans, size_t nchans, uint32_t request_id,
+                              const char *method, Array args)
 {
-  msgpack_packer pac;
-  msgpack_packer_init(&pac, sbuffer, msgpack_sbuffer_write);
-  msgpack_rpc_serialize_request(request_id, method, args, &pac);
-  log_server_msg(channel_id, sbuffer);
-  WBuffer *rv = wstream_new_buffer(xmemdup(sbuffer->data, sbuffer->size),
-                                   sbuffer->size,
-                                   refcount,
-                                   xfree);
-  msgpack_sbuffer_clear(sbuffer);
-  return rv;
+  PackerBuffer packer;
+  packer_buffer_init_channels(chans, nchans, &packer);
+
+  mpack_array(&packer.ptr, request_id ? 4 : 3);
+  mpack_w(&packer.ptr, request_id ? 0 : 2);
+
+  if (request_id) {
+    mpack_uint(&packer.ptr, request_id);
+  }
+
+  mpack_str(cstr_as_string(method), &packer);
+  mpack_object_array(args, &packer);
+
+  packer_buffer_finish_channels(&packer);
 }
 
-static WBuffer *serialize_response(uint64_t channel_id, MsgpackRpcRequestHandler handler,
-                                   MessageType type, uint32_t response_id, Error *err, Object *arg,
-                                   msgpack_sbuffer *sbuffer)
+void serialize_response(Channel *channel, MsgpackRpcRequestHandler handler, MessageType type,
+                        uint32_t response_id, Error *err, Object *arg)
 {
-  msgpack_packer pac;
-  msgpack_packer_init(&pac, sbuffer, msgpack_sbuffer_write);
   if (ERROR_SET(err) && type == kMessageTypeNotification) {
     if (handler.fn == handle_nvim_paste) {
       // TODO(bfredl): this is pretty much ad-hoc. maybe TUI and UI:s should be
@@ -685,19 +601,65 @@ static WBuffer *serialize_response(uint64_t channel_id, MsgpackRpcRequestHandler
       MAXSIZE_TEMP_ARRAY(args, 2);
       ADD_C(args, INTEGER_OBJ(err->type));
       ADD_C(args, CSTR_AS_OBJ(err->msg));
-      msgpack_rpc_serialize_request(0, cstr_as_string("nvim_error_event"),
-                                    args, &pac);
+      serialize_request(&channel, 1, 0, "nvim_error_event", args);
     }
+    return;
+  }
+
+  PackerBuffer packer;
+  packer_buffer_init_channels(&channel, 1, &packer);
+
+  mpack_array(&packer.ptr, 4);
+  mpack_w(&packer.ptr, 1);
+  mpack_uint(&packer.ptr, response_id);
+
+  if (ERROR_SET(err)) {
+    // error represented by a [type, message] array
+    mpack_array(&packer.ptr, 2);
+    mpack_integer(&packer.ptr, err->type);
+    mpack_str(cstr_as_string(err->msg), &packer);
+    // Nil result
+    mpack_nil(&packer.ptr);
   } else {
-    msgpack_rpc_serialize_response(response_id, err, arg, &pac);
+    // Nil error
+    mpack_nil(&packer.ptr);
+    // Return value
+    mpack_object(arg, &packer);
   }
-  log_server_msg(channel_id, sbuffer);
-  WBuffer *rv = wstream_new_buffer(xmemdup(sbuffer->data, sbuffer->size),
-                                   sbuffer->size,
-                                   1,  // responses only go though 1 channel
-                                   xfree);
-  msgpack_sbuffer_clear(sbuffer);
-  return rv;
+
+  packer_buffer_finish_channels(&packer);
+
+  log_response(SEND, channel->id, ERROR_SET(err) ? ERR : RES, response_id);
+}
+
+static void packer_buffer_init_channels(Channel **chans, size_t nchans, PackerBuffer *packer)
+{
+  packer->startptr = alloc_block();
+  packer->ptr = packer->startptr;
+  packer->endptr = packer->startptr + ARENA_BLOCK_SIZE;
+  packer->packer_flush = channel_flush_callback;
+  packer->anydata = chans;
+  packer->anylen = nchans;
+}
+
+static void packer_buffer_finish_channels(PackerBuffer *packer)
+{
+  size_t len = (size_t)(packer->ptr - packer->startptr);
+  if (len > 0) {
+    WBuffer *buf = wstream_new_buffer(packer->startptr, len, packer->anylen, free_block);
+    Channel **chans = packer->anydata;
+    for (size_t i = 0; i < packer->anylen; i++) {
+      channel_write(chans[i], buf);
+    }
+  } else {
+    free_block(packer->startptr);
+  }
+}
+
+static void channel_flush_callback(PackerBuffer *packer)
+{
+  packer_buffer_finish_channels(packer);
+  packer_buffer_init_channels(packer->anydata, packer->anylen, packer);
 }
 
 void rpc_set_client_info(uint64_t id, Dictionary info)
@@ -762,7 +724,6 @@ void rpc_free_all_mem(void)
   });
   set_destroy(cstr_t, &event_strings);
 
-  msgpack_sbuffer_destroy(&out_buffer);
   multiqueue_free(ch_before_blocking_events);
 }
 #endif
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
deleted file mode 100644
index bc1eefb270..0000000000
--- a/src/nvim/msgpack_rpc/helpers.c
+++ /dev/null
@@ -1,245 +0,0 @@
-#include <msgpack/object.h>
-#include <msgpack/sbuffer.h>
-#include <msgpack/unpack.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-
-#include "klib/kvec.h"
-#include "msgpack/pack.h"
-#include "nvim/api/private/helpers.h"
-#include "nvim/assert_defs.h"
-#include "nvim/lua/executor.h"
-#include "nvim/memory.h"
-#include "nvim/msgpack_rpc/helpers.h"
-#include "nvim/types_defs.h"
-
-#ifdef INCLUDE_GENERATED_DECLARATIONS
-# include "msgpack_rpc/helpers.c.generated.h"
-#endif
-
-static msgpack_sbuffer sbuffer;
-
-void msgpack_rpc_helpers_init(void)
-{
-  msgpack_sbuffer_init(&sbuffer);
-}
-
-#ifdef EXITFREE
-void msgpack_rpc_helpers_free_all_mem(void)
-{
-  msgpack_sbuffer_destroy(&sbuffer);
-}
-#endif
-
-// uncrustify:off
-
-void msgpack_rpc_from_boolean(Boolean result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  if (result) {
-    msgpack_pack_true(res);
-  } else {
-    msgpack_pack_false(res);
-  }
-}
-
-void msgpack_rpc_from_integer(Integer result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_int64(res, result);
-}
-
-void msgpack_rpc_from_float(Float result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_double(res, result);
-}
-
-void msgpack_rpc_from_string(const String result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_str(res, result.size);
-  if (result.size > 0) {
-    msgpack_pack_str_body(res, result.data, result.size);
-  }
-}
-
-static void msgpack_rpc_from_handle(ObjectType type, Integer o, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(3)
-{
-  msgpack_packer pac;
-  msgpack_packer_init(&pac, &sbuffer, msgpack_sbuffer_write);
-  msgpack_pack_int64(&pac, (handle_T)o);
-  msgpack_pack_ext(res, sbuffer.size, (int8_t)(type - EXT_OBJECT_TYPE_SHIFT));
-  msgpack_pack_ext_body(res, sbuffer.data, sbuffer.size);
-  msgpack_sbuffer_clear(&sbuffer);
-}
-
-typedef struct {
-  Object *aobj;
-  bool container;
-  size_t idx;
-} APIToMPObjectStackItem;
-
-/// Convert type used by Nvim API to msgpack type.
-///
-/// consumes (frees) any luaref inside `result`, even though they are not used
-/// (just represented as NIL)
-///
-/// @param[in]  result  Object to convert.
-/// @param[out]  res  Structure that defines where conversion results are saved.
-///
-/// @return true in case of success, false otherwise.
-void msgpack_rpc_from_object(Object *result, msgpack_packer *const res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  kvec_withinit_t(APIToMPObjectStackItem, 2) stack = KV_INITIAL_VALUE;
-  kvi_init(stack);
-  kvi_push(stack, ((APIToMPObjectStackItem) { result, false, 0 }));
-  while (kv_size(stack)) {
-    APIToMPObjectStackItem cur = kv_last(stack);
-    STATIC_ASSERT(kObjectTypeWindow == kObjectTypeBuffer + 1
-                  && kObjectTypeTabpage == kObjectTypeWindow + 1,
-                  "Buffer, window and tabpage enum items are in order");
-    switch (cur.aobj->type) {
-    case kObjectTypeLuaRef:
-      // TODO(bfredl): could also be an error. Though kObjectTypeLuaRef
-      // should only appear when the caller has opted in to handle references,
-      // see nlua_pop_Object.
-      api_free_luaref(cur.aobj->data.luaref);
-      cur.aobj->data.luaref = LUA_NOREF;
-      FALLTHROUGH;
-    case kObjectTypeNil:
-      msgpack_pack_nil(res);
-      break;
-    case kObjectTypeBoolean:
-      msgpack_rpc_from_boolean(cur.aobj->data.boolean, res);
-      break;
-    case kObjectTypeInteger:
-      msgpack_rpc_from_integer(cur.aobj->data.integer, res);
-      break;
-    case kObjectTypeFloat:
-      msgpack_rpc_from_float(cur.aobj->data.floating, res);
-      break;
-    case kObjectTypeString:
-      msgpack_rpc_from_string(cur.aobj->data.string, res);
-      break;
-    case kObjectTypeBuffer:
-    case kObjectTypeWindow:
-    case kObjectTypeTabpage:
-      msgpack_rpc_from_handle(cur.aobj->type, cur.aobj->data.integer, res);
-      break;
-    case kObjectTypeArray: {
-      const size_t size = cur.aobj->data.array.size;
-      if (cur.container) {
-        if (cur.idx >= size) {
-          (void)kv_pop(stack);
-        } else {
-          const size_t idx = cur.idx;
-          cur.idx++;
-          kv_last(stack) = cur;
-          kvi_push(stack, ((APIToMPObjectStackItem) {
-              .aobj = &cur.aobj->data.array.items[idx],
-              .container = false,
-          }));
-        }
-      } else {
-        msgpack_pack_array(res, size);
-        cur.container = true;
-        kv_last(stack) = cur;
-      }
-      break;
-    }
-    case kObjectTypeDictionary: {
-      const size_t size = cur.aobj->data.dictionary.size;
-      if (cur.container) {
-        if (cur.idx >= size) {
-          (void)kv_pop(stack);
-        } else {
-          const size_t idx = cur.idx;
-          cur.idx++;
-          kv_last(stack) = cur;
-          msgpack_rpc_from_string(cur.aobj->data.dictionary.items[idx].key, res);
-          kvi_push(stack, ((APIToMPObjectStackItem) {
-              .aobj = &cur.aobj->data.dictionary.items[idx].value,
-              .container = false,
-          }));
-        }
-      } else {
-        msgpack_pack_map(res, size);
-        cur.container = true;
-        kv_last(stack) = cur;
-      }
-      break;
-    }
-    }
-    if (!cur.container) {
-      (void)kv_pop(stack);
-    }
-  }
-  kvi_destroy(stack);
-}
-
-// uncrustify:on
-
-void msgpack_rpc_from_array(Array result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_array(res, result.size);
-
-  for (size_t i = 0; i < result.size; i++) {
-    msgpack_rpc_from_object(&result.items[i], res);
-  }
-}
-
-void msgpack_rpc_from_dictionary(Dictionary result, msgpack_packer *res)
-  FUNC_ATTR_NONNULL_ARG(2)
-{
-  msgpack_pack_map(res, result.size);
-
-  for (size_t i = 0; i < result.size; i++) {
-    msgpack_rpc_from_string(result.items[i].key, res);
-    msgpack_rpc_from_object(&result.items[i].value, res);
-  }
-}
-
-/// Serializes a msgpack-rpc request or notification(id == 0)
-void msgpack_rpc_serialize_request(uint32_t request_id, const String method, Array args,
-                                   msgpack_packer *pac)
-  FUNC_ATTR_NONNULL_ARG(4)
-{
-  msgpack_pack_array(pac, request_id ? 4 : 3);
-  msgpack_pack_int(pac, request_id ? 0 : 2);
-
-  if (request_id) {
-    msgpack_pack_uint32(pac, request_id);
-  }
-
-  msgpack_rpc_from_string(method, pac);
-  msgpack_rpc_from_array(args, pac);
-}
-
-/// Serializes a msgpack-rpc response
-void msgpack_rpc_serialize_response(uint32_t response_id, Error *err, Object *arg,
-                                    msgpack_packer *pac)
-  FUNC_ATTR_NONNULL_ALL
-{
-  msgpack_pack_array(pac, 4);
-  msgpack_pack_int(pac, 1);
-  msgpack_pack_uint32(pac, response_id);
-
-  if (ERROR_SET(err)) {
-    // error represented by a [type, message] array
-    msgpack_pack_array(pac, 2);
-    msgpack_rpc_from_integer(err->type, pac);
-    msgpack_rpc_from_string(cstr_as_string(err->msg), pac);
-    // Nil result
-    msgpack_pack_nil(pac);
-  } else {
-    // Nil error
-    msgpack_pack_nil(pac);
-    // Return value
-    msgpack_rpc_from_object(arg, pac);
-  }
-}
diff --git a/src/nvim/msgpack_rpc/helpers.h b/src/nvim/msgpack_rpc/helpers.h
deleted file mode 100644
index 6344d8c567..0000000000
--- a/src/nvim/msgpack_rpc/helpers.h
+++ /dev/null
@@ -1,17 +0,0 @@
-#pragma once
-
-#include <msgpack.h>  // IWYU pragma: keep
-
-#include "nvim/api/private/defs.h"
-
-/// Value by which objects represented as EXT type are shifted
-///
-/// Subtracted when packing, added when unpacking. Used to allow moving
-/// buffer/window/tabpage block inside ObjectType enum. This block yet cannot be
-/// split or reordered.
-#define EXT_OBJECT_TYPE_SHIFT kObjectTypeBuffer
-#define EXT_OBJECT_TYPE_MAX (kObjectTypeTabpage - EXT_OBJECT_TYPE_SHIFT)
-
-#ifdef INCLUDE_GENERATED_DECLARATIONS
-# include "msgpack_rpc/helpers.h.generated.h"
-#endif
diff --git a/src/nvim/msgpack_rpc/packer.c b/src/nvim/msgpack_rpc/packer.c
new file mode 100644
index 0000000000..cac68f76f0
--- /dev/null
+++ b/src/nvim/msgpack_rpc/packer.c
@@ -0,0 +1,235 @@
+#include <assert.h>
+
+#include "nvim/api/private/defs.h"
+#include "nvim/lua/executor.h"
+#include "nvim/msgpack_rpc/packer.h"
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "msgpack_rpc/packer.c.generated.h"
+#endif
+
+static void check_buffer(PackerBuffer *packer)
+{
+  ptrdiff_t remaining = packer->endptr - packer->ptr;
+  if (remaining < MPACK_ITEM_SIZE) {
+    packer->packer_flush(packer);
+  }
+}
+
+static void mpack_w8(char **b, const char *data)
+{
+#ifdef ORDER_BIG_ENDIAN
+  memcpy(*b, data, 8);
+  *b += 8;
+#else
+  for (int i = 7; i >= 0; i--) {
+    *(*b)++ = data[i];
+  }
+#endif
+}
+
+void mpack_integer(char **ptr, Integer i)
+{
+  if (i >= 0) {
+    if (i > 0xfffffff) {
+      mpack_w(ptr, 0xcf);
+      mpack_w8(ptr, (char *)&i);
+    } else {
+      mpack_uint(ptr, (uint32_t)i);
+    }
+  } else {
+    if (i < -0x80000000LL) {
+      mpack_w(ptr, 0xd3);
+      mpack_w8(ptr, (char *)&i);
+    } else if (i < -0x8000) {
+      mpack_w(ptr, 0xd2);
+      mpack_w4(ptr, (uint32_t)i);
+    } else if (i < -0x80) {
+      mpack_w(ptr, 0xd1);
+      mpack_w2(ptr, (uint32_t)i);
+    } else if (i < -0x20) {
+      mpack_w(ptr, 0xd0);
+      mpack_w(ptr, (char)i);
+    } else {
+      mpack_w(ptr, (char)i);
+    }
+  }
+}
+
+void mpack_float8(char **ptr, double i)
+{
+  mpack_w(ptr, 0xcb);
+  mpack_w8(ptr, (char *)&i);
+}
+
+void mpack_str(String str, PackerBuffer *packer)
+{
+  const size_t len = str.size;
+  if (len < 20) {
+    mpack_w(&packer->ptr, 0xa0 | len);
+  } else if (len < 0xff) {
+    mpack_w(&packer->ptr, 0xd9);
+    mpack_w(&packer->ptr, len);
+  } else if (len < 0xffff) {
+    mpack_w(&packer->ptr, 0xda);
+    mpack_w2(&packer->ptr, (uint32_t)len);
+  } else if (len < 0xffffffff) {
+    mpack_w(&packer->ptr, 0xdb);
+    mpack_w4(&packer->ptr, (uint32_t)len);
+  } else {
+    abort();
+  }
+
+  size_t pos = 0;
+  while (pos < len) {
+    ptrdiff_t remaining = packer->endptr - packer->ptr;
+    size_t to_copy = MIN(len - pos, (size_t)remaining);
+    memcpy(packer->ptr, str.data + pos, to_copy);
+    packer->ptr += to_copy;
+    pos += to_copy;
+
+    if (pos < len) {
+      packer->packer_flush(packer);
+    }
+  }
+}
+
+void mpack_handle(ObjectType type, handle_T handle, PackerBuffer *packer)
+{
+  char exttype = (char)(type - EXT_OBJECT_TYPE_SHIFT);
+  if (-0x1f <= handle && handle <= 0x7f) {
+    mpack_w(&packer->ptr, 0xd4);
+    mpack_w(&packer->ptr, exttype);
+    mpack_w(&packer->ptr, (char)handle);
+  } else {
+    // we want to encode some small negative sentinel like -1. This is handled above
+    assert(handle >= 0);
+    // FAIL: we cannot use fixext 4/8 due to a design error
+    // (in theory fixext 2 for handle<=0xff but we don't gain much from it)
+    char buf[MPACK_ITEM_SIZE];
+    char *pos = buf;
+    mpack_uint(&pos, (uint32_t)handle);
+    ptrdiff_t packsize = pos - buf;
+    mpack_w(&packer->ptr, 0xc7);
+    mpack_w(&packer->ptr, packsize);
+    mpack_w(&packer->ptr, exttype);
+    // check_buffer(packer);
+    memcpy(packer->ptr, buf, (size_t)packsize);
+    packer->ptr += packsize;
+  }
+}
+
+void mpack_object(Object *obj, PackerBuffer *packer)
+{
+  mpack_object_inner(obj, NULL, 0, packer);
+}
+
+void mpack_object_array(Array arr, PackerBuffer *packer)
+{
+  mpack_array(&packer->ptr, (uint32_t)arr.size);
+  if (arr.size > 0) {
+    Object container = ARRAY_OBJ(arr);
+    mpack_object_inner(&arr.items[0], arr.size > 1 ? &container : NULL, 1, packer);
+  }
+}
+
+typedef struct {
+  Object *container;
+  size_t idx;
+} ContainerStackItem;
+
+void mpack_object_inner(Object *current, Object *container, size_t container_idx,
+                        PackerBuffer *packer)
+  FUNC_ATTR_NONNULL_ARG(1, 4)
+{
+  // The inner loop of this function packs "current" and then fetches the next
+  // value from "container". "stack" is only used for nested containers.
+  kvec_withinit_t(ContainerStackItem, 2) stack = KV_INITIAL_VALUE;
+  kvi_init(stack);
+
+  while (true) {
+    check_buffer(packer);
+    switch (current->type) {
+    case kObjectTypeLuaRef:
+      // TODO(bfredl): could also be an error. Though kObjectTypeLuaRef
+      // should only appear when the caller has opted in to handle references,
+      // see nlua_pop_Object.
+      api_free_luaref(current->data.luaref);
+      current->data.luaref = LUA_NOREF;
+      FALLTHROUGH;
+    case kObjectTypeNil:
+      mpack_nil(&packer->ptr);
+      break;
+    case kObjectTypeBoolean:
+      mpack_bool(&packer->ptr, current->data.boolean);
+      break;
+    case kObjectTypeInteger:
+      mpack_integer(&packer->ptr, current->data.integer);
+      break;
+    case kObjectTypeFloat:
+      mpack_float8(&packer->ptr, current->data.floating);
+      break;
+    case kObjectTypeString:
+      mpack_str(current->data.string, packer);
+      break;
+    case kObjectTypeBuffer:
+    case kObjectTypeWindow:
+    case kObjectTypeTabpage:
+      mpack_handle(current->type, (handle_T)current->data.integer, packer);
+      break;
+    case kObjectTypeDictionary:
+    case kObjectTypeArray: {}
+      size_t current_size;
+      if (current->type == kObjectTypeArray) {
+        current_size = current->data.array.size;
+        mpack_array(&packer->ptr, (uint32_t)current_size);
+      } else {
+        current_size = current->data.dictionary.size;
+        mpack_map(&packer->ptr, (uint32_t)current_size);
+      }
+      if (current_size > 0) {
+        if (current->type == kObjectTypeArray && current_size == 1) {
+          current = &current->data.array.items[0];
+          continue;
+        }
+        if (container) {
+          kvi_push(stack, ((ContainerStackItem) {
+            .container = container,
+            .idx = container_idx,
+          }));
+        }
+        container = current;
+        container_idx = 0;
+      }
+      break;
+    }
+
+    if (!container) {
+      if (kv_size(stack)) {
+        ContainerStackItem it = kv_pop(stack);
+        container = it.container;
+        container_idx = it.idx;
+      } else {
+        break;
+      }
+    }
+
+    if (container->type == kObjectTypeArray) {
+      Array arr = container->data.array;
+      current = &arr.items[container_idx++];
+      if (container_idx >= arr.size) {
+        container = NULL;
+      }
+    } else {
+      Dictionary dict = container->data.dictionary;
+      KeyValuePair *it = &dict.items[container_idx++];
+      check_buffer(packer);
+      mpack_str(it->key, packer);
+      current = &it->value;
+      if (container_idx >= dict.size) {
+        container = NULL;
+      }
+    }
+  }
+  kvi_destroy(stack);
+}
diff --git a/src/nvim/msgpack_rpc/packer.h b/src/nvim/msgpack_rpc/packer.h
new file mode 100644
index 0000000000..8117bd09bd
--- /dev/null
+++ b/src/nvim/msgpack_rpc/packer.h
@@ -0,0 +1,76 @@
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "nvim/api/private/defs.h"
+#include "nvim/msgpack_rpc/packer_defs.h"
+
+#define mpack_w(b, byte) *(*(b))++ = (char)(byte);
+static inline void mpack_w2(char **b, uint32_t v)
+{
+  *(*b)++ = (char)((v >> 8) & 0xff);
+  *(*b)++ = (char)(v & 0xff);
+}
+
+static inline void mpack_w4(char **b, uint32_t v)
+{
+  *(*b)++ = (char)((v >> 24) & 0xff);
+  *(*b)++ = (char)((v >> 16) & 0xff);
+  *(*b)++ = (char)((v >> 8) & 0xff);
+  *(*b)++ = (char)(v & 0xff);
+}
+
+static inline void mpack_uint(char **buf, uint32_t val)
+{
+  if (val > 0xffff) {
+    mpack_w(buf, 0xce);
+    mpack_w4(buf, val);
+  } else if (val > 0xff) {
+    mpack_w(buf, 0xcd);
+    mpack_w2(buf, val);
+  } else if (val > 0x7f) {
+    mpack_w(buf, 0xcc);
+    mpack_w(buf, val);
+  } else {
+    mpack_w(buf, val);
+  }
+}
+
+#define mpack_nil(buf) mpack_w(buf, 0xc0)
+static inline void mpack_bool(char **buf, bool val)
+{
+  mpack_w(buf, 0xc2 | (val ? 1 : 0));
+}
+
+static inline void mpack_array(char **buf, uint32_t len)
+{
+  if (len < 0x10) {
+    mpack_w(buf, 0x90 | len);
+  } else if (len < 0x10000) {
+    mpack_w(buf, 0xdc);
+    mpack_w2(buf, len);
+  } else {
+    mpack_w(buf, 0xdd);
+    mpack_w4(buf, len);
+  }
+}
+
+static inline void mpack_map(char **buf, uint32_t len)
+{
+  if (len < 0x10) {
+    mpack_w(buf, 0x80 | len);
+  } else if (len < 0x10000) {
+    mpack_w(buf, 0xde);
+    mpack_w2(buf, len);
+  } else {
+    mpack_w(buf, 0xdf);
+    mpack_w4(buf, len);
+  }
+}
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "msgpack_rpc/packer.h.generated.h"
+#endif
diff --git a/src/nvim/msgpack_rpc/packer_defs.h b/src/nvim/msgpack_rpc/packer_defs.h
new file mode 100644
index 0000000000..420f3dc424
--- /dev/null
+++ b/src/nvim/msgpack_rpc/packer_defs.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+// Max possible length: bytecode + 8 int/float bytes
+// Ext objects are maximum 8=3+5 (nested uint32 payload)
+#define MPACK_ITEM_SIZE 9
+
+typedef struct packer_buffer_t PackerBuffer;
+
+// Must ensure at least MPACK_ITEM_SIZE of space.
+typedef void (*PackerBufferFlush)(PackerBuffer *self);
+
+struct packer_buffer_t {
+  char *startptr;
+  char *ptr;
+  char *endptr;
+
+  // these are free to be used by packer_flush for any purpose, if want
+  void *anydata;
+  size_t anylen;
+  PackerBufferFlush packer_flush;
+};
diff --git a/src/nvim/msgpack_rpc/unpacker.c b/src/nvim/msgpack_rpc/unpacker.c
index 6bc138f65a..dbb30b0c9a 100644
--- a/src/nvim/msgpack_rpc/unpacker.c
+++ b/src/nvim/msgpack_rpc/unpacker.c
@@ -10,7 +10,6 @@
 #include "nvim/macros_defs.h"
 #include "nvim/memory.h"
 #include "nvim/msgpack_rpc/channel_defs.h"
-#include "nvim/msgpack_rpc/helpers.h"
 #include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/ui_client.h"
 
-- 
cgit