refactor(msgpack): allow flushing buffer while packing msgpack

Before, we needed to always pack an entire msgpack_rpc Object to
a continous memory buffer before sending it out to a channel.
But this is generally wasteful. it is better to just flush
whatever is in the buffer and then continue packing to a new buffer.

This is also done for the UI event packer where there are some extra logic
to "finish" of an existing batch of nevents/ncalls. This doesn't really
stop us from flushing the buffer, just that we need to update the state
machine accordingly so the next call to prepare_call() always will
start with a new event (even though the buffer might contain overflow
data from a large event).

2 files changed, 9 insertions, 2 deletions

diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index 0cdc90e50f..ca088d7a55 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -105,6 +105,14 @@ typedef enum {
   kObjectTypeTabpage,
 } ObjectType;
 
+/// Value by which objects represented as EXT type are shifted
+///
+/// Subtracted when packing, added when unpacking. Used to allow moving
+/// buffer/window/tabpage block inside ObjectType enum. This block yet cannot be
+/// split or reordered.
+#define EXT_OBJECT_TYPE_SHIFT kObjectTypeBuffer
+#define EXT_OBJECT_TYPE_MAX (kObjectTypeTabpage - EXT_OBJECT_TYPE_SHIFT)
+
 struct object {
   ObjectType type;
   union {
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index 97ba15e871..1cd98aa0c4 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -29,7 +29,6 @@
 #include "nvim/memory.h"
 #include "nvim/memory_defs.h"
 #include "nvim/message.h"
-#include "nvim/msgpack_rpc/helpers.h"
 #include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/pos_defs.h"
 #include "nvim/types_defs.h"
@@ -984,7 +983,7 @@ Dictionary api_keydict_to_dict(void *value, KeySetLink *table, size_t max_size,
       val = DICTIONARY_OBJ(*(Dictionary *)mem);
     } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
                || field->type == kObjectTypeTabpage) {
-      val.data.integer = *(Integer *)mem;
+      val.data.integer = *(handle_T *)mem;
       val.type = field->type;
     } else if (field->type == kObjectTypeLuaRef) {
       // do nothing