Merge pull request #29241 from bfredl/shadapack

refactor(shada): use msgpack_sbuffer less

13 files changed, 174 insertions, 204 deletions

diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index a17e78cc31..ac621b1486 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -548,10 +548,6 @@ String buf_get_text(buf_T *buf, int64_t lnum, int64_t start_col, int64_t end_col
 
 void api_free_string(String value)
 {
-  if (!value.data) {
-    return;
-  }
-
   xfree(value.data);
 }
 
diff --git a/src/nvim/context.c b/src/nvim/context.c
index 95e2618f62..2e3634329a 100644
--- a/src/nvim/context.c
+++ b/src/nvim/context.c
@@ -66,21 +66,11 @@ Context *ctx_get(size_t index)
 void ctx_free(Context *ctx)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (ctx->regs.data) {
-    msgpack_sbuffer_destroy(&ctx->regs);
-  }
-  if (ctx->jumps.data) {
-    msgpack_sbuffer_destroy(&ctx->jumps);
-  }
-  if (ctx->bufs.data) {
-    msgpack_sbuffer_destroy(&ctx->bufs);
-  }
-  if (ctx->gvars.data) {
-    msgpack_sbuffer_destroy(&ctx->gvars);
-  }
-  if (ctx->funcs.items) {
-    api_free_array(ctx->funcs);
-  }
+  api_free_string(ctx->regs);
+  api_free_string(ctx->jumps);
+  api_free_string(ctx->bufs);
+  api_free_string(ctx->gvars);
+  api_free_array(ctx->funcs);
 }
 
 /// Saves the editor state to a context.
@@ -98,19 +88,19 @@ void ctx_save(Context *ctx, const int flags)
   }
 
   if (flags & kCtxRegs) {
-    ctx_save_regs(ctx);
+    ctx->regs = shada_encode_regs();
   }
 
   if (flags & kCtxJumps) {
-    ctx_save_jumps(ctx);
+    ctx->jumps = shada_encode_jumps();
   }
 
   if (flags & kCtxBufs) {
-    ctx_save_bufs(ctx);
+    ctx->bufs = shada_encode_buflist();
   }
 
   if (flags & kCtxGVars) {
-    ctx_save_gvars(ctx);
+    ctx->gvars = shada_encode_gvars();
   }
 
   if (flags & kCtxFuncs) {
@@ -173,33 +163,13 @@ bool ctx_restore(Context *ctx, const int flags)
   return true;
 }
 
-/// Saves the global registers to a context.
-///
-/// @param  ctx    Save to this context.
-static inline void ctx_save_regs(Context *ctx)
-  FUNC_ATTR_NONNULL_ALL
-{
-  msgpack_sbuffer_init(&ctx->regs);
-  shada_encode_regs(&ctx->regs);
-}
-
 /// Restores the global registers from a context.
 ///
 /// @param  ctx   Restore from this context.
 static inline void ctx_restore_regs(Context *ctx)
   FUNC_ATTR_NONNULL_ALL
 {
-  shada_read_sbuf(&ctx->regs, kShaDaWantInfo | kShaDaForceit);
-}
-
-/// Saves the jumplist to a context.
-///
-/// @param  ctx  Save to this context.
-static inline void ctx_save_jumps(Context *ctx)
-  FUNC_ATTR_NONNULL_ALL
-{
-  msgpack_sbuffer_init(&ctx->jumps);
-  shada_encode_jumps(&ctx->jumps);
+  shada_read_string(ctx->regs, kShaDaWantInfo | kShaDaForceit);
 }
 
 /// Restores the jumplist from a context.
@@ -208,17 +178,7 @@ static inline void ctx_save_jumps(Context *ctx)
 static inline void ctx_restore_jumps(Context *ctx)
   FUNC_ATTR_NONNULL_ALL
 {
-  shada_read_sbuf(&ctx->jumps, kShaDaWantInfo | kShaDaForceit);
-}
-
-/// Saves the buffer list to a context.
-///
-/// @param  ctx  Save to this context.
-static inline void ctx_save_bufs(Context *ctx)
-  FUNC_ATTR_NONNULL_ALL
-{
-  msgpack_sbuffer_init(&ctx->bufs);
-  shada_encode_buflist(&ctx->bufs);
+  shada_read_string(ctx->jumps, kShaDaWantInfo | kShaDaForceit);
 }
 
 /// Restores the buffer list from a context.
@@ -227,17 +187,7 @@ static inline void ctx_save_bufs(Context *ctx)
 static inline void ctx_restore_bufs(Context *ctx)
   FUNC_ATTR_NONNULL_ALL
 {
-  shada_read_sbuf(&ctx->bufs, kShaDaWantInfo | kShaDaForceit);
-}
-
-/// Saves global variables to a context.
-///
-/// @param  ctx  Save to this context.
-static inline void ctx_save_gvars(Context *ctx)
-  FUNC_ATTR_NONNULL_ALL
-{
-  msgpack_sbuffer_init(&ctx->gvars);
-  shada_encode_gvars(&ctx->gvars);
+  shada_read_string(ctx->bufs, kShaDaWantInfo | kShaDaForceit);
 }
 
 /// Restores global variables from a context.
@@ -246,7 +196,7 @@ static inline void ctx_save_gvars(Context *ctx)
 static inline void ctx_restore_gvars(Context *ctx)
   FUNC_ATTR_NONNULL_ALL
 {
-  shada_read_sbuf(&ctx->gvars, kShaDaWantInfo | kShaDaForceit);
+  shada_read_string(ctx->gvars, kShaDaWantInfo | kShaDaForceit);
 }
 
 /// Saves functions to a context.
@@ -291,41 +241,16 @@ static inline void ctx_restore_funcs(Context *ctx)
   }
 }
 
-/// Convert msgpack_sbuffer to readfile()-style array.
-///
-/// @param[in]  sbuf  msgpack_sbuffer to convert.
-///
-/// @return readfile()-style array representation of "sbuf".
-static inline Array sbuf_to_array(msgpack_sbuffer sbuf, Arena *arena)
-{
-  list_T *const list = tv_list_alloc(kListLenMayKnow);
-  tv_list_append_string(list, "", 0);
-  if (sbuf.size > 0) {
-    encode_list_write(list, sbuf.data, sbuf.size);
-  }
-
-  typval_T list_tv = (typval_T) {
-    .v_lock = VAR_UNLOCKED,
-    .v_type = VAR_LIST,
-    .vval.v_list = list
-  };
-
-  Array array = vim_to_object(&list_tv, arena, false).data.array;
-  tv_clear(&list_tv);
-  return array;
-}
-
-/// Convert readfile()-style array to msgpack_sbuffer.
+/// Convert readfile()-style array to String
 ///
 /// @param[in]  array  readfile()-style array to convert.
 /// @param[out]  err   Error object.
 ///
-/// @return msgpack_sbuffer with conversion result.
-static inline msgpack_sbuffer array_to_sbuf(Array array, Error *err)
+/// @return String with conversion result.
+static inline String array_to_string(Array array, Error *err)
   FUNC_ATTR_NONNULL_ALL
 {
-  msgpack_sbuffer sbuf;
-  msgpack_sbuffer_init(&sbuf);
+  String sbuf = STRING_INIT;
 
   typval_T list_tv;
   object_to_vim(ARRAY_OBJ(array), &list_tv, err);
@@ -335,7 +260,6 @@ static inline msgpack_sbuffer array_to_sbuf(Array array, Error *err)
     api_set_error(err, kErrorTypeException, "%s",
                   "E474: Failed to convert list to msgpack string buffer");
   }
-  sbuf.alloc = sbuf.size;
 
   tv_clear(&list_tv);
   return sbuf;
@@ -353,10 +277,10 @@ Dictionary ctx_to_dict(Context *ctx, Arena *arena)
 
   Dictionary rv = arena_dict(arena, 5);
 
-  PUT_C(rv, "regs", ARRAY_OBJ(sbuf_to_array(ctx->regs, arena)));
-  PUT_C(rv, "jumps", ARRAY_OBJ(sbuf_to_array(ctx->jumps, arena)));
-  PUT_C(rv, "bufs", ARRAY_OBJ(sbuf_to_array(ctx->bufs, arena)));
-  PUT_C(rv, "gvars", ARRAY_OBJ(sbuf_to_array(ctx->gvars, arena)));
+  PUT_C(rv, "regs", ARRAY_OBJ(string_to_array(ctx->regs, false, arena)));
+  PUT_C(rv, "jumps", ARRAY_OBJ(string_to_array(ctx->jumps, false, arena)));
+  PUT_C(rv, "bufs", ARRAY_OBJ(string_to_array(ctx->bufs, false, arena)));
+  PUT_C(rv, "gvars", ARRAY_OBJ(string_to_array(ctx->gvars, false, arena)));
   PUT_C(rv, "funcs", ARRAY_OBJ(copy_array(ctx->funcs, arena)));
 
   return rv;
@@ -382,16 +306,16 @@ int ctx_from_dict(Dictionary dict, Context *ctx, Error *err)
     }
     if (strequal(item.key.data, "regs")) {
       types |= kCtxRegs;
-      ctx->regs = array_to_sbuf(item.value.data.array, err);
+      ctx->regs = array_to_string(item.value.data.array, err);
     } else if (strequal(item.key.data, "jumps")) {
       types |= kCtxJumps;
-      ctx->jumps = array_to_sbuf(item.value.data.array, err);
+      ctx->jumps = array_to_string(item.value.data.array, err);
     } else if (strequal(item.key.data, "bufs")) {
       types |= kCtxBufs;
-      ctx->bufs = array_to_sbuf(item.value.data.array, err);
+      ctx->bufs = array_to_string(item.value.data.array, err);
     } else if (strequal(item.key.data, "gvars")) {
       types |= kCtxGVars;
-      ctx->gvars = array_to_sbuf(item.value.data.array, err);
+      ctx->gvars = array_to_string(item.value.data.array, err);
     } else if (strequal(item.key.data, "funcs")) {
       types |= kCtxFuncs;
       ctx->funcs = copy_object(item.value, NULL).data.array;
diff --git a/src/nvim/context.h b/src/nvim/context.h
index 1c18a1af7c..0bb7138b6c 100644
--- a/src/nvim/context.h
+++ b/src/nvim/context.h
@@ -7,25 +7,19 @@
 #include "nvim/api/private/defs.h"
 
 typedef struct {
-  msgpack_sbuffer regs;     ///< Registers.
-  msgpack_sbuffer jumps;    ///< Jumplist.
-  msgpack_sbuffer bufs;     ///< Buffer list.
-  msgpack_sbuffer gvars;    ///< Global variables.
+  String regs;     ///< Registers.
+  String jumps;    ///< Jumplist.
+  String bufs;     ///< Buffer list.
+  String gvars;    ///< Global variables.
   Array funcs;              ///< Functions.
 } Context;
 typedef kvec_t(Context) ContextVec;
 
-#define MSGPACK_SBUFFER_INIT (msgpack_sbuffer) { \
-  .size = 0, \
-  .data = NULL, \
-  .alloc = 0, \
-}
-
 #define CONTEXT_INIT (Context) { \
-  .regs = MSGPACK_SBUFFER_INIT, \
-  .jumps = MSGPACK_SBUFFER_INIT, \
-  .bufs = MSGPACK_SBUFFER_INIT, \
-  .gvars = MSGPACK_SBUFFER_INIT, \
+  .regs = STRING_INIT, \
+  .jumps = STRING_INIT, \
+  .bufs = STRING_INIT, \
+  .gvars = STRING_INIT, \
   .funcs = ARRAY_DICT_INIT, \
 }
 
diff --git a/src/nvim/memory.h b/src/nvim/memory.h
index 277ffe97aa..a6ff82e39d 100644
--- a/src/nvim/memory.h
+++ b/src/nvim/memory.h
@@ -45,8 +45,6 @@ EXTERN size_t arena_alloc_count INIT( = 0);
   ((v).capacity = (s), \
    (v).items = (void *)arena_alloc(a, sizeof((v).items[0]) * (v).capacity, true))
 
-#define ARENA_BLOCK_SIZE 4096
-
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "memory.h.generated.h"
 #endif
diff --git a/src/nvim/memory_defs.h b/src/nvim/memory_defs.h
index bde0e54f54..df271ceca5 100644
--- a/src/nvim/memory_defs.h
+++ b/src/nvim/memory_defs.h
@@ -11,5 +11,7 @@ typedef struct {
   size_t pos, size;
 } Arena;
 
+#define ARENA_BLOCK_SIZE 4096
+
 // inits an empty arena.
 #define ARENA_EMPTY { .cur_blk = NULL, .pos = 0, .size = 0 }
diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 6a0dc10214..0930f0c7b3 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -590,16 +590,16 @@ static void packer_buffer_init_channels(Channel **chans, size_t nchans, PackerBu
   packer->endptr = packer->startptr + ARENA_BLOCK_SIZE;
   packer->packer_flush = channel_flush_callback;
   packer->anydata = chans;
-  packer->anylen = nchans;
+  packer->anyint = (int64_t)nchans;
 }
 
 static void packer_buffer_finish_channels(PackerBuffer *packer)
 {
   size_t len = (size_t)(packer->ptr - packer->startptr);
   if (len > 0) {
-    WBuffer *buf = wstream_new_buffer(packer->startptr, len, packer->anylen, free_block);
+    WBuffer *buf = wstream_new_buffer(packer->startptr, len, (size_t)packer->anyint, free_block);
     Channel **chans = packer->anydata;
-    for (size_t i = 0; i < packer->anylen; i++) {
+    for (int64_t i = 0; i < packer->anyint; i++) {
       channel_write(chans[i], buf);
     }
   } else {
@@ -610,7 +610,7 @@ static void packer_buffer_finish_channels(PackerBuffer *packer)
 static void channel_flush_callback(PackerBuffer *packer)
 {
   packer_buffer_finish_channels(packer);
-  packer_buffer_init_channels(packer->anydata, packer->anylen, packer);
+  packer_buffer_init_channels(packer->anydata, (size_t)packer->anyint, packer);
 }
 
 void rpc_set_client_info(uint64_t id, Dictionary info)
diff --git a/src/nvim/msgpack_rpc/packer.c b/src/nvim/msgpack_rpc/packer.c
index 9c0d2910fa..58318b88b0 100644
--- a/src/nvim/msgpack_rpc/packer.c
+++ b/src/nvim/msgpack_rpc/packer.c
@@ -10,8 +10,7 @@
 
 static void check_buffer(PackerBuffer *packer)
 {
-  ptrdiff_t remaining = packer->endptr - packer->ptr;
-  if (remaining < MPACK_ITEM_SIZE) {
+  if (mpack_remaining(packer) < MPACK_ITEM_SIZE) {
     packer->packer_flush(packer);
   }
 }
@@ -28,15 +27,20 @@ static void mpack_w8(char **b, const char *data)
 #endif
 }
 
+void mpack_uint64(char **ptr, uint64_t i)
+{
+  if (i > 0xfffffff) {
+    mpack_w(ptr, 0xcf);
+    mpack_w8(ptr, (char *)&i);
+  } else {
+    mpack_uint(ptr, (uint32_t)i);
+  }
+}
+
 void mpack_integer(char **ptr, Integer i)
 {
   if (i >= 0) {
-    if (i > 0xfffffff) {
-      mpack_w(ptr, 0xcf);
-      mpack_w8(ptr, (char *)&i);
-    } else {
-      mpack_uint(ptr, (uint32_t)i);
-    }
+    mpack_uint64(ptr, (uint64_t)i);
   } else {
     if (i < -0x80000000LL) {
       mpack_w(ptr, 0xd3);
@@ -80,11 +84,16 @@ void mpack_str(String str, PackerBuffer *packer)
     abort();
   }
 
+  mpack_raw(str.data, len, packer);
+}
+
+void mpack_raw(char *data, size_t len, PackerBuffer *packer)
+{
   size_t pos = 0;
   while (pos < len) {
     ptrdiff_t remaining = packer->endptr - packer->ptr;
     size_t to_copy = MIN(len - pos, (size_t)remaining);
-    memcpy(packer->ptr, str.data + pos, to_copy);
+    memcpy(packer->ptr, data + pos, to_copy);
     packer->ptr += to_copy;
     pos += to_copy;
 
diff --git a/src/nvim/msgpack_rpc/packer.h b/src/nvim/msgpack_rpc/packer.h
index 8117bd09bd..299962bab4 100644
--- a/src/nvim/msgpack_rpc/packer.h
+++ b/src/nvim/msgpack_rpc/packer.h
@@ -71,6 +71,11 @@ static inline void mpack_map(char **buf, uint32_t len)
   }
 }
 
+static inline size_t mpack_remaining(PackerBuffer *packer)
+{
+  return (size_t)(packer->endptr - packer->ptr);
+}
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "msgpack_rpc/packer.h.generated.h"
 #endif
diff --git a/src/nvim/msgpack_rpc/packer_defs.h b/src/nvim/msgpack_rpc/packer_defs.h
index 420f3dc424..95d86caaab 100644
--- a/src/nvim/msgpack_rpc/packer_defs.h
+++ b/src/nvim/msgpack_rpc/packer_defs.h
@@ -19,6 +19,6 @@ struct packer_buffer_t {
 
   // these are free to be used by packer_flush for any purpose, if want
   void *anydata;
-  size_t anylen;
+  int64_t anyint;
   PackerBufferFlush packer_flush;
 };
diff --git a/src/nvim/os/fileio.c b/src/nvim/os/fileio.c
index 585c4964e2..89834bed80 100644
--- a/src/nvim/os/fileio.c
+++ b/src/nvim/os/fileio.c
@@ -320,9 +320,8 @@ ptrdiff_t file_write(FileDescriptor *const fp, const char *const buf, const size
   FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ARG(1)
 {
   assert(fp->wr);
-  ptrdiff_t space = (fp->buffer + ARENA_BLOCK_SIZE) - fp->write_pos;
   // includes the trivial case of size==0
-  if (size < (size_t)space) {
+  if (size < file_space(fp)) {
     memcpy(fp->write_pos, buf, size);
     fp->write_pos += size;
     return (ptrdiff_t)size;
diff --git a/src/nvim/os/fileio.h b/src/nvim/os/fileio.h
index e8fd2209db..523f9657a4 100644
--- a/src/nvim/os/fileio.h
+++ b/src/nvim/os/fileio.h
@@ -2,6 +2,7 @@
 
 #include <stddef.h>  // IWYU pragma: keep
 
+#include "nvim/memory_defs.h"
 #include "nvim/os/fileio_defs.h"  // IWYU pragma: keep
 
 /// file_open() flags
@@ -32,6 +33,11 @@ enum {
   kRWBufferSize = 1024,
 };
 
+static inline size_t file_space(FileDescriptor *fp)
+{
+  return (size_t)((fp->buffer + ARENA_BLOCK_SIZE) - fp->write_pos);
+}
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "os/fileio.h.generated.h"
 #endif
diff --git a/src/nvim/shada.c b/src/nvim/shada.c
index d5655b1754..5736ea0f09 100644
--- a/src/nvim/shada.c
+++ b/src/nvim/shada.c
@@ -42,6 +42,7 @@
 #include "nvim/mbyte.h"
 #include "nvim/memory.h"
 #include "nvim/message.h"
+#include "nvim/msgpack_rpc/packer.h"
 #include "nvim/normal_defs.h"
 #include "nvim/ops.h"
 #include "nvim/option.h"
@@ -602,20 +603,6 @@ static void close_file(FileDescriptor *cookie)
   }
 }
 
-/// Msgpack callback for writing to FileDescriptor*
-static int msgpack_sd_writer_write(void *data, const char *buf, size_t len)
-{
-  FileDescriptor *const sd_writer = (FileDescriptor *)data;
-  const ptrdiff_t ret = file_write(sd_writer, buf, len);
-  if (ret < 0) {
-    semsg(_(SERR "System error while writing ShaDa file: %s"),
-          os_strerror((int)ret));
-    return -1;
-  }
-
-  return 0;
-}
-
 /// Check whether writing to shada file was disabled ("-i NONE" or "--clean").
 ///
 /// @return true if it was disabled, false otherwise.
@@ -1365,6 +1352,8 @@ static char *shada_filename(const char *file)
     } \
   } while (0)
 
+#define SHADA_MPACK_FREE_SPACE (4 * MPACK_ITEM_SIZE)
+
 /// Write single ShaDa entry
 ///
 /// @param[in]  packer     Packer used to write entry.
@@ -1373,7 +1362,7 @@ static char *shada_filename(const char *file)
 ///                        restrictions.
 ///
 /// @return kSDWriteSuccessful, kSDWriteFailed or kSDWriteIgnError.
-static ShaDaWriteResult shada_pack_entry(msgpack_packer *const packer, ShadaEntry entry,
+static ShaDaWriteResult shada_pack_entry(PackerBuffer *const packer, ShadaEntry entry,
                                          const size_t max_kbyte)
   FUNC_ATTR_NONNULL_ALL
 {
@@ -1657,29 +1646,26 @@ static ShaDaWriteResult shada_pack_entry(msgpack_packer *const packer, ShadaEntr
 #undef CHECK_DEFAULT
 #undef ONE_IF_NOT_DEFAULT
   if (!max_kbyte || sbuf.size <= max_kbyte * 1024) {
+    if (mpack_remaining(packer) < SHADA_MPACK_FREE_SPACE) {
+      packer->packer_flush(packer);
+    }
+
     if (entry.type == kSDItemUnknown) {
-      if (msgpack_pack_uint64(packer, entry.data.unknown_item.type) == -1) {
-        goto shada_pack_entry_error;
-      }
+      mpack_uint64(&packer->ptr, entry.data.unknown_item.type);
     } else {
-      if (msgpack_pack_uint64(packer, (uint64_t)entry.type) == -1) {
-        goto shada_pack_entry_error;
-      }
-    }
-    if (msgpack_pack_uint64(packer, (uint64_t)entry.timestamp) == -1) {
-      goto shada_pack_entry_error;
+      mpack_uint64(&packer->ptr, (uint64_t)entry.type);
     }
+    mpack_uint64(&packer->ptr, (uint64_t)entry.timestamp);
     if (sbuf.size > 0) {
-      if ((msgpack_pack_uint64(packer, (uint64_t)sbuf.size) == -1)
-          || (packer->callback(packer->data, sbuf.data,
-                               (unsigned)sbuf.size) == -1)) {
-        goto shada_pack_entry_error;
-      }
+      mpack_uint64(&packer->ptr, (uint64_t)sbuf.size);
+      mpack_raw(sbuf.data, sbuf.size, packer);
+    }
+
+    if (packer->anyint != 0) {  // error code
+      goto shada_pack_entry_error;
     }
   }
-  msgpack_packer_free(spacker);
-  msgpack_sbuffer_destroy(&sbuf);
-  return kSDWriteSuccessful;
+  ret = kSDWriteSuccessful;
 shada_pack_entry_error:
   msgpack_packer_free(spacker);
   msgpack_sbuffer_destroy(&sbuf);
@@ -1694,7 +1680,7 @@ shada_pack_entry_error:
 /// @param[in]  entry      Entry written.
 /// @param[in]  max_kbyte  Maximum size of an item in KiB. Zero means no
 ///                        restrictions.
-static inline ShaDaWriteResult shada_pack_pfreed_entry(msgpack_packer *const packer,
+static inline ShaDaWriteResult shada_pack_pfreed_entry(PackerBuffer *const packer,
                                                        PossiblyFreedShadaEntry entry,
                                                        const size_t max_kbyte)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE
@@ -1908,7 +1894,7 @@ static inline ShaDaWriteResult shada_read_when_writing(FileDescriptor *const sd_
                                                        const unsigned srni_flags,
                                                        const size_t max_kbyte,
                                                        WriteMergerState *const wms,
-                                                       msgpack_packer *const packer)
+                                                       PackerBuffer *const packer)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
   ShaDaWriteResult ret = kSDWriteSuccessful;
@@ -2360,6 +2346,29 @@ static int hist_type2char(const int type)
   return NUL;
 }
 
+static PackerBuffer packer_buffer_for_file(FileDescriptor *file)
+{
+  if (file_space(file) < SHADA_MPACK_FREE_SPACE) {
+    file_flush(file);
+  }
+  return (PackerBuffer) {
+    .startptr = file->buffer,
+    .ptr = file->write_pos,
+    .endptr = file->buffer + ARENA_BLOCK_SIZE,
+    .anydata = file,
+    .anyint = 0,  // set to nonzero if error
+    .packer_flush = flush_file_buffer,
+  };
+}
+
+static void flush_file_buffer(PackerBuffer *buffer)
+{
+  FileDescriptor *fd = buffer->anydata;
+  fd->write_pos = buffer->ptr;
+  buffer->anyint = file_flush(fd);
+  buffer->ptr = fd->write_pos;
+}
+
 /// Write ShaDa file
 ///
 /// @param[in]  sd_writer  Structure containing file writer definition.
@@ -2408,8 +2417,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
     }
   }
 
-  const unsigned srni_flags = (unsigned)(
-                                         kSDReadUndisableableData
+  const unsigned srni_flags = (unsigned)(kSDReadUndisableableData
                                          | kSDReadUnknown
                                          | (dump_history ? kSDReadHistory : 0)
                                          | (dump_registers ? kSDReadRegisters : 0)
@@ -2418,8 +2426,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
                                          | (num_marked_files ? kSDReadLocalMarks |
                                             kSDReadChanges : 0));
 
-  msgpack_packer *const packer = msgpack_packer_new(sd_writer,
-                                                    &msgpack_sd_writer_write);
+  PackerBuffer packer = packer_buffer_for_file(sd_writer);
 
   // Set b_last_cursor for all the buffers that have a window.
   //
@@ -2433,7 +2440,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
   find_removable_bufs(&removable_bufs);
 
   // Write header
-  if (shada_pack_entry(packer, (ShadaEntry) {
+  if (shada_pack_entry(&packer, (ShadaEntry) {
     .type = kSDItemHeader,
     .timestamp = os_time(),
     .data = {
@@ -2462,7 +2469,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
   // Write buffer list
   if (find_shada_parameter('%') != NULL) {
     ShadaEntry buflist_entry = shada_get_buflist(&removable_bufs);
-    if (shada_pack_entry(packer, buflist_entry, 0) == kSDWriteFailed) {
+    if (shada_pack_entry(&packer, buflist_entry, 0) == kSDWriteFailed) {
       xfree(buflist_entry.data.buffer_list.buffers);
       ret = kSDWriteFailed;
       goto shada_write_exit;
@@ -2512,7 +2519,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
       typval_T tgttv;
       tv_copy(&vartv, &tgttv);
       ShaDaWriteResult spe_ret;
-      if ((spe_ret = shada_pack_entry(packer, (ShadaEntry) {
+      if ((spe_ret = shada_pack_entry(&packer, (ShadaEntry) {
         .type = kSDItemVariable,
         .timestamp = cur_timestamp,
         .data = {
@@ -2689,7 +2696,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
 
   if (sd_reader != NULL) {
     const ShaDaWriteResult srww_ret = shada_read_when_writing(sd_reader, srni_flags, max_kbyte, wms,
-                                                              packer);
+                                                              &packer);
     if (srww_ret != kSDWriteSuccessful) {
       ret = srww_ret;
     }
@@ -2720,7 +2727,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
   do { \
     for (size_t i_ = 0; i_ < ARRAY_SIZE(wms_array); i_++) { \
       if ((wms_array)[i_].data.type != kSDItemMissing) { \
-        if (shada_pack_pfreed_entry(packer, (wms_array)[i_], max_kbyte) \
+        if (shada_pack_pfreed_entry(&packer, (wms_array)[i_], max_kbyte) \
             == kSDWriteFailed) { \
           ret = kSDWriteFailed; \
           goto shada_write_exit; \
@@ -2732,7 +2739,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
   PACK_WMS_ARRAY(wms->numbered_marks);
   PACK_WMS_ARRAY(wms->registers);
   for (size_t i = 0; i < wms->jumps_size; i++) {
-    if (shada_pack_pfreed_entry(packer, wms->jumps[i], max_kbyte)
+    if (shada_pack_pfreed_entry(&packer, wms->jumps[i], max_kbyte)
         == kSDWriteFailed) {
       ret = kSDWriteFailed;
       goto shada_write_exit;
@@ -2741,7 +2748,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
 #define PACK_WMS_ENTRY(wms_entry) \
   do { \
     if ((wms_entry).data.type != kSDItemMissing) { \
-      if (shada_pack_pfreed_entry(packer, wms_entry, max_kbyte) \
+      if (shada_pack_pfreed_entry(&packer, wms_entry, max_kbyte) \
           == kSDWriteFailed) { \
         ret = kSDWriteFailed; \
         goto shada_write_exit; \
@@ -2767,14 +2774,14 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
   for (size_t i = 0; i < file_markss_to_dump; i++) {
     PACK_WMS_ARRAY(all_file_markss[i]->marks);
     for (size_t j = 0; j < all_file_markss[i]->changes_size; j++) {
-      if (shada_pack_pfreed_entry(packer, all_file_markss[i]->changes[j],
+      if (shada_pack_pfreed_entry(&packer, all_file_markss[i]->changes[j],
                                   max_kbyte) == kSDWriteFailed) {
         ret = kSDWriteFailed;
         goto shada_write_exit;
       }
     }
     for (size_t j = 0; j < all_file_markss[i]->additional_marks_size; j++) {
-      if (shada_pack_entry(packer, all_file_markss[i]->additional_marks[j],
+      if (shada_pack_entry(&packer, all_file_markss[i]->additional_marks[j],
                            0) == kSDWriteFailed) {
         shada_free_shada_entry(&all_file_markss[i]->additional_marks[j]);
         ret = kSDWriteFailed;
@@ -2792,7 +2799,7 @@ static ShaDaWriteResult shada_write(FileDescriptor *const sd_writer,
       if (dump_one_history[i]) {
         hms_insert_whole_neovim_history(&wms->hms[i]);
         HMS_ITER(&wms->hms[i], cur_entry, {
-          if (shada_pack_pfreed_entry(packer, (PossiblyFreedShadaEntry) {
+          if (shada_pack_pfreed_entry(&packer, (PossiblyFreedShadaEntry) {
             .data = cur_entry->data,
             .can_free_entry = cur_entry->can_free_entry,
           }, max_kbyte) == kSDWriteFailed) {
@@ -2818,7 +2825,7 @@ shada_write_exit:
   })
   map_destroy(cstr_t, &wms->file_marks);
   set_destroy(ptr_t, &removable_bufs);
-  msgpack_packer_free(packer);
+  packer.packer_flush(&packer);
   set_destroy(cstr_t, &wms->dumped_variables);
   xfree(wms);
   return ret;
@@ -3953,16 +3960,43 @@ static inline size_t shada_init_jumps(PossiblyFreedShadaEntry *jumps,
   return jumps_size;
 }
 
+static PackerBuffer packer_string_buffer(void)
+{
+  const size_t initial_size = 64;  // must be larger than SHADA_MPACK_FREE_SPACE
+  char *alloc = xmalloc(initial_size);
+  return (PackerBuffer) {
+    .startptr = alloc,
+    .ptr = alloc,
+    .endptr = alloc + initial_size,
+    .packer_flush = flush_string_buffer,
+  };
+}
+
+static void flush_string_buffer(PackerBuffer *buffer)
+{
+  size_t current_capacity = (size_t)(buffer->endptr - buffer->startptr);
+  size_t new_capacity = 2 * current_capacity;
+  size_t len = (size_t)(buffer->ptr - buffer->startptr);
+
+  buffer->startptr = xrealloc(buffer->startptr, new_capacity);
+  buffer->ptr = buffer->startptr + len;
+  buffer->endptr = buffer->startptr + new_capacity;
+}
+
+static String packer_take_string(PackerBuffer *buffer)
+{
+  return (String){ .data = buffer->startptr, .size = (size_t)(buffer->ptr - buffer->startptr) };
+}
+
 /// Write registers ShaDa entries in given msgpack_sbuffer.
 ///
 /// @param[in]  sbuf  target msgpack_sbuffer to write to.
-void shada_encode_regs(msgpack_sbuffer *const sbuf)
+String shada_encode_regs(void)
   FUNC_ATTR_NONNULL_ALL
 {
   WriteMergerState *const wms = xcalloc(1, sizeof(*wms));
   shada_initialize_registers(wms, -1);
-  msgpack_packer packer;
-  msgpack_packer_init(&packer, sbuf, msgpack_sbuffer_write);
+  PackerBuffer packer = packer_string_buffer();
   for (size_t i = 0; i < ARRAY_SIZE(wms->registers); i++) {
     if (wms->registers[i].data.type == kSDItemRegister) {
       if (kSDWriteFailed
@@ -3972,52 +4006,53 @@ void shada_encode_regs(msgpack_sbuffer *const sbuf)
     }
   }
   xfree(wms);
+  return packer_take_string(&packer);
 }
 
 /// Write jumplist ShaDa entries in given msgpack_sbuffer.
 ///
 /// @param[in]  sbuf            target msgpack_sbuffer to write to.
-void shada_encode_jumps(msgpack_sbuffer *const sbuf)
+String shada_encode_jumps(void)
   FUNC_ATTR_NONNULL_ALL
 {
   Set(ptr_t) removable_bufs = SET_INIT;
   find_removable_bufs(&removable_bufs);
   PossiblyFreedShadaEntry jumps[JUMPLISTSIZE];
   size_t jumps_size = shada_init_jumps(jumps, &removable_bufs);
-  msgpack_packer packer;
-  msgpack_packer_init(&packer, sbuf, msgpack_sbuffer_write);
+  PackerBuffer packer = packer_string_buffer();
   for (size_t i = 0; i < jumps_size; i++) {
     if (kSDWriteFailed == shada_pack_pfreed_entry(&packer, jumps[i], 0)) {
       abort();
     }
   }
+  return packer_take_string(&packer);
 }
 
 /// Write buffer list ShaDa entry in given msgpack_sbuffer.
 ///
 /// @param[in]  sbuf            target msgpack_sbuffer to write to.
-void shada_encode_buflist(msgpack_sbuffer *const sbuf)
+String shada_encode_buflist(void)
   FUNC_ATTR_NONNULL_ALL
 {
   Set(ptr_t) removable_bufs = SET_INIT;
   find_removable_bufs(&removable_bufs);
   ShadaEntry buflist_entry = shada_get_buflist(&removable_bufs);
-  msgpack_packer packer;
-  msgpack_packer_init(&packer, sbuf, msgpack_sbuffer_write);
+
+  PackerBuffer packer = packer_string_buffer();
   if (kSDWriteFailed == shada_pack_entry(&packer, buflist_entry, 0)) {
     abort();
   }
   xfree(buflist_entry.data.buffer_list.buffers);
+  return packer_take_string(&packer);
 }
 
 /// Write global variables ShaDa entries in given msgpack_sbuffer.
 ///
 /// @param[in]  sbuf            target msgpack_sbuffer to write to.
-void shada_encode_gvars(msgpack_sbuffer *const sbuf)
+String shada_encode_gvars(void)
   FUNC_ATTR_NONNULL_ALL
 {
-  msgpack_packer packer;
-  msgpack_packer_init(&packer, sbuf, msgpack_sbuffer_write);
+  PackerBuffer packer = packer_string_buffer();
   const void *var_iter = NULL;
   const Timestamp cur_timestamp = os_time();
   do {
@@ -4049,21 +4084,21 @@ void shada_encode_gvars(msgpack_sbuffer *const sbuf)
     }
     tv_clear(&vartv);
   } while (var_iter != NULL);
+  return packer_take_string(&packer);
 }
 
-/// Read ShaDa from msgpack_sbuffer.
+/// Read ShaDa from String.
 ///
-/// @param[in]  file   msgpack_sbuffer to read from.
+/// @param[in]  string   string to read from.
 /// @param[in]  flags  Flags, see ShaDaReadFileFlags enum.
-void shada_read_sbuf(msgpack_sbuffer *const sbuf, const int flags)
+void shada_read_string(String string, const int flags)
   FUNC_ATTR_NONNULL_ALL
 {
-  assert(sbuf != NULL);
-  if (sbuf->data == NULL) {
+  if (string.size == 0) {
     return;
   }
   FileDescriptor sd_reader;
-  file_open_buffer(&sd_reader, sbuf->data, sbuf->size);
+  file_open_buffer(&sd_reader, string.data, string.size);
   shada_read(&sd_reader, flags);
   close_file(&sd_reader);
 }
diff --git a/src/nvim/shada.h b/src/nvim/shada.h
index d7cac24afc..58689a5bd7 100644
--- a/src/nvim/shada.h
+++ b/src/nvim/shada.h
@@ -2,6 +2,8 @@
 
 #include <msgpack.h>  // IWYU pragma: keep
 
+#include "nvim/api/private/defs.h"
+
 /// Flags for shada_read_file and children
 typedef enum {
   kShaDaWantInfo = 1,       ///< Load non-mark information