Merge remote-tracking branch 'upstream/master' into mix_20240309

17 files changed, 2633 insertions, 1917 deletions

diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c
index d7df7bb150..afc2efddf6 100644
--- a/src/nvim/eval/decode.c
+++ b/src/nvim/eval/decode.c
@@ -1,5 +1,4 @@
 #include <assert.h>
-#include <msgpack/object.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -7,6 +6,7 @@
 #include <string.h>
 
 #include "klib/kvec.h"
+#include "mpack/object.h"
 #include "nvim/ascii_defs.h"
 #include "nvim/charset.h"
 #include "nvim/eval.h"
@@ -247,45 +247,29 @@ list_T *decode_create_map_special_dict(typval_T *const ret_tv, const ptrdiff_t l
 ///
 /// @param[in]  s  String to decode.
 /// @param[in]  len  String length.
-/// @param[in]  hasnul  Whether string has NUL byte, not or it was not yet
-///                     determined.
-/// @param[in]  binary  Determines decode type if string has NUL bytes.
-///                     If true convert string to VAR_BLOB, otherwise to the
-///                     kMPString special type.
+/// @param[in]  force_blob  whether string always should be decoded as a blob,
+///                         or only when embedded NUL bytes were present
 /// @param[in]  s_allocated  If true, then `s` was allocated and can be saved in
 ///                          a returned structure. If it is not saved there, it
 ///                          will be freed.
 ///
 /// @return Decoded string.
-typval_T decode_string(const char *const s, const size_t len, const TriState hasnul,
-                       const bool binary, const bool s_allocated)
+typval_T decode_string(const char *const s, const size_t len, bool force_blob,
+                       const bool s_allocated)
   FUNC_ATTR_WARN_UNUSED_RESULT
 {
   assert(s != NULL || len == 0);
-  const bool really_hasnul = (hasnul == kNone
-                              ? ((s != NULL) && (memchr(s, NUL, len) != NULL))
-                              : (bool)hasnul);
-  if (really_hasnul) {
+  const bool use_blob = force_blob || ((s != NULL) && (memchr(s, NUL, len) != NULL));
+  if (use_blob) {
     typval_T tv;
     tv.v_lock = VAR_UNLOCKED;
-    if (binary) {
-      tv_blob_alloc_ret(&tv);
-      ga_concat_len(&tv.vval.v_blob->bv_ga, s, len);
+    blob_T *b = tv_blob_alloc_ret(&tv);
+    if (s_allocated) {
+      b->bv_ga.ga_data = (void *)s;
+      b->bv_ga.ga_len = (int)len;
+      b->bv_ga.ga_maxlen = (int)len;
     } else {
-      list_T *const list = tv_list_alloc(kListLenMayKnow);
-      tv_list_ref(list);
-      create_special_dict(&tv, kMPString,
-                          (typval_T){ .v_type = VAR_LIST,
-                                      .v_lock = VAR_UNLOCKED,
-                                      .vval = { .v_list = list } });
-      const int elw_ret = encode_list_write((void *)list, s, len);
-      if (s_allocated) {
-        xfree((void *)s);
-      }
-      if (elw_ret == -1) {
-        tv_clear(&tv);
-        return (typval_T) { .v_type = VAR_UNKNOWN, .v_lock = VAR_UNLOCKED };
-      }
+      ga_concat_len(&b->bv_ga, s, len);
     }
     return tv;
   }
@@ -405,7 +389,6 @@ static inline int parse_json_string(const char *const buf, const size_t buf_len,
   char *str = xmalloc(len + 1);
   int fst_in_pair = 0;
   char *str_end = str;
-  bool hasnul = false;
 #define PUT_FST_IN_PAIR(fst_in_pair, str_end) \
   do { \
     if ((fst_in_pair) != 0) { \
@@ -426,9 +409,6 @@ static inline int parse_json_string(const char *const buf, const size_t buf_len,
         uvarnumber_T ch;
         vim_str2nr(ubuf, NULL, NULL,
                    STR2NR_HEX | STR2NR_FORCE, NULL, &ch, 4, true, NULL);
-        if (ch == 0) {
-          hasnul = true;
-        }
         if (SURROGATE_HI_START <= ch && ch <= SURROGATE_HI_END) {
           PUT_FST_IN_PAIR(fst_in_pair, str_end);
           fst_in_pair = (int)ch;
@@ -476,10 +456,7 @@ static inline int parse_json_string(const char *const buf, const size_t buf_len,
   PUT_FST_IN_PAIR(fst_in_pair, str_end);
 #undef PUT_FST_IN_PAIR
   *str_end = NUL;
-  typval_T obj = decode_string(str, (size_t)(str_end - str), hasnul ? kTrue : kFalse, false, true);
-  if (obj.v_type == VAR_UNKNOWN) {
-    goto parse_json_string_fail;
-  }
+  typval_T obj = decode_string(str, (size_t)(str_end - str), false, true);
   POP(obj, obj.v_type != VAR_STRING);
   goto parse_json_string_ret;
 parse_json_string_fail:
@@ -908,182 +885,250 @@ json_decode_string_ret:
 
 #undef DICT_LEN
 
-/// Convert msgpack object to a Vimscript one
-int msgpack_to_vim(const msgpack_object mobj, typval_T *const rettv)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+static void positive_integer_to_special_typval(typval_T *rettv, uint64_t val)
 {
-  switch (mobj.type) {
-  case MSGPACK_OBJECT_NIL:
+  if (val <= VARNUMBER_MAX) {
     *rettv = (typval_T) {
+      .v_type = VAR_NUMBER,
+      .v_lock = VAR_UNLOCKED,
+      .vval = { .v_number = (varnumber_T)val },
+    };
+  } else {
+    list_T *const list = tv_list_alloc(4);
+    tv_list_ref(list);
+    create_special_dict(rettv, kMPInteger, ((typval_T) {
+      .v_type = VAR_LIST,
+      .v_lock = VAR_UNLOCKED,
+      .vval = { .v_list = list },
+    }));
+    tv_list_append_number(list, 1);
+    tv_list_append_number(list, (varnumber_T)((val >> 62) & 0x3));
+    tv_list_append_number(list, (varnumber_T)((val >> 31) & 0x7FFFFFFF));
+    tv_list_append_number(list, (varnumber_T)(val & 0x7FFFFFFF));
+  }
+}
+
+static void typval_parse_enter(mpack_parser_t *parser, mpack_node_t *node)
+{
+  typval_T *result = NULL;
+
+  mpack_node_t *parent = MPACK_PARENT_NODE(node);
+  if (parent) {
+    switch (parent->tok.type) {
+    case MPACK_TOKEN_ARRAY: {
+      list_T *list = parent->data[1].p;
+      result = tv_list_append_owned_tv(list, (typval_T) { .v_type = VAR_UNKNOWN });
+      break;
+    }
+    case MPACK_TOKEN_MAP: {
+      typval_T(*items)[2] = parent->data[1].p;
+      result = &items[parent->pos][parent->key_visited];
+      break;
+    }
+
+    case MPACK_TOKEN_STR:
+    case MPACK_TOKEN_BIN:
+    case MPACK_TOKEN_EXT:
+      assert(node->tok.type == MPACK_TOKEN_CHUNK);
+      break;
+
+    default:
+      abort();
+    }
+  } else {
+    result = parser->data.p;
+  }
+
+  // for types that are completed in typval_parse_exit
+  node->data[0].p = result;
+  node->data[1].p = NULL;  // free on error if non-NULL
+
+  switch (node->tok.type) {
+  case MPACK_TOKEN_NIL:
+    *result = (typval_T) {
       .v_type = VAR_SPECIAL,
       .v_lock = VAR_UNLOCKED,
       .vval = { .v_special = kSpecialVarNull },
     };
     break;
-  case MSGPACK_OBJECT_BOOLEAN:
-    *rettv = (typval_T) {
+  case MPACK_TOKEN_BOOLEAN:
+    *result = (typval_T) {
       .v_type = VAR_BOOL,
       .v_lock = VAR_UNLOCKED,
       .vval = {
-        .v_bool = mobj.via.boolean ? kBoolVarTrue : kBoolVarFalse
+        .v_bool = mpack_unpack_boolean(node->tok) ? kBoolVarTrue : kBoolVarFalse
       },
     };
     break;
-  case MSGPACK_OBJECT_POSITIVE_INTEGER:
-    if (mobj.via.u64 <= VARNUMBER_MAX) {
-      *rettv = (typval_T) {
-        .v_type = VAR_NUMBER,
-        .v_lock = VAR_UNLOCKED,
-        .vval = { .v_number = (varnumber_T)mobj.via.u64 },
-      };
-    } else {
-      list_T *const list = tv_list_alloc(4);
-      tv_list_ref(list);
-      create_special_dict(rettv, kMPInteger, ((typval_T) {
-        .v_type = VAR_LIST,
-        .v_lock = VAR_UNLOCKED,
-        .vval = { .v_list = list },
-      }));
-      uint64_t n = mobj.via.u64;
-      tv_list_append_number(list, 1);
-      tv_list_append_number(list, (varnumber_T)((n >> 62) & 0x3));
-      tv_list_append_number(list, (varnumber_T)((n >> 31) & 0x7FFFFFFF));
-      tv_list_append_number(list, (varnumber_T)(n & 0x7FFFFFFF));
-    }
+  case MPACK_TOKEN_SINT: {
+    *result = (typval_T) {
+      .v_type = VAR_NUMBER,
+      .v_lock = VAR_UNLOCKED,
+      .vval = { .v_number = mpack_unpack_sint(node->tok) },
+    };
     break;
-  case MSGPACK_OBJECT_NEGATIVE_INTEGER:
-    if (mobj.via.i64 >= VARNUMBER_MIN) {
-      *rettv = (typval_T) {
-        .v_type = VAR_NUMBER,
-        .v_lock = VAR_UNLOCKED,
-        .vval = { .v_number = (varnumber_T)mobj.via.i64 },
-      };
-    } else {
-      list_T *const list = tv_list_alloc(4);
-      tv_list_ref(list);
-      create_special_dict(rettv, kMPInteger, ((typval_T) {
-        .v_type = VAR_LIST,
-        .v_lock = VAR_UNLOCKED,
-        .vval = { .v_list = list },
-      }));
-      uint64_t n = -((uint64_t)mobj.via.i64);
-      tv_list_append_number(list, -1);
-      tv_list_append_number(list, (varnumber_T)((n >> 62) & 0x3));
-      tv_list_append_number(list, (varnumber_T)((n >> 31) & 0x7FFFFFFF));
-      tv_list_append_number(list, (varnumber_T)(n & 0x7FFFFFFF));
-    }
+  }
+  case MPACK_TOKEN_UINT:
+    positive_integer_to_special_typval(result, mpack_unpack_uint(node->tok));
     break;
-  case MSGPACK_OBJECT_FLOAT32:
-  case MSGPACK_OBJECT_FLOAT64:
-    *rettv = (typval_T) {
+  case MPACK_TOKEN_FLOAT:
+    *result = (typval_T) {
       .v_type = VAR_FLOAT,
       .v_lock = VAR_UNLOCKED,
-      .vval = { .v_float = mobj.via.f64 },
+      .vval = { .v_float = mpack_unpack_float(node->tok) },
     };
     break;
-  case MSGPACK_OBJECT_STR:
-    *rettv = decode_string(mobj.via.bin.ptr, mobj.via.bin.size, kTrue, false,
-                           false);
-    if (rettv->v_type == VAR_UNKNOWN) {
-      return FAIL;
-    }
+
+  case MPACK_TOKEN_BIN:
+  case MPACK_TOKEN_STR:
+  case MPACK_TOKEN_EXT:
+    // actually converted in typval_parse_exit after the data chunks
+    node->data[1].p = xmallocz(node->tok.length);
     break;
-  case MSGPACK_OBJECT_BIN:
-    *rettv = decode_string(mobj.via.bin.ptr, mobj.via.bin.size, kNone, true,
-                           false);
-    if (rettv->v_type == VAR_UNKNOWN) {
-      return FAIL;
-    }
+  case MPACK_TOKEN_CHUNK: {
+    char *data = parent->data[1].p;
+    memcpy(data + parent->pos,
+           node->tok.data.chunk_ptr, node->tok.length);
     break;
-  case MSGPACK_OBJECT_ARRAY: {
-    list_T *const list = tv_list_alloc((ptrdiff_t)mobj.via.array.size);
+  }
+
+  case MPACK_TOKEN_ARRAY: {
+    list_T *const list = tv_list_alloc((ptrdiff_t)node->tok.length);
     tv_list_ref(list);
-    *rettv = (typval_T) {
+    *result = (typval_T) {
       .v_type = VAR_LIST,
       .v_lock = VAR_UNLOCKED,
       .vval = { .v_list = list },
     };
-    for (size_t i = 0; i < mobj.via.array.size; i++) {
-      // Not populated yet, need to create list item to push.
-      tv_list_append_owned_tv(list, (typval_T) { .v_type = VAR_UNKNOWN });
-      if (msgpack_to_vim(mobj.via.array.ptr[i],
-                         TV_LIST_ITEM_TV(tv_list_last(list)))
-          == FAIL) {
-        return FAIL;
-      }
+    node->data[1].p = list;
+    break;
+  }
+  case MPACK_TOKEN_MAP:
+    // we don't know if this will be safe to convert to a typval dict yet
+    node->data[1].p = xmallocz(node->tok.length * 2 * sizeof(typval_T));
+    break;
+  }
+}
+
+/// Free node which was entered but never exited, due to a nested error
+///
+/// Don't bother with typvals as these will be GC:d eventually
+void typval_parser_error_free(mpack_parser_t *parser)
+{
+  for (uint32_t i = 0; i < parser->size; i++) {
+    mpack_node_t *node = &parser->items[i];
+    switch (node->tok.type) {
+    case MPACK_TOKEN_BIN:
+    case MPACK_TOKEN_STR:
+    case MPACK_TOKEN_EXT:
+    case MPACK_TOKEN_MAP:
+      XFREE_CLEAR(node->data[1].p);
+      break;
+    default:
+      break;
     }
+  }
+}
+
+static void typval_parse_exit(mpack_parser_t *parser, mpack_node_t *node)
+{
+  typval_T *result = node->data[0].p;
+  switch (node->tok.type) {
+  case MPACK_TOKEN_BIN:
+  case MPACK_TOKEN_STR:
+    *result = decode_string(node->data[1].p, node->tok.length, false, true);
+    node->data[1].p = NULL;
     break;
+
+  case MPACK_TOKEN_EXT: {
+    list_T *const list = tv_list_alloc(2);
+    tv_list_ref(list);
+    tv_list_append_number(list, node->tok.data.ext_type);
+    list_T *const ext_val_list = tv_list_alloc(kListLenMayKnow);
+    tv_list_append_list(list, ext_val_list);
+    create_special_dict(result, kMPExt, ((typval_T) { .v_type = VAR_LIST,
+                                                      .v_lock = VAR_UNLOCKED,
+                                                      .vval = { .v_list = list } }));
+    // TODO(bfredl): why not use BLOB?
+    encode_list_write((void *)ext_val_list, node->data[1].p, node->tok.length);
+    XFREE_CLEAR(node->data[1].p);
   }
-  case MSGPACK_OBJECT_MAP: {
-    for (size_t i = 0; i < mobj.via.map.size; i++) {
-      if (mobj.via.map.ptr[i].key.type != MSGPACK_OBJECT_STR
-          || mobj.via.map.ptr[i].key.via.str.size == 0
-          || memchr(mobj.via.map.ptr[i].key.via.str.ptr, NUL,
-                    mobj.via.map.ptr[i].key.via.str.size) != NULL) {
+  break;
+
+  case MPACK_TOKEN_MAP: {
+    typval_T(*items)[2] = node->data[1].p;
+    for (size_t i = 0; i < node->tok.length; i++) {
+      typval_T *key = &items[i][0];
+      if (key->v_type != VAR_STRING
+          || key->vval.v_string == NULL
+          || key->vval.v_string[0] == NUL) {
         goto msgpack_to_vim_generic_map;
       }
     }
     dict_T *const dict = tv_dict_alloc();
     dict->dv_refcount++;
-    *rettv = (typval_T) {
+    *result = (typval_T) {
       .v_type = VAR_DICT,
       .v_lock = VAR_UNLOCKED,
       .vval = { .v_dict = dict },
     };
-    for (size_t i = 0; i < mobj.via.map.size; i++) {
-      dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key)
-                                      + mobj.via.map.ptr[i].key.via.str.size);
-      memcpy(&di->di_key[0], mobj.via.map.ptr[i].key.via.str.ptr,
-             mobj.via.map.ptr[i].key.via.str.size);
+    for (size_t i = 0; i < node->tok.length; i++) {
+      char *key = items[i][0].vval.v_string;
+      size_t keylen = strlen(key);
+      dictitem_T *const di = xmallocz(offsetof(dictitem_T, di_key) + keylen);
+      memcpy(&di->di_key[0], key, keylen);
       di->di_tv.v_type = VAR_UNKNOWN;
       if (tv_dict_add(dict, di) == FAIL) {
         // Duplicate key: fallback to generic map
-        tv_clear(rettv);
+        TV_DICT_ITER(dict, d, {
+            d->di_tv.v_type = VAR_SPECIAL;  // don't free values in tv_clear(), they will be reused
+            d->di_tv.vval.v_special = kSpecialVarNull;
+          });
+        tv_clear(result);
         xfree(di);
         goto msgpack_to_vim_generic_map;
       }
-      if (msgpack_to_vim(mobj.via.map.ptr[i].val, &di->di_tv) == FAIL) {
-        return FAIL;
-      }
+      di->di_tv = items[i][1];
+    }
+    for (size_t i = 0; i < node->tok.length; i++) {
+      xfree(items[i][0].vval.v_string);
     }
+    XFREE_CLEAR(node->data[1].p);
     break;
 msgpack_to_vim_generic_map: {}
-    list_T *const list = decode_create_map_special_dict(rettv, (ptrdiff_t)mobj.via.map.size);
-    for (size_t i = 0; i < mobj.via.map.size; i++) {
+    list_T *const list = decode_create_map_special_dict(result, node->tok.length);
+    for (size_t i = 0; i < node->tok.length; i++) {
       list_T *const kv_pair = tv_list_alloc(2);
       tv_list_append_list(list, kv_pair);
 
-      typval_T key_tv = { .v_type = VAR_UNKNOWN };
-      if (msgpack_to_vim(mobj.via.map.ptr[i].key, &key_tv) == FAIL) {
-        tv_clear(&key_tv);
-        return FAIL;
-      }
-      tv_list_append_owned_tv(kv_pair, key_tv);
-
-      typval_T val_tv = { .v_type = VAR_UNKNOWN };
-      if (msgpack_to_vim(mobj.via.map.ptr[i].val, &val_tv) == FAIL) {
-        tv_clear(&val_tv);
-        return FAIL;
-      }
-      tv_list_append_owned_tv(kv_pair, val_tv);
+      tv_list_append_owned_tv(kv_pair, items[i][0]);
+      tv_list_append_owned_tv(kv_pair, items[i][1]);
     }
+    XFREE_CLEAR(node->data[1].p);
     break;
   }
-  case MSGPACK_OBJECT_EXT: {
-    list_T *const list = tv_list_alloc(2);
-    tv_list_ref(list);
-    tv_list_append_number(list, mobj.via.ext.type);
-    list_T *const ext_val_list = tv_list_alloc(kListLenMayKnow);
-    tv_list_append_list(list, ext_val_list);
-    create_special_dict(rettv, kMPExt, ((typval_T) { .v_type = VAR_LIST,
-                                                     .v_lock = VAR_UNLOCKED,
-                                                     .vval = { .v_list = list } }));
-    if (encode_list_write((void *)ext_val_list, mobj.via.ext.ptr,
-                          mobj.via.ext.size) == -1) {
-      return FAIL;
-    }
+
+  default:
+    // other kinds are handled completely in typval_parse_enter
     break;
   }
+}
+
+int mpack_parse_typval(mpack_parser_t *parser, const char **data, size_t *size)
+{
+  return mpack_parse(parser, data, size, typval_parse_enter, typval_parse_exit);
+}
+
+int unpack_typval(const char **data, size_t *size, typval_T *ret)
+{
+  ret->v_type = VAR_UNKNOWN;
+  mpack_parser_t parser;
+  mpack_parser_init(&parser, 0);
+  parser.data.p = ret;
+  int status = mpack_parse_typval(&parser, data, size);
+  if (status != MPACK_OK) {
+    typval_parser_error_free(&parser);
+    tv_clear(ret);
   }
-  return OK;
+  return status;
 }
diff --git a/src/nvim/eval/decode.h b/src/nvim/eval/decode.h
index c0d10a469a..485cc65561 100644
--- a/src/nvim/eval/decode.h
+++ b/src/nvim/eval/decode.h
@@ -1,8 +1,8 @@
 #pragma once
 
-#include <msgpack.h>  // IWYU pragma: keep
 #include <stddef.h>  // IWYU pragma: keep
 
+#include "mpack/object.h"
 #include "nvim/eval/typval_defs.h"  // IWYU pragma: keep
 #include "nvim/types_defs.h"  // IWYU pragma: keep
 
diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c
index d35ac4eb7b..79f334601d 100644
--- a/src/nvim/eval/encode.c
+++ b/src/nvim/eval/encode.c
@@ -13,7 +13,7 @@
 #include <string.h>
 
 #include "klib/kvec.h"
-#include "msgpack/pack.h"
+#include "nvim/api/private/helpers.h"
 #include "nvim/ascii_defs.h"
 #include "nvim/eval.h"
 #include "nvim/eval/encode.h"
@@ -28,6 +28,7 @@
 #include "nvim/mbyte.h"
 #include "nvim/memory.h"
 #include "nvim/message.h"
+#include "nvim/msgpack_rpc/packer.h"
 #include "nvim/strings.h"
 #include "nvim/types_defs.h"
 #include "nvim/vim_defs.h"  // For _()
@@ -54,11 +55,11 @@ int encode_blob_write(void *const data, const char *const buf, const size_t len)
 }
 
 /// Msgpack callback for writing to readfile()-style list
-int encode_list_write(void *const data, const char *const buf, const size_t len)
+void encode_list_write(void *const data, const char *const buf, const size_t len)
   FUNC_ATTR_NONNULL_ARG(1)
 {
   if (len == 0) {
-    return 0;
+    return;
   }
   list_T *const list = (list_T *)data;
   const char *const end = buf + len;
@@ -96,7 +97,6 @@ int encode_list_write(void *const data, const char *const buf, const size_t len)
   if (line_end == end) {
     tv_list_append_allocated_string(list, NULL);
   }
-  return 0;
 }
 
 /// Abort conversion to string after a recursion error.
@@ -412,6 +412,8 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, const s
 #define TYPVAL_ENCODE_CONV_EMPTY_DICT(tv, dict) \
   ga_concat(gap, "{}")
 
+#define TYPVAL_ENCODE_CHECK_BEFORE
+
 #define TYPVAL_ENCODE_CONV_NIL(tv) \
   ga_concat(gap, "v:null")
 
@@ -536,6 +538,8 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, const s
 #undef TYPVAL_ENCODE_ALLOW_SPECIALS
 #define TYPVAL_ENCODE_ALLOW_SPECIALS true
 
+#define TYPVAL_ENCODE_CHECK_BEFORE
+
 #undef TYPVAL_ENCODE_CONV_NIL
 #define TYPVAL_ENCODE_CONV_NIL(tv) \
   ga_concat(gap, "null")
@@ -771,8 +775,7 @@ bool encode_check_json_key(const typval_T *const tv)
   const dictitem_T *val_di;
   if ((type_di = tv_dict_find(spdict, S_LEN("_TYPE"))) == NULL
       || type_di->di_tv.v_type != VAR_LIST
-      || (type_di->di_tv.vval.v_list != eval_msgpack_type_lists[kMPString]
-          && type_di->di_tv.vval.v_list != eval_msgpack_type_lists[kMPBinary])
+      || type_di->di_tv.vval.v_list != eval_msgpack_type_lists[kMPString]
       || (val_di = tv_dict_find(spdict, S_LEN("_VAL"))) == NULL
       || val_di->di_tv.v_type != VAR_LIST) {
     return false;
@@ -821,6 +824,7 @@ bool encode_check_json_key(const typval_T *const tv)
 #undef TYPVAL_ENCODE_CONV_LIST_START
 #undef TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START
 #undef TYPVAL_ENCODE_CONV_EMPTY_DICT
+#undef TYPVAL_ENCODE_CHECK_BEFORE
 #undef TYPVAL_ENCODE_CONV_NIL
 #undef TYPVAL_ENCODE_CONV_BOOL
 #undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER
@@ -855,7 +859,7 @@ char *encode_tv2string(typval_T *tv, size_t *len)
   if (len != NULL) {
     *len = (size_t)ga.ga_len;
   }
-  ga_append(&ga, '\0');
+  ga_append(&ga, NUL);
   return (char *)ga.ga_data;
 }
 
@@ -883,7 +887,7 @@ char *encode_tv2echo(typval_T *tv, size_t *len)
   if (len != NULL) {
     *len = (size_t)ga.ga_len;
   }
-  ga_append(&ga, '\0');
+  ga_append(&ga, NUL);
   return (char *)ga.ga_data;
 }
 
@@ -908,57 +912,27 @@ char *encode_tv2json(typval_T *tv, size_t *len)
   if (len != NULL) {
     *len = (size_t)ga.ga_len;
   }
-  ga_append(&ga, '\0');
+  ga_append(&ga, NUL);
   return (char *)ga.ga_data;
 }
 
 #define TYPVAL_ENCODE_CONV_STRING(tv, buf, len) \
-  do { \
-    if ((buf) == NULL) { \
-      msgpack_pack_bin(packer, 0); \
-    } else { \
-      const size_t len_ = (len); \
-      msgpack_pack_bin(packer, len_); \
-      msgpack_pack_bin_body(packer, buf, len_); \
-    } \
-  } while (0)
+  mpack_bin(cbuf_as_string(buf, (len)), packer); \
 
 #define TYPVAL_ENCODE_CONV_STR_STRING(tv, buf, len) \
-  do { \
-    if ((buf) == NULL) { \
-      msgpack_pack_str(packer, 0); \
-    } else { \
-      const size_t len_ = (len); \
-      msgpack_pack_str(packer, len_); \
-      msgpack_pack_str_body(packer, buf, len_); \
-    } \
-  } while (0)
+  mpack_str(cbuf_as_string(buf, (len)), packer); \
 
 #define TYPVAL_ENCODE_CONV_EXT_STRING(tv, buf, len, type) \
-  do { \
-    if ((buf) == NULL) { \
-      msgpack_pack_ext(packer, 0, (int8_t)(type)); \
-    } else { \
-      const size_t len_ = (len); \
-      msgpack_pack_ext(packer, len_, (int8_t)(type)); \
-      msgpack_pack_ext_body(packer, buf, len_); \
-    } \
-  } while (0)
+  mpack_ext(buf, (len), (int8_t)(type), packer); \
 
 #define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \
-  do { \
-    const size_t len_ = (size_t)(len); \
-    msgpack_pack_bin(packer, len_); \
-    if (len_ > 0) { \
-      msgpack_pack_bin_body(packer, (blob)->bv_ga.ga_data, len_); \
-    } \
-  } while (0)
+  mpack_bin(cbuf_as_string((blob) ? (blob)->bv_ga.ga_data : NULL, (size_t)(len)), packer);
 
 #define TYPVAL_ENCODE_CONV_NUMBER(tv, num) \
-  msgpack_pack_int64(packer, (int64_t)(num))
+  mpack_integer(&packer->ptr, (Integer)(num))
 
 #define TYPVAL_ENCODE_CONV_FLOAT(tv, flt) \
-  msgpack_pack_double(packer, (double)(flt))
+  mpack_float8(&packer->ptr, (double)(flt))
 
 #define TYPVAL_ENCODE_CONV_FUNC_START(tv, fun) \
   return conv_error(_("E5004: Error while dumping %s, %s: " \
@@ -970,33 +944,30 @@ char *encode_tv2json(typval_T *tv, size_t *len)
 #define TYPVAL_ENCODE_CONV_FUNC_END(tv)
 
 #define TYPVAL_ENCODE_CONV_EMPTY_LIST(tv) \
-  msgpack_pack_array(packer, 0)
+  mpack_array(&packer->ptr, 0)
 
 #define TYPVAL_ENCODE_CONV_LIST_START(tv, len) \
-  msgpack_pack_array(packer, (size_t)(len))
+  mpack_array(&packer->ptr, (uint32_t)(len))
 
 #define TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START(tv, mpsv)
 
 #define TYPVAL_ENCODE_CONV_EMPTY_DICT(tv, dict) \
-  msgpack_pack_map(packer, 0)
+  mpack_map(&packer->ptr, 0)
+
+#define TYPVAL_ENCODE_CHECK_BEFORE \
+  mpack_check_buffer(packer)
 
 #define TYPVAL_ENCODE_CONV_NIL(tv) \
-  msgpack_pack_nil(packer)
+  mpack_nil(&packer->ptr)
 
 #define TYPVAL_ENCODE_CONV_BOOL(tv, num) \
-  do { \
-    if (num) { \
-      msgpack_pack_true(packer); \
-    } else { \
-      msgpack_pack_false(packer); \
-    } \
-  } while (0)
+  mpack_bool(&packer->ptr, (bool)num); \
 
 #define TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER(tv, num) \
-  msgpack_pack_uint64(packer, (num))
+  mpack_uint64(&packer->ptr, (num))
 
 #define TYPVAL_ENCODE_CONV_DICT_START(tv, dict, len) \
-  msgpack_pack_map(packer, (size_t)(len))
+  mpack_map(&packer->ptr, (uint32_t)(len))
 
 #define TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START(tv, dict, mpsv)
 
@@ -1021,7 +992,7 @@ char *encode_tv2json(typval_T *tv, size_t *len)
 
 #define TYPVAL_ENCODE_SCOPE
 #define TYPVAL_ENCODE_NAME msgpack
-#define TYPVAL_ENCODE_FIRST_ARG_TYPE msgpack_packer *const
+#define TYPVAL_ENCODE_FIRST_ARG_TYPE PackerBuffer *const
 #define TYPVAL_ENCODE_FIRST_ARG_NAME packer
 #include "nvim/eval/typval_encode.c.h"
 #undef TYPVAL_ENCODE_SCOPE
@@ -1043,6 +1014,7 @@ char *encode_tv2json(typval_T *tv, size_t *len)
 #undef TYPVAL_ENCODE_CONV_LIST_START
 #undef TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START
 #undef TYPVAL_ENCODE_CONV_EMPTY_DICT
+#undef TYPVAL_ENCODE_CHECK_BEFORE
 #undef TYPVAL_ENCODE_CONV_NIL
 #undef TYPVAL_ENCODE_CONV_BOOL
 #undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER
diff --git a/src/nvim/eval/encode.h b/src/nvim/eval/encode.h
index 6d1c0b61c5..2bacc82b0d 100644
--- a/src/nvim/eval/encode.h
+++ b/src/nvim/eval/encode.h
@@ -1,10 +1,10 @@
 #pragma once
 
-#include <msgpack/pack.h>
 #include <string.h>
 
 #include "nvim/eval/typval_defs.h"
 #include "nvim/garray_defs.h"
+#include "nvim/msgpack_rpc/packer_defs.h"
 
 /// Convert Vimscript value to msgpack string
 ///
@@ -13,7 +13,7 @@
 /// @param[in]  objname  Object name, used for error message.
 ///
 /// @return OK in case of success, FAIL otherwise.
-int encode_vim_to_msgpack(msgpack_packer *packer, typval_T *tv, const char *objname);
+int encode_vim_to_msgpack(PackerBuffer *packer, typval_T *tv, const char *objname);
 
 /// Convert Vimscript value to :echo output
 ///
diff --git a/src/nvim/eval/executor.c b/src/nvim/eval/executor.c
index 1b8c057d7c..5b92f217d1 100644
--- a/src/nvim/eval/executor.c
+++ b/src/nvim/eval/executor.c
@@ -1,6 +1,7 @@
 #include <inttypes.h>
 #include <stdlib.h>
 
+#include "nvim/errors.h"
 #include "nvim/eval.h"
 #include "nvim/eval/executor.h"
 #include "nvim/eval/typval.h"
@@ -20,6 +21,174 @@
 char *e_list_index_out_of_range_nr
   = N_("E684: List index out of range: %" PRId64);
 
+/// Handle "blob1 += blob2".
+/// Returns OK or FAIL.
+static int tv_op_blob(typval_T *tv1, const typval_T *tv2, const char *op)
+  FUNC_ATTR_NONNULL_ALL
+{
+  if (*op != '+' || tv2->v_type != VAR_BLOB) {
+    return FAIL;
+  }
+
+  // Blob += Blob
+  if (tv2->vval.v_blob == NULL) {
+    return OK;
+  }
+
+  if (tv1->vval.v_blob == NULL) {
+    tv1->vval.v_blob = tv2->vval.v_blob;
+    tv1->vval.v_blob->bv_refcount++;
+    return OK;
+  }
+
+  blob_T *const b1 = tv1->vval.v_blob;
+  blob_T *const b2 = tv2->vval.v_blob;
+  const int len = tv_blob_len(b2);
+
+  for (int i = 0; i < len; i++) {
+    ga_append(&b1->bv_ga, tv_blob_get(b2, i));
+  }
+
+  return OK;
+}
+
+/// Handle "list1 += list2".
+/// Returns OK or FAIL.
+static int tv_op_list(typval_T *tv1, const typval_T *tv2, const char *op)
+  FUNC_ATTR_NONNULL_ALL
+{
+  if (*op != '+' || tv2->v_type != VAR_LIST) {
+    return FAIL;
+  }
+
+  // List += List
+  if (tv2->vval.v_list == NULL) {
+    return OK;
+  }
+
+  if (tv1->vval.v_list == NULL) {
+    tv1->vval.v_list = tv2->vval.v_list;
+    tv_list_ref(tv1->vval.v_list);
+  } else {
+    tv_list_extend(tv1->vval.v_list, tv2->vval.v_list, NULL);
+  }
+
+  return OK;
+}
+
+/// Handle number operations:
+///      nr += nr , nr -= nr , nr *=nr , nr /= nr , nr %= nr
+///
+/// Returns OK or FAIL.
+static int tv_op_number(typval_T *tv1, const typval_T *tv2, const char *op)
+  FUNC_ATTR_NONNULL_ALL
+{
+  varnumber_T n = tv_get_number(tv1);
+  if (tv2->v_type == VAR_FLOAT) {
+    float_T f = (float_T)n;
+    if (*op == '%') {
+      return FAIL;
+    }
+    switch (*op) {
+    case '+':
+      f += tv2->vval.v_float; break;
+    case '-':
+      f -= tv2->vval.v_float; break;
+    case '*':
+      f *= tv2->vval.v_float; break;
+    case '/':
+      f /= tv2->vval.v_float; break;
+    }
+    tv_clear(tv1);
+    tv1->v_type = VAR_FLOAT;
+    tv1->vval.v_float = f;
+  } else {
+    switch (*op) {
+    case '+':
+      n += tv_get_number(tv2); break;
+    case '-':
+      n -= tv_get_number(tv2); break;
+    case '*':
+      n *= tv_get_number(tv2); break;
+    case '/':
+      n = num_divide(n, tv_get_number(tv2)); break;
+    case '%':
+      n = num_modulus(n, tv_get_number(tv2)); break;
+    }
+    tv_clear(tv1);
+    tv1->v_type = VAR_NUMBER;
+    tv1->vval.v_number = n;
+  }
+
+  return OK;
+}
+
+/// Handle "str1 .= str2"
+/// Returns OK or FAIL.
+static int tv_op_string(typval_T *tv1, const typval_T *tv2, const char *op)
+  FUNC_ATTR_NONNULL_ALL
+{
+  if (tv2->v_type == VAR_FLOAT) {
+    return FAIL;
+  }
+
+  // str .= str
+  const char *tvs = tv_get_string(tv1);
+  char numbuf[NUMBUFLEN];
+  char *const s = concat_str(tvs, tv_get_string_buf(tv2, numbuf));
+  tv_clear(tv1);
+  tv1->v_type = VAR_STRING;
+  tv1->vval.v_string = s;
+
+  return OK;
+}
+
+/// Handle "tv1 += tv2", "tv1 -= tv2", "tv1 *= tv2", "tv1 /= tv2", "tv1 %= tv2"
+/// and "tv1 .= tv2"
+/// Returns OK or FAIL.
+static int tv_op_nr_or_string(typval_T *tv1, const typval_T *tv2, const char *op)
+  FUNC_ATTR_NONNULL_ALL
+{
+  if (tv2->v_type == VAR_LIST) {
+    return FAIL;
+  }
+
+  if (vim_strchr("+-*/%", (uint8_t)(*op)) != NULL) {
+    return tv_op_number(tv1, tv2, op);
+  }
+
+  return tv_op_string(tv1, tv2, op);
+}
+
+/// Handle "f1 += f2", "f1 -= f2", "f1 *= f2", "f1 /= f2".
+/// Returns OK or FAIL.
+static int tv_op_float(typval_T *tv1, const typval_T *tv2, const char *op)
+  FUNC_ATTR_NONNULL_ALL
+{
+  if (*op == '%' || *op == '.'
+      || (tv2->v_type != VAR_FLOAT
+          && tv2->v_type !