Merge #4607 from ZyX-I/luaviml'/lua'/encode_vim_to_object

Remove recursion from various serializers/converters

12 files changed, 736 insertions, 359 deletions

diff --git a/src/clint.py b/src/clint.py
index 80e4c4ac39..efc5f18378 100755
--- a/src/clint.py
+++ b/src/clint.py
@@ -2515,7 +2515,8 @@ def CheckSpacing(filename, clean_lines, linenum, nesting_state, error):
               'after the hash')
 
     cast_line = re.sub(r'^# *define +\w+\([^)]*\)', '', line)
-    match = Search(r'\((?:const )?(?:struct )?[a-zA-Z_]\w*(?: *\*(?:const)?)*\)'
+    match = Search(r'(?<!\bkvec_t)'
+                   r'\((?:const )?(?:struct )?[a-zA-Z_]\w*(?: *\*(?:const)?)*\)'
                    r' +'
                    r'-?(?:\*+|&)?(?:\w+|\+\+|--|\()', cast_line)
     if match and line[0] == ' ':
diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index 80a88becf4..5fb95a163f 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -41,6 +41,12 @@ typedef bool Boolean;
 typedef int64_t Integer;
 typedef double Float;
 
+/// Maximum value of an Integer
+#define API_INTEGER_MAX INT64_MAX
+
+/// Minimum value of an Integer
+#define API_INTEGER_MIN INT64_MIN
+
 typedef struct {
   char *data;
   size_t size;
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index db3e499427..c88bf2127a 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -17,6 +17,13 @@
 #include "nvim/map.h"
 #include "nvim/option.h"
 #include "nvim/option_defs.h"
+#include "nvim/eval/typval_encode.h"
+#include "nvim/lib/kvec.h"
+
+/// Helper structure for vim_to_object
+typedef struct {
+  kvec_t(Object) stack;  ///< Object stack.
+} EncodedData;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "api/private/helpers.c.generated.h"
@@ -310,6 +317,179 @@ void set_option_to(void *to, int type, String name, Object value, Error *err)
   }
 }
 
+#define TYPVAL_ENCODE_ALLOW_SPECIALS false
+
+#define TYPVAL_ENCODE_CONV_NIL() \
+    kv_push(edata->stack, NIL)
+
+#define TYPVAL_ENCODE_CONV_BOOL(num) \
+    kv_push(edata->stack, BOOLEAN_OBJ((Boolean)(num)))
+
+#define TYPVAL_ENCODE_CONV_NUMBER(num) \
+    kv_push(edata->stack, INTEGER_OBJ((Integer)(num)))
+
+#define TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER TYPVAL_ENCODE_CONV_NUMBER
+
+#define TYPVAL_ENCODE_CONV_FLOAT(flt) \
+    kv_push(edata->stack, FLOATING_OBJ((Float)(flt)))
+
+#define TYPVAL_ENCODE_CONV_STRING(str, len) \
+    do { \
+      const size_t len_ = (size_t)(len); \
+      const char *const str_ = (const char *)(str); \
+      assert(len_ == 0 || str_ != NULL); \
+      kv_push(edata->stack, STRING_OBJ(((String) { \
+        .data = xmemdupz((len_?str_:""), len_), \
+        .size = len_ \
+      }))); \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_STR_STRING TYPVAL_ENCODE_CONV_STRING
+
+#define TYPVAL_ENCODE_CONV_EXT_STRING(str, len, type) \
+    TYPVAL_ENCODE_CONV_NIL()
+
+#define TYPVAL_ENCODE_CONV_FUNC(fun) \
+    TYPVAL_ENCODE_CONV_NIL()
+
+#define TYPVAL_ENCODE_CONV_EMPTY_LIST() \
+    kv_push(edata->stack, ARRAY_OBJ(((Array) { .capacity = 0, .size = 0 })))
+
+#define TYPVAL_ENCODE_CONV_EMPTY_DICT() \
+    kv_push(edata->stack, \
+            DICTIONARY_OBJ(((Dictionary) { .capacity = 0, .size = 0 })))
+
+static inline void typval_encode_list_start(EncodedData *const edata,
+                                            const size_t len)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  const Object obj = OBJECT_INIT;
+  kv_push(edata->stack, ARRAY_OBJ(((Array) {
+    .capacity = len,
+    .size = 0,
+    .items = xmalloc(len * sizeof(*obj.data.array.items)),
+  })));
+}
+
+#define TYPVAL_ENCODE_CONV_LIST_START(len) \
+    typval_encode_list_start(edata, (size_t)(len))
+
+static inline void typval_encode_between_list_items(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  Object item = kv_pop(edata->stack);
+  Object *const list = &kv_last(edata->stack);
+  assert(list->type == kObjectTypeArray);
+  assert(list->data.array.size < list->data.array.capacity);
+  list->data.array.items[list->data.array.size++] = item;
+}
+
+#define TYPVAL_ENCODE_CONV_LIST_BETWEEN_ITEMS() \
+    typval_encode_between_list_items(edata)
+
+static inline void typval_encode_list_end(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  typval_encode_between_list_items(edata);
+#ifndef NDEBUG
+  const Object *const list = &kv_last(edata->stack);
+  assert(list->data.array.size == list->data.array.capacity);
+#endif
+}
+
+#define TYPVAL_ENCODE_CONV_LIST_END() \
+    typval_encode_list_end(edata)
+
+static inline void typval_encode_dict_start(EncodedData *const edata,
+                                            const size_t len)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  const Object obj = OBJECT_INIT;
+  kv_push(edata->stack, DICTIONARY_OBJ(((Dictionary) {
+    .capacity = len,
+    .size = 0,
+    .items = xmalloc(len * sizeof(*obj.data.dictionary.items)),
+  })));
+}
+
+#define TYPVAL_ENCODE_CONV_DICT_START(len) \
+    typval_encode_dict_start(edata, (size_t)(len))
+
+#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, kv_pair)
+
+static inline void typval_encode_after_key(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  Object key = kv_pop(edata->stack);
+  Object *const dict = &kv_last(edata->stack);
+  assert(dict->type == kObjectTypeDictionary);
+  assert(dict->data.dictionary.size < dict->data.dictionary.capacity);
+  if (key.type == kObjectTypeString) {
+    dict->data.dictionary.items[dict->data.dictionary.size].key
+        = key.data.string;
+  } else {
+    api_free_object(key);
+    dict->data.dictionary.items[dict->data.dictionary.size].key
+        = STATIC_CSTR_TO_STRING("__INVALID_KEY__");
+  }
+}
+
+#define TYPVAL_ENCODE_CONV_DICT_AFTER_KEY() \
+    typval_encode_after_key(edata)
+
+static inline void typval_encode_between_dict_items(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  Object val = kv_pop(edata->stack);
+  Object *const dict = &kv_last(edata->stack);
+  assert(dict->type == kObjectTypeDictionary);
+  assert(dict->data.dictionary.size < dict->data.dictionary.capacity);
+  dict->data.dictionary.items[dict->data.dictionary.size++].value = val;
+}
+
+#define TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS() \
+    typval_encode_between_dict_items(edata)
+
+static inline void typval_encode_dict_end(EncodedData *const edata)
+  FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
+{
+  typval_encode_between_dict_items(edata);
+#ifndef NDEBUG
+  const Object *const dict = &kv_last(edata->stack);
+  assert(dict->data.dictionary.size == dict->data.dictionary.capacity);
+#endif
+}
+
+#define TYPVAL_ENCODE_CONV_DICT_END() \
+    typval_encode_dict_end(edata)
+
+#define TYPVAL_ENCODE_CONV_RECURSE(val, conv_type) \
+    TYPVAL_ENCODE_CONV_NIL()
+
+TYPVAL_ENCODE_DEFINE_CONV_FUNCTIONS(static, object, EncodedData *const, edata)
+
+#undef TYPVAL_ENCODE_CONV_STRING
+#undef TYPVAL_ENCODE_CONV_STR_STRING
+#undef TYPVAL_ENCODE_CONV_EXT_STRING
+#undef TYPVAL_ENCODE_CONV_NUMBER
+#undef TYPVAL_ENCODE_CONV_FLOAT
+#undef TYPVAL_ENCODE_CONV_FUNC
+#undef TYPVAL_ENCODE_CONV_EMPTY_LIST
+#undef TYPVAL_ENCODE_CONV_LIST_START
+#undef TYPVAL_ENCODE_CONV_EMPTY_DICT
+#undef TYPVAL_ENCODE_CONV_NIL
+#undef TYPVAL_ENCODE_CONV_BOOL
+#undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER
+#undef TYPVAL_ENCODE_CONV_DICT_START
+#undef TYPVAL_ENCODE_CONV_DICT_END
+#undef TYPVAL_ENCODE_CONV_DICT_AFTER_KEY
+#undef TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS
+#undef TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK
+#undef TYPVAL_ENCODE_CONV_LIST_END
+#undef TYPVAL_ENCODE_CONV_LIST_BETWEEN_ITEMS
+#undef TYPVAL_ENCODE_CONV_RECURSE
+#undef TYPVAL_ENCODE_ALLOW_SPECIALS
+
 /// Convert a vim object to an `Object` instance, recursively expanding
 /// Arrays/Dictionaries.
 ///
@@ -317,13 +497,12 @@ void set_option_to(void *to, int type, String name, Object value, Error *err)
 /// @return The converted value
 Object vim_to_object(typval_T *obj)
 {
-  Object rv;
-  // We use a lookup table to break out of cyclic references
-  PMap(ptr_t) *lookup = pmap_new(ptr_t)();
-  rv = vim_to_object_rec(obj, lookup);
-  // Free the table
-  pmap_free(ptr_t)(lookup);
-  return rv;
+  EncodedData edata = { .stack = KV_INITIAL_VALUE };
+  encode_vim_to_object(&edata, obj, "vim_to_object argument");
+  Object ret = kv_A(edata.stack, 0);
+  assert(kv_size(edata.stack) == 1);
+  kv_destroy(edata.stack);
+  return ret;
 }
 
 buf_T *find_buffer_by_handle(Buffer buffer, Error *err)
@@ -633,131 +812,6 @@ Object copy_object(Object obj)
   }
 }
 
-/// Recursion helper for the `vim_to_object`. This uses a pointer table
-/// to avoid infinite recursion due to cyclic references
-///
-/// @param obj The source object
-/// @param lookup Lookup table containing pointers to all processed objects
-/// @return The converted value
-static Object vim_to_object_rec(typval_T *obj, PMap(ptr_t) *lookup)
-{
-  Object rv = OBJECT_INIT;
-
-  if (obj->v_type == VAR_LIST || obj->v_type == VAR_DICT) {
-    // Container object, add it to the lookup table
-    if (pmap_has(ptr_t)(lookup, obj)) {
-      // It's already present, meaning we alredy processed it so just return
-      // nil instead.
-      return rv;
-    }
-    pmap_put(ptr_t)(lookup, obj, NULL);
-  }
-
-  switch (obj->v_type) {
-    case VAR_SPECIAL:
-      switch (obj->vval.v_special) {
-        case kSpecialVarTrue:
-        case kSpecialVarFalse: {
-          rv.type = kObjectTypeBoolean;
-          rv.data.boolean = (obj->vval.v_special == kSpecialVarTrue);
-          break;
-        }
-        case kSpecialVarNull: {
-          rv.type = kObjectTypeNil;
-          break;
-        }
-      }
-      break;
-
-    case VAR_STRING:
-      rv.type = kObjectTypeString;
-      rv.data.string = cstr_to_string((char *) obj->vval.v_string);
-      break;
-
-    case VAR_NUMBER:
-      rv.type = kObjectTypeInteger;
-      rv.data.integer = obj->vval.v_number;
-      break;
-
-    case VAR_FLOAT:
-      rv.type = kObjectTypeFloat;
-      rv.data.floating = obj->vval.v_float;
-      break;
-
-    case VAR_LIST:
-      {
-        list_T *list = obj->vval.v_list;
-        listitem_T *item;
-
-        if (list != NULL) {
-          rv.type = kObjectTypeArray;
-          assert(list->lv_len >= 0);
-          rv.data.array.size = (size_t)list->lv_len;
-          rv.data.array.items = xmalloc(rv.data.array.size * sizeof(Object));
-
-          uint32_t i = 0;
-          for (item = list->lv_first; item != NULL; item = item->li_next) {
-            rv.data.array.items[i] = vim_to_object_rec(&item->li_tv, lookup);
-            i++;
-          }
-        }
-      }
-      break;
-
-    case VAR_DICT:
-      {
-        dict_T *dict = obj->vval.v_dict;
-        hashtab_T *ht;
-        uint64_t todo;
-        hashitem_T *hi;
-        dictitem_T *di;
-
-        if (dict != NULL) {
-          ht = &obj->vval.v_dict->dv_hashtab;
-          todo = ht->ht_used;
-          rv.type = kObjectTypeDictionary;
-
-          // Count items
-          rv.data.dictionary.size = 0;
-          for (hi = ht->ht_array; todo > 0; ++hi) {
-            if (!HASHITEM_EMPTY(hi)) {
-              todo--;
-              rv.data.dictionary.size++;
-            }
-          }
-
-          rv.data.dictionary.items =
-            xmalloc(rv.data.dictionary.size * sizeof(KeyValuePair));
-          todo = ht->ht_used;
-          uint32_t i = 0;
-
-          // Convert all
-          for (hi = ht->ht_array; todo > 0; ++hi) {
-            if (!HASHITEM_EMPTY(hi)) {
-              di = dict_lookup(hi);
-              // Convert key
-              rv.data.dictionary.items[i].key =
-                cstr_to_string((char *) hi->hi_key);
-              // Convert value
-              rv.data.dictionary.items[i].value =
-                vim_to_object_rec(&di->di_tv, lookup);
-              todo--;
-              i++;
-            }
-          }
-        }
-      }
-      break;
-
-    case VAR_UNKNOWN:
-    case VAR_FUNC:
-      break;
-  }
-
-  return rv;
-}
-
-
 static void set_option_value_for(char *key,
                                  int numval,
                                  char *stringval,
diff --git a/src/nvim/api/private/helpers.h b/src/nvim/api/private/helpers.h
index 731f186ecc..a946e35149 100644
--- a/src/nvim/api/private/helpers.h
+++ b/src/nvim/api/private/helpers.h
@@ -27,6 +27,10 @@
     .type = kObjectTypeInteger, \
     .data.integer = i })
 
+#define FLOATING_OBJ(f) ((Object) { \
+    .type = kObjectTypeFloat, \
+    .data.floating = f })
+
 #define STRING_OBJ(s) ((Object) { \
     .type = kObjectTypeString, \
     .data.string = s })
@@ -61,6 +65,13 @@
 
 #define STATIC_CSTR_AS_STRING(s) ((String) {.data = s, .size = sizeof(s) - 1})
 
+/// Create a new String instance, putting data in allocated memory
+///
+/// @param[in]  s  String to work with. Must be a string literal.
+#define STATIC_CSTR_TO_STRING(s) ((String){ \
+    .data = xmemdupz(s, sizeof(s) - 1), \
+    .size = sizeof(s) - 1 })
+
 // Helpers used by the generated msgpack-rpc api wrappers
 #define api_init_boolean
 #define api_init_integer
diff --git a/src/nvim/eval.c b/src/nvim/eval.c
index aa5cd676af..f5cffbc3e1 100644
--- a/src/nvim/eval.c
+++ b/src/nvim/eval.c
@@ -91,6 +91,7 @@
 #include "nvim/os/input.h"
 #include "nvim/event/loop.h"
 #include "nvim/lib/queue.h"
+#include "nvim/eval/typval_encode.h"
 
 #define DICT_MAXNEST 100        /* maximum nesting of lists and dicts */
 
@@ -18148,45 +18149,147 @@ void free_tv(typval_T *varp)
   }
 }
 
-/*
- * Free the memory for a variable value and set the value to NULL or 0.
- */
+#define TYPVAL_ENCODE_ALLOW_SPECIALS false
+
+#define TYPVAL_ENCODE_CONV_NIL() \
+    do { \
+      tv->vval.v_special = kSpecialVarFalse; \
+      tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_BOOL(ignored) \
+    TYPVAL_ENCODE_CONV_NIL()
+
+#define TYPVAL_ENCODE_CONV_NUMBER(ignored) \
+    do { \
+      (void)ignored; \
+      tv->vval.v_number = 0; \
+      tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER(ignored) \
+    assert(false)
+
+#define TYPVAL_ENCODE_CONV_FLOAT(ignored) \
+    do { \
+      tv->vval.v_float = 0; \
+      tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_STRING(str, ignored) \
+    do { \
+      xfree(str); \
+      tv->vval.v_string = NULL; \
+      tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_STR_STRING(ignored1, ignored2)
+
+#define TYPVAL_ENCODE_CONV_EXT_STRING(ignored1, ignored2, ignored3)
+
+#define TYPVAL_ENCODE_CONV_FUNC(fun) \
+    do { \
+      func_unref(fun); \
+      if (fun != empty_string) { \
+        xfree(fun); \
+      } \
+      tv->vval.v_string = NULL; \
+      tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_EMPTY_LIST() \
+    do { \
+      list_unref(tv->vval.v_list); \
+      tv->vval.v_list = NULL; \
+      tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_EMPTY_DICT() \
+    do { \
+      dict_unref(tv->vval.v_dict); \
+      tv->vval.v_dict = NULL; \
+      tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_LIST_START(ignored) \
+    do { \
+      if (tv->vval.v_list->lv_refcount > 1) { \
+        tv->vval.v_list->lv_refcount--; \
+        tv->vval.v_list = NULL; \
+        tv->v_lock = VAR_UNLOCKED; \
+        return OK; \
+      } \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_LIST_BETWEEN_ITEMS()
+
+#define TYPVAL_ENCODE_CONV_LIST_END() \
+    do { \
+      typval_T *const cur_tv = cur_mpsv->tv; \
+      assert(cur_tv->v_type == VAR_LIST); \
+      list_unref(cur_tv->vval.v_list); \
+      cur_tv->vval.v_list = NULL; \
+      cur_tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_DICT_START(ignored) \
+    do { \
+      if (tv->vval.v_dict->dv_refcount > 1) { \
+        tv->vval.v_dict->dv_refcount--; \
+        tv->vval.v_dict = NULL; \
+        tv->v_lock = VAR_UNLOCKED; \
+        return OK; \
+      } \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(ignored1, ignored2)
+
+#define TYPVAL_ENCODE_CONV_DICT_AFTER_KEY()
+
+#define TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS()
+
+#define TYPVAL_ENCODE_CONV_DICT_END() \
+    do { \
+      typval_T *const cur_tv = cur_mpsv->tv; \
+      assert(cur_tv->v_type == VAR_DICT); \
+      dict_unref(cur_tv->vval.v_dict); \
+      cur_tv->vval.v_dict = NULL; \
+      cur_tv->v_lock = VAR_UNLOCKED; \
+    } while (0)
+
+#define TYPVAL_ENCODE_CONV_RECURSE(ignored1, ignored2)
+
+TYPVAL_ENCODE_DEFINE_CONV_FUNCTIONS(static, nothing, void *, ignored)
+
+#undef TYPVAL_ENCODE_ALLOW_SPECIALS
+#undef TYPVAL_ENCODE_CONV_NIL
+#undef TYPVAL_ENCODE_CONV_BOOL
+#undef TYPVAL_ENCODE_CONV_NUMBER
+#undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER
+#undef TYPVAL_ENCODE_CONV_FLOAT
+#undef TYPVAL_ENCODE_CONV_STRING
+#undef TYPVAL_ENCODE_CONV_STR_STRING
+#undef TYPVAL_ENCODE_CONV_EXT_STRING
+#undef TYPVAL_ENCODE_CONV_FUNC
+#undef TYPVAL_ENCODE_CONV_EMPTY_LIST
+#undef TYPVAL_ENCODE_CONV_EMPTY_DICT
+#undef TYPVAL_ENCODE_CONV_LIST_START
+#undef TYPVAL_ENCODE_CONV_LIST_BETWEEN_ITEMS
+#undef TYPVAL_ENCODE_CONV_LIST_END
+#undef TYPVAL_ENCODE_CONV_DICT_START
+#undef TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK
+#undef TYPVAL_ENCODE_CONV_DICT_AFTER_KEY
+#undef TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS
+#undef TYPVAL_ENCODE_CONV_DICT_END
+#undef TYPVAL_ENCODE_CONV_RECURSE
+
+/// Free memory for a variable value and set the value to NULL or 0
+///
+/// @param[in,out]  varp  Value to free.
 void clear_tv(typval_T *varp)
 {
-  if (varp != NULL) {
-    switch (varp->v_type) {
-    case VAR_FUNC:
-      func_unref(varp->vval.v_string);
-      if (varp->vval.v_string != empty_string) {
-        xfree(varp->vval.v_string);
-      }
-      varp->vval.v_string = NULL;
-      break;
-    case VAR_STRING:
-      xfree(varp->vval.v_string);
-      varp->vval.v_string = NULL;
-      break;
-    case VAR_LIST:
-      list_unref(varp->vval.v_list);
-      varp->vval.v_list = NULL;
-      break;
-    case VAR_DICT:
-      dict_unref(varp->vval.v_dict);
-      varp->vval.v_dict = NULL;
-      break;
-    case VAR_NUMBER:
-      varp->vval.v_number = 0;
-      break;
-    case VAR_FLOAT:
-      varp->vval.v_float = 0.0;
-      break;
-    case VAR_SPECIAL:
-      varp->vval.v_special = kSpecialVarFalse;
-      break;
-    case VAR_UNKNOWN:
-      break;
-    }
-    varp->v_lock = 0;
+  if (varp != NULL && varp->v_type != VAR_UNKNOWN) {
+    encode_vim_to_nothing(varp, varp, "clear_tv argument");
   }
 }
 
diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c
index 54daf7557e..670437ceda 100644
--- a/src/nvim/eval/encode.c
+++ b/src/nvim/eval/encode.c
@@ -344,7 +344,7 @@ int encode_read_from_list(ListReaderState *const state, char *const buf,
 #define TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS() \
     ga_concat(gap, ", ")
 
-#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, kv_pair)
+#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, key)
 
 #define TYPVAL_ENCODE_CONV_LIST_END() \
     ga_append(gap, ']')
@@ -379,7 +379,6 @@ int encode_read_from_list(ListReaderState *const state, char *const buf,
       } \
       vim_snprintf(ebuf, ARRAY_SIZE(ebuf), "{E724@%zu}", backref); \
       ga_concat(gap, &ebuf[0]); \
-      return OK; \
     } while (0)
 
 #define TYPVAL_ENCODE_ALLOW_SPECIALS false
@@ -426,7 +425,6 @@ TYPVAL_ENCODE_DEFINE_CONV_FUNCTIONS(, echo, garray_T *const, gap)
         EMSG(_("E724: unable to correctly dump variable " \
                "with self-referencing container")); \
       } \
-      return OK; \
     } while (0)
 
 #undef TYPVAL_ENCODE_ALLOW_SPECIALS
@@ -662,9 +660,8 @@ static inline int convert_to_json_string(garray_T *const gap,
 /// Check whether given key can be used in json_encode()
 ///
 /// @param[in]  tv  Key to check.
-static inline bool check_json_key(const typval_T *const tv)
+bool encode_check_json_key(const typval_T *const tv)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_PURE
-  FUNC_ATTR_ALWAYS_INLINE
 {
   if (tv->v_type == VAR_STRING) {
     return true;
@@ -699,9 +696,9 @@ static inline bool check_json_key(const typval_T *const tv)
 }
 
 #undef TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK
-#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, kv_pair) \
+#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, key) \
     do { \
-      if (!check_json_key(&kv_pair->lv_first->li_tv)) { \
+      if (!encode_check_json_key(&key)) { \
         EMSG(_("E474: Invalid key in special dictionary")); \
         goto label; \
       } \
@@ -876,7 +873,7 @@ char *encode_tv2json(typval_T *tv, size_t *len)
 
 #define TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS()
 
-#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, kv_pair)
+#define TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK(label, key)
 
 #define TYPVAL_ENCODE_CONV_LIST_END()
 
diff --git a/src/nvim/eval/typval_encode.h b/src/nvim/eval/typval_encode.h
index f70a6c9e94..98fa7b26c6 100644
--- a/src/nvim/eval/typval_encode.h
+++ b/src/nvim/eval/typval_encode.h
@@ -104,7 +104,7 @@
 /// @brief Macros used to check special dictionary key
 ///
 /// @param  label  Label for goto in case check was not successfull.
-/// @param  kv_pair  List with two elements: key and value.
+/// @param  key  typval_T key to check.
 
 /// @def TYPVAL_ENCODE_CONV_DICT_AFTER_KEY
 /// @brief Macros used after finishing converting dictionary key
@@ -154,6 +154,7 @@ typedef enum {
 /// Structure representing current VimL to messagepack conversion state
 typedef struct {
   MPConvStackValType type;  ///< Type of the stack entry.
+  typval_T *tv;  ///< Currently converted typval_T.
   union {
     struct {
       dict_T *dict;    ///< Currently converted dictionary.
@@ -168,13 +169,13 @@ typedef struct {
 } MPConvStackVal;
 
 /// Stack used to convert VimL values to messagepack.
-typedef kvec_t(MPConvStackVal) MPConvStack;
+typedef kvec_withinit_t(MPConvStackVal, 8) MPConvStack;
 
 // Defines for MPConvStack
 #define _mp_size kv_size
-#define _mp_init kv_init
-#define _mp_destroy kv_destroy
-#define _mp_push kv_push
+#define _mp_init kvi_init
+#define _mp_destroy kvi_destroy
+#define _mp_push kvi_push
 #define _mp_pop kv_pop
 #define _mp_last kv_last
 
@@ -184,10 +185,12 @@ typedef kvec_t(MPConvStackVal) MPConvStack;
 /// @param  copyID_attr  Name of the container attribute that holds copyID.
 ///                      After checking whether value of this attribute is
 ///                      copyID (variable) it is set to copyID.
+/// @param  conv_type  Type of the conversion, @see MPConvStackValType.
 #define _TYPVAL_ENCODE_CHECK_SELF_REFERENCE(val, copyID_attr, conv_type) \
     do { \
       if ((val)->copyID_attr == copyID) { \
         TYPVAL_ENCODE_CONV_RECURSE((val), conv_type); \
+        return OK; \
       } \
       (val)->copyID_attr = copyID; \
     } while (0)
@@ -215,10 +218,10 @@ static inline size_t tv_strlen(const typval_T *const tv)
 /// tv)` which returns OK or FAIL and helper functions.
 ///
 /// @param  scope  Scope of the main function: either nothing or `static`.
+/// @param  name  Name of the target converter.
 /// @param  firstargtype  Type of the first argument. It will be used to return
 ///                       the results.
 /// @param  firstargname  Name of the first argument.
-/// @param  name  Name of the target converter.
 #define TYPVAL_ENCODE_DEFINE_CONV_FUNCTIONS(scope, name, firstargtype, \
                                             firstargname) \
 static int name##_convert_one_value(firstargtype firstargname, \
@@ -255,6 +258,7 @@ static int name##_convert_one_value(firstargtype firstargname, \
       TYPVAL_ENCODE_CONV_LIST_START(tv->vval.v_list->lv_len); \
       _mp_push(*mpstack, ((MPConvStackVal) { \
         .type = kMPConvList, \
+        .tv = tv, \
         .data = { \
           .l = { \
             .list = tv->vval.v_list, \
@@ -385,6 +389,7 @@ static int name##_convert_one_value(firstargtype firstargname, \
                                                 lv_copyID, kMPConvList); \
             TYPVAL_ENCODE_CONV_LIST_START(val_di->di_tv.vval.v_list->lv_len); \
             _mp_push(*mpstack, ((MPConvStackVal) { \
+              .tv = tv, \
               .type = kMPConvList, \
               .data = { \
                 .l = { \
@@ -415,6 +420,7 @@ static int name##_convert_one_value(firstargtype firstargname, \
                                                 kMPConvPairs); \
             TYPVAL_ENCODE_CONV_DICT_START(val_list->lv_len); \
             _mp_push(*mpstack, ((MPConvStackVal) { \
+              .tv = tv, \
               .type = kMPConvPairs, \
               .data = { \
                 .l = { \
@@ -455,6 +461,7 @@ name##_convert_one_value_regular_dict: \
                                           kMPConvDict); \
       TYPVAL_ENCODE_CONV_DICT_START(tv->vval.v_dict->dv_hashtab.ht_used); \
       _mp_push(*mpstack, ((MPConvStackVal) { \
+        .tv = tv, \
         .type = kMPConvDict, \
         .data = { \
           .d = { \
@@ -535,7 +542,7 @@ scope int encode_vim_to_##name(firstargtype firstargname, typval_T *const tv, \
         } \
         const list_T *const kv_pair = cur_mpsv->data.l.li->li_tv.vval.v_list; \
         TYPVAL_ENCODE_CONV_SPECIAL_DICT_KEY_CHECK( \
-            encode_vim_to_##name##_error_ret, kv_pair); \
+            encode_vim_to_##name##_error_ret, kv_pair->lv_first->li_tv); \
         if (name##_convert_one_value(firstargname, &mpstack, \
                                      &kv_pair->lv_first->li_tv, copyID, \
                                      objname) == FAIL) { \
diff --git a/src/nvim/eval_defs.h b/src/nvim/eval_defs.h
index 8ffc0c98ce..d5c9b2c1ec 100644
--- a/src/nvim/eval_defs.h
+++ b/src/nvim/eval_defs.h
@@ -155,6 +155,10 @@ typedef struct list_stack_S {
 /// Convert a hashitem key pointer to a dictitem pointer
 #define HIKEY2DI(p)  ((dictitem_T *)(p - offsetof(dictitem_T, di_key)))
 
+/// Convert a hashitem value pointer to a dictitem pointer
+#define HIVAL2DI(p) \
+    ((dictitem_T *)(((char *)p) - offsetof(dictitem_T, di_tv)))
+
 /// Convert a hashitem pointer to a dictitem pointer
 #define HI2DI(hi)     HIKEY2DI((hi)->hi_key)
 
diff --git a/src/nvim/lib/kvec.h b/src/nvim/lib/kvec.h
index 36c91c86b2..584282d773 100644
--- a/src/nvim/lib/kvec.h
+++ b/src/nvim/lib/kvec.h
@@ -134,6 +134,9 @@ static inline void *_memcpy_free(void *const restrict dest,
 
 /// Resize vector with preallocated array
 ///
+/// @note May not resize to an array smaller then init_array: if requested,
+///       init_array will be used.
+///
 /// @param[out]  v  Vector to resize.
 /// @param[in]  s  New size.
 #define kvi_resize(v, s) \
@@ -159,10 +162,10 @@ static inline void *_memcpy_free(void *const restrict dest,
     /* ARRAY_SIZE((v).init_array) is the minimal capacity of this vector. */ \
     /* Thus when vector is full capacity may not be zero and it is safe */ \
     /* not to bother with checking whether (v).capacity is 0. But now */ \
-    /* capacity is not guaranteed to have size that is a power of 2. */ \
-    kvi_resize(v, ((v).capacity == ARRAY_SIZE((v).init_array) \
-                   ? ((v).capacity++, kv_roundup32((v).capacity)) \
-                   : (v).capacity << 1))
+    /* capacity is not guaranteed to have size that is a power of 2, it is */ \
+    /* hard to fix this here and is not very necessary if users will use */ \
+    /* 2^x initial array size. */ \
+    kvi_resize(v, (v).capacity << 1)
 
 /// Get location where to store new element to a vector with preallocated array
 ///
diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 0d7d5a247e..5b249ee1c7 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -816,20 +816,55 @@ static void decref(Channel *channel)
 #define REQ "[request]      "
 #define RES "[response]     "
 #define NOT "[notification] "
+#define ERR "[error]        "
+
+// Cannot define array with negative offsets, so this one is needed to be added
+// to MSGPACK_UNPACK_\* values.
+#define MUR_OFF 2
+
+static const char *const msgpack_error_messages[] = {
+  [MSGPACK_UNPACK_EXTRA_BYTES + MUR_OFF] = "extra bytes found",
+  [MSGPACK_UNPACK_CONTINUE + MUR_OFF] = "incomplete string",
+  [MSGPACK_UNPACK_PARSE_ERROR + MUR_OFF] = "parse error",
+  [MSGPACK_UNPACK_NOMEM_ERROR + MUR_OFF] = "not enough memory",
+};
 
 static void log_server_msg(uint64_t channel_id,
                            msgpack_sbuffer *packed)
 {
   msgpack_unpacked unpacked;
   msgpack_unpacked_init(&unpacked);
-  msgpack_unpack_next(&unpacked, packed->data, packed->size, NULL);
-  uint64_t type = unpacked.data.via.array.ptr[0].via.u64;
   DLOGN("[msgpack-rpc] nvim -> client(%" PRIu64 ") ", channel_id);
-  log_lock();
-  FILE *f = open_log_file();
-  fprintf(f, type ? (type == 1 ? RES : NOT) : REQ);
-  log_msg_close(f, unpacked.data);
-  msgpack_unpacked_destroy(&unpacked);
+  const msgpack_unpack_return result =
+      msgpack_unpack_next(&unpacked, packed->data, packed->size, NULL);
+  switch (result) {
+    case MSGPACK_UNPACK_SUCCESS: {
+      uint64_t type = unpacked.data.via.array.ptr[0].via.u64;
+      log_lock();
+      FILE *f = open_log_file();
+      fprintf(f, type ? (type == 1 ? RES : NOT) : REQ);
+      log_msg_close(f, unpacked.data);
+      msgpack_unpacked_destroy(&unpacked);
+      break;
+    }
+    case MSGPACK_UNPACK_EXTRA_BYTES:
+    case MSGPACK_UNPACK_CONTINUE:
+    case MSGPACK_UNPACK_PARSE_ERROR:
+    case MSGPACK_UNPACK_NOMEM_ERROR: {
+      log_lock();
+      FILE *f = open_log_file();
+      fprintf(f, ERR);
+      log_msg_close(f, (msgpack_object) {
+          .type = MSGPACK_OBJECT_STR,
+          .via.str = {
+              .ptr = (char *)msgpack_error_messages[result + MUR_OFF],
+              .size = (uint32_t)strlen(
+                  msgpack_error_messages[result + MUR_OFF]),
+          },
+      });
+      break;
+    }
+  }
 }
 
 static void log_client_msg(uint64_t channel_id,
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index c643bae0e8..9195b10614 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -7,9 +7,11 @@
 #include "nvim/api/private/helpers.h"
 #include "nvim/msgpack_rpc/helpers.h"
 #include "nvim/msgpack_rpc/defs.h"
+#include "nvim/lib/kvec.h"
 #include "nvim/vim.h"
 #include "nvim/log.h"
 #include "nvim/memory.h"
+#include "nvim/assert.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "msgpack_rpc/helpers.c.generated.h"
@@ -19,7 +21,7 @@ static msgpack_zone zone;
 static msgpack_sbuffer sbuffer;
 
 #define HANDLE_TYPE_CONVERSION_IMPL(t, lt) \
-  bool msgpack_rpc_to_##lt(msgpack_object *obj, t *arg) \
+  bool msgpack_rpc_to_##lt(const msgpack_object *const obj, t *const arg) \
     FUNC_ATTR_NONNULL_ALL \
   { \
     if (obj->type != MSGPACK_OBJECT_EXT \
@@ -63,34 +65,182 @@ HANDLE_TYPE_CONVERSION_IMPL(Buffer, buffer)
 HANDLE_TYPE_CONVERSION_IMPL(Window, window)
 HANDLE_TYPE_CONVERSION_IMPL(Tabpage, tabpage)
 
-bool msgpack_rpc_to_boolean(msgpack_object *obj, Boolean *arg)
+typedef struct {
+  const msgpack_object *mobj;
+  Object *aobj;
+  bool container;
+  size_t idx;
+} MPToAPIObjectStackItem;
+
+/// Convert type used by msgpack parser to Neovim own API type
+///
+/// @param[in]  obj  Msgpack value to convert.
+/// @param[out]  arg  Location where result of conversion will be saved.
+///
+/// @return true in case of success, false otherwise.
+bool msgpack_rpc_to_object(const msgpack_object *const obj, Object *const arg)
   FUNC_ATTR_NONNULL_ALL
 {
-  *arg = obj->via.boolean;
-  return obj->type == MSGPACK_OBJECT_BOOLEAN;
-}
-
-bool msgpack_rpc_to_integer(msgpack_object *obj, Integer *arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  if (obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER
-      && obj->via.u64 <= INT64_MAX) {
-    *arg = (int64_t)obj->via.u64;
-    return true;
+  bool ret = true;
+  kvec_t(MPToAPIObjectStackItem) stack = KV_INITIAL_VALUE;
+  kv_push(stack, ((MPToAPIObjectStackItem) { obj, arg, false, 0 }));
+  while (ret && kv_size(stack)) {
+    MPToAPIObjectStackItem cur = kv_last(stack);
+    if (!cur.container) {
+      *cur.aobj = NIL;
+    }
+    switch (cur.mobj->type) {
+      case MSGPACK_OBJECT_NIL: {
+        break;
+      }
+      case MSGPACK_OBJECT_BOOLEAN: {
+        *cur.aobj = BOOLEAN_OBJ(cur.mobj->via.boolean);
+        break;
+      }
+      case MSGPACK_OBJECT_NEGATIVE_INTEGER: {
+        STATIC_ASSERT(sizeof(Integer) == sizeof(cur.mobj->via.i64),
+                      "Msgpack integer size does not match API integer");
+        *cur.aobj = INTEGER_OBJ(cur.mobj->via.i64);
+        break;
+      }
+      case MSGPACK_OBJECT_POSITIVE_INTEGER: {
+        STATIC_ASSERT(sizeof(Integer) == sizeof(cur.mobj->via.u64),
+                      "Msgpack integer size does not match API integer");
+        if (cur.mobj->via.u64 > API_INTEGER_MAX) {
+          ret = false;
+        } else {
+          *cur.aobj = INTEGER_OBJ((Integer)cur.mobj->via.u64);
+        }
+        break;
+      }
+      case MSGPACK_OBJECT_FLOAT: {
+        STATIC_ASSERT(sizeof(Float) == sizeof(cur.mobj->via.f64),
+                      "Msgpack floating-point size does not match API integer");
+        *cur.aobj = FLOATING_OBJ(cur.mobj->via.f64);
+        break;
+      }
+#define STR_CASE(type, attr, obj, dest, conv) \
+      case type: { \
+        dest = conv(((String) { \
+          .size = obj->via.attr.size, \
+          .data = (obj->via.attr.ptr == NULL || obj->via.attr.size == 0 \
+                   ? NULL \
+                   : xmemdupz(obj->via.attr.ptr, obj->via.attr.size)), \
+        })); \
+        break; \
+      }
+      STR_CASE(MSGPACK_OBJECT_STR, str, cur.mobj, *cur.aobj, STRING_OBJ)
+      STR_CASE(MSGPACK_OBJECT_BIN, bin, cur.mobj, *cur.aobj, STRING_OBJ)
+      case MSGPACK_OBJECT_ARRAY: {
+        const size_t size = cur.mobj->via.array.size;
+        if (cur.container) {
+          if (cur.idx >= size) {
+            (void)kv_pop(stack);
+          } else {
+            const size_t idx = cur.idx;
+            cur.idx++;
+            kv_last(stack) = cur;
+            kv_push(stack, ((MPToAPIObjectStackItem) {
+              .mobj = &cur.mobj->via.array.ptr[idx],
+              .aobj = &cur.aobj->data.array.items[idx],
+              .container = false,
+            }));
+          }
+        } else {
+          *cur.aobj = ARRAY_OBJ(((Array) {
+            .size = size,
+            .capacity = size,
+            .items = (size > 0
+                      ? xcalloc(size, sizeof(*cur.aobj->data.array.items))
+                      : NULL),
+          }));
+          cur.container = true;
+          kv_last(stack) = cur;
+        }
+        break;
+      }
+      case MSGPACK_OBJECT_MAP: {
+        const size_t size = cur.mobj->via.map.size;
+        if (cur.container) {
+          if (cur.idx >= size) {
+            (void)kv_pop(stack);
+          } else {
+            const size_t idx = cur.idx;
+            cur.idx++;
+            kv_last(stack) = cur;
+            const msgpack_object *const key = &cur.mobj->via.map.ptr[idx].key;
+            switch (key->type) {
+#define ID(x) x
+              STR_CASE(MSGPACK_OBJECT_STR, str, key,
+                       cur.aobj->data.dictionary.items[idx].key, ID)
+              STR_CASE(MSGPACK_OBJECT_BIN, bin, key,
+                       cur.aobj->data.dictionary.items[idx].key, ID)
+#undef ID
+              case MSGPACK_OBJECT_NIL:
+              case MSGPACK_OBJECT_BOOLEAN:
+              case MSGPACK_OBJECT_POSITIVE_INTEGER:
+              case MSGPACK_OBJECT_NEGATIVE_INTEGER:
+              case MSGPACK_OBJECT_FLOAT:
+              case MSGPACK_OBJECT_EXT:
+              case MSGPACK_OBJECT_MAP:
+              case MSGPACK_OBJECT_ARRAY: {
+                ret = false;
+                break;
+              }
+            }
+            if (ret) {
+              kv_push(stack, ((MPToAPIObjectStackItem) {
+                .mobj = &cur.mobj->via.map.ptr[idx].val,
+                .aobj = &cur.aobj->data.dictionary.items[idx].value,
+                .container = false,
+              }));
+            }
+          }
+        } else {
+          *cur.aobj = DICTIONARY_OBJ(((Dictionary) {
+            .size = size,
+            .capacity = size,
+            .items = (size > 0
+                      ? xcalloc(size, sizeof(*cur.aobj->data.dictionary.items))
+                      : NULL),
+          }));
+          cur.container = true;
+          kv_last(stack) = cur;
+        }
+        break;
+      }
+      case MSGPACK_OBJECT_EXT: {
+        switch (cur.mobj->via.ext.type) {
+          case kObjectTypeBuffer: {
+            cur.aobj->type = kObjectTypeBuffer;
+            ret = msgpack_rpc_to_buffer(cur.mobj, &cur.aobj->data.buffer);
+            break;
+          }
+          case kObjectTypeWindow: {
+            cur.aobj->type = kObjectTypeWindow;
+            ret = msgpack_rpc_to_window(cur.mobj, &cur.aobj->data.window);
+            break;
+          }
+          case kObjectTypeTabpage: {
+            cur.aobj->type = kObjectTypeTabpage;
+            ret = msgpack_rpc_to_tabpage(cur.mobj, &cur.aobj->data.tabpage);
+            break;
+          }
+        }
+        break;
+      }
+#undef STR_CASE
+    }
+    if (!cur.container) {
+      (void)kv_pop(stack);
+    }
   }
-
-  *arg = obj->via.i64;
-  return obj->type == MSGPACK_OBJECT_NEGATIVE_INTEGER;
+  kv_destroy(stack);
+  return ret;
 }
 
-bool msgpack_rpc_to_float(msgpack_object *obj, Float *arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  *arg = obj->via.f64;
-  return obj->type == MSGPACK_OBJECT_FLOAT;
-}
-
-bool msgpack_rpc_to_string(msgpack_object *obj, String *arg)
+static bool msgpack_rpc_to_string(const msgpack_object *const obj,
+                                  String *const arg)
   FUNC_ATTR_NONNULL_ALL
 {
   if (obj->type == MSGPACK_OBJECT_BIN || obj->type == MSGPACK_OBJECT_STR) {
@@ -103,58 +253,7 @@ bool msgpack_rpc_to_string(msgpack_object *obj, String *arg)
   return false;
 }
 
-bool msgpack_rpc_to_object(msgpack_object *obj, Object *arg)
-  FUNC_ATTR_NONNULL_ALL
-{
-  switch (obj->type) {
-    case MSGPACK_OBJECT_NIL:
-      arg->type = kObjectTypeNil;
-      return true;
-
-    case MSGPACK_OBJECT_BOOLEAN:
-      arg->type = kObjectTypeBoolean;
-      return msgpack_rpc_to_boolean(obj, &arg->data.boolean);
-
-    case MSGPACK_OBJECT_POSITIVE_INTEGER:
-    case MSGPACK_OBJECT_NEGATIVE_INTEGER:
-      arg->type = kObjectTypeInteger;
-      return msgpack_rpc_to_integer(obj, &arg->data.integer);
-
-    case MSGPACK_OBJECT_FLOAT:
-      arg->type = kObjectTypeFloat;
-      return msgpack_rpc_to_float(obj, &arg->data.floating);
-
-    case MSGPACK_OBJECT_BIN:
-    case MSGPACK_OBJECT_STR:
-      arg->type = kObjectTypeString;
-      return msgpack_rpc_to_string(obj, &arg->data.string);
-
-    case MSGPACK_OBJECT_ARRAY:
-      arg->type = kObjectTypeArray;
-      return msgpack_rpc_to_array(obj, &arg->data.array);
-
-    case MSGPACK_OBJECT_MAP:
-      arg->type = kObjectTypeDictionary;
-      return msgpack_rpc_to_dictionary(obj, &arg->data.dictionary);
-
-    case MSGPACK_OBJECT_EXT:
-      switch (obj->via.ext.type) {
-        case kObjectTypeBuffer:
-          arg->type = kObjectTypeBuffer;
-          return msgpack_rpc_to_buffer(obj, &arg->data.buffer);
-        case kObjectTypeWindow:
-          arg->type = kObjectTypeWindow;
-          return msgpack_rpc_to_window(obj, &arg->data.window);
-        case kObjectTypeTabpage:
-          arg->type = kObjectTypeTabpage;
-          return msgpack_rpc_to_tabpage(obj, &arg->data.tabpage);
-      }
-    default:
-      return false;
-  }
-}
-
-bool msgpack_rpc_to_array(msgpack_object *obj, Array *arg)
+bool msgpack_rpc_to_array(const msgpack_object *const obj, Array *const arg)
   FUNC_ATTR_NONNULL_ALL
 {
   if (obj->type != MSGPACK_OBJECT_ARRAY) {
@@ -173,7 +272,8 @@ bool msgpack_rpc_to_array(msgpack_object *obj, Array *arg)
   return true;
 }
 
-bool msgpack_rpc_to_dictionary(msgpack_object *obj, Dictionary *arg)
+bool msgpack_rpc_to_dictionary(const msgpack_object *const obj,
+                               Dictionary *const arg)
   FUNC_ATTR_NONNULL_ALL
 {
   if (obj->type != MSGPACK_OBJECT_MAP) {
@@ -228,50 +328,108 @@ void msgpack_rpc_from_string(String result, msgpack_packer *res)
   msgpack_pack_str_body(res, result.data, result.size);
 }
 
-void msgpack_rpc_from_object(Object result, msgpack_packer *res)
+typedef struct {
+  const Object *aobj;
+  bool container;
+  size_t idx;
+} APIToMPObjectStackItem;
+
+/// Convert type used by Neovim API to msgpack
+///
+/// @param[in]  result  Object to convert.
+/// @param[out]  res  Structure that defines where conversion results are saved.
+///
+/// @return true in case of success, false otherwise.
+void msgpack_rpc_from_object(const Object result, msgpack_packer *const res)
   FUNC_ATTR_NONNULL_ARG(2)
 {
-  switch (result.type) {
-    case kObjectTypeNil:
-      msgpack_pack_nil(res);
-      break;
-
-    case kObjectTypeBoolean:
-      msgpack_rpc_from_boolean(result.data.boolean, res);
-      break;
-
-    case kObjectTypeInteger:
-      msgpack_rpc_from_integer(result.data.integer, res);
-      break;
-
-    case kObjectTypeFloat:
-      msgpack_rpc_from_float(result.data.floating, res);
-      break;
-
-    case kObjectTypeString:
-      msgpack_rpc_from_string(result.data.string, res);
-      break;
-
-    case kObjectTypeArray:
-      msgpack_rpc_from_array(result.data.array, res);
-      break;
-
-    case kObjectTypeBuffer:
-      msgpack_rpc_from_buffer(result.data.buffer, res);
-      break;
-
-    case kObjectTypeWindow:
-      msgpack_rpc_from_window(result.data.window, res);
-      break;
-
-    case kObjectTypeTabpage:
-      msgpack_rpc_from_tabpage(result.data.tabpage, res);
-      break;
-
-    case kObjectTypeDictionary:
-      msgpack_rpc_from_dictionary(result.data.dictionary, res);
-      break;
+  kvec_t(APIToMPObjectStackItem) stack = KV_INITIAL_VALUE;
+  kv_push(stack, ((APIToMPObjectStackItem) { &result, false, 0 }));
+  while (kv_size(stack)) {
+    APIToMPObjectStackItem cur = kv_last(stack);
+    switch (cur.aobj->type) {
+      case kObjectTypeNil: {
+        msgpack_pack_nil(res);
+        break;
+      }
+      case kObjectTypeBoolean: {
+        msgpack_rpc_from_boolean(cur.aobj->data.boolean, res);
+        break;
+      }
+      case kObjectTypeInteger: {
+        msgpack_rpc_from_integer(cur.aobj->data.integer, res);
+        break;
+      }
+      case kObjectTypeFloat: {
+        msgpack_rpc_from_float(cur.aobj->data.floating, res);
+        break;
+      }
+      case kObjectTypeString: {
+        msgpack_rpc_from_string(cur.aobj->data.string, res);
+        break;
+      }
+      case kObjectTypeBuffer: {
+        msgpack_rpc_from_buffer(cur.aobj->data.buffer, res);
+        break;
+      }
+      case kObjectTypeWindow: {
+        msgpack_rpc_from_window(cur.aobj->data.window, res);
+        break;
+      }
+      case kObjectTypeTabpage: {
+        msgpack_rpc_from_tabpage(cur.aobj->data.tabpage, res);
+        break;
+      }
+      case kObjectTypeArray: {
+        const size_t size = cur.aobj->data.array.size;
+        if (cur.container) {
+          if (cur.idx >= size) {
+            (void)kv_pop(stack);
+          } else {
+            const size_t idx = cur.idx;
+            cur.idx++;
+            kv_last(stack) = cur;
+            kv_push(stack, ((APIToMPObjectStackItem) {
+              .aobj = &cur.aobj->data.array.items[idx],
+              .container = false,
+            }));
+          }
+        } else {
+          msgpack_pack_array(res, size);
+          cur.container = true;
+          kv_last(stack) = cur;
+        }
+        break;
+      }
+      case kObjectTypeDictionary: {
+        const size_t size = cur.aobj->data.dictionary.size;
+        if (cur.container) {
+          if (cur.idx >= size) {
+            (void)kv_pop(stack);
+          } else {
+            const size_t idx = cur.idx;
+            cur.idx++;
+            kv_last(stack) = cur;
+            msgpack_rpc_from_string(cur.aobj->data.dictionary.items[idx].key,
+                                    res);
+            kv_push(stack, ((APIToMPObjectStackItem) {
+              .aobj = &cur.aobj->data.dictionary.items[idx].value,
+              .container = false,
+            }));
+          }
+        } else {
+          msgpack_pack_map(res, size);
+          cur.container = true;
+          kv_last(stack) = cur;
+        }
+        break;
+      }
+    }
+    if (!cur.container) {
+      (void)kv_pop(stack);
+    }
   }
+  kv_destroy(stack);
 }
 
 void msgpack_rpc_from_array(Array result, msgpack_packer *res)
diff --git a/src/nvim/option.c b/src/nvim/option.c
index 52a1fd8558..020a119fd3 100644
--- a/src/nvim/option.c
+++ b/src/nvim/option.c
@@ -2314,50 +2314,46 @@ set_string_option_global (
   }
 }
 
-/*
- * Set a string option to a new value, and handle the effects.
- *
- * Returns NULL on success or error message on error.
- */
-static char_u *
-set_string_option (
-    int opt_idx,
-    char_u *value,
-    int opt_flags                  /* OPT_LOCAL and/or OPT_GLOBAL */
-)
+/// Set a string option to a new value, handling the effects
+///
+/// @param[in]  opt_idx  Option to set.
+/// @param[in]  value  New value.
+/// @param[in]  opt_flags  Option flags: expected to contain #OPT_LOCAL and/or
+///                        #OPT_GLOBAL.
+///
+/// @return NULL on success, error message on error.
+static char *set_string_option(const int opt_idx, const char *const value,
+                               const int opt_flags)
+  FUNC_ATTR_NONNULL_ARG(2) FUNC_ATTR_WARN_UNUSED_RESULT
 {
-  char_u      *s;
-  char_u      **varp;
-  char_u      *oldval;
-  char *saved_oldval = NULL;
-  char_u      *r = NULL;
-
-  if (options[opt_idx].var == NULL)     /* don't set hidden option */
+  if (options[opt_idx].var == NULL) {  // don't set hidden option
     return NULL;
+  }
 
-  s = vim_strsave(value);
-  varp = (char_u **)get_varp_scope(&(options[opt_idx]),
-      (opt_flags & (OPT_LOCAL | OPT_GLOBAL)) == 0
-      ? (((int)options[opt_idx].indir & PV_BOTH)
-        ? OPT_GLOBAL : OPT_LOCAL)
-      : opt_flags);
-  oldval = *varp;
+  char *const s = xstrdup(value);
+  char **const varp = (char **)get_varp_scope(
+      &(options[opt_idx]),
+      ((opt_flags & (OPT_LOCAL | OPT_GLOBAL)) == 0
+       ? (((int)options[opt_idx].indir & PV_BOTH)
+          ? OPT_GLOBAL : OPT_LOCAL)
+       : opt_flags));
+  char *const oldval = *varp;
   *varp = s;
 
-  if (!starting) {
-    saved_oldval = xstrdup((char *) oldval);
-  }
+  char *const saved_oldval = (starting ? NULL : xstrdup(oldval));
 
-  if ((r = did_set_string_option(opt_idx, varp, (int)true, oldval, NULL,
-          opt_flags)) == NULL)
-    did_set_option(opt_idx, opt_flags, TRUE);
+  char *const r = (char *)did_set_string_option(
+      opt_idx, (char_u **)varp, (int)true, (char_u *)oldval, NULL, opt_flags);
+  if (r == NULL) {
+    did_set_option(opt_idx, opt_flags, true);
+  }
 
   // call autocommand after handling side effects
   if (saved_oldval != NULL) {
     char buf_type[7];
     vim_snprintf(buf_type, ARRAY_SIZE(buf_type), "%s",
                  (opt_flags & OPT_LOCAL) ? "local" : "global");
-    set_vim_var_string(VV_OPTION_NEW, (char *) (*varp), -1);
+    set_vim_var_string(VV_OPTION_NEW, (char *)(*varp), -1);
     set_vim_var_string(VV_OPTION_OLD, saved_oldval, -1);
     set_vim_var_string(VV_OPTION_TYPE, buf_type, -1);
     apply_autocmds(EVENT_OPTIONSET,
@@ -4655,26 +4651,28 @@ set_option_value (
       EMSG(_(e_sandbox));
       return NULL;
     }
-    if (flags & P_STRING)
-      return set_string_option(opt_idx, string, opt_flags);
-    else {
+    if (flags & P_STRING) {
+      const char *s = (const char *)string;
+      if (s == NULL) {
+        s = "";
+      }
+      return (char_u *)set_string_option(opt_idx, s, opt_flags);
+    } else {
       varp = get_varp_scope(&(options[opt_idx]), opt_flags);
       if (varp != NULL) {       /* hidden option is not changed */
         if (number == 0 && string != NULL) {
           int idx;
 
-          /* Either we are given a string or we are setting option
-           * to zero. */
-          for (idx = 0; string[idx] == '0'; ++idx)
-            ;
+          // Either we are given a string or we are setting option
+          // to zero.
+          for (idx = 0; string[idx] == '0'; idx++) {}
           if (string[idx] != NUL || idx == 0) {
-            /* There's another character after zeros or the string
-             * is empty.  In both cases, we are trying to set a
-             * num option using a string. */
+            // There's another character after zeros or the string
+            // is empty.  In both cases, we are trying to set a
+            // num option using a string.
             EMSG3(_("E521: Number required: &%s = '%s'"),
-                name, string);
-            return NULL;                 /* do nothing as we hit an error */
-
+                  name, string);
+            return NULL;  // do nothing as we hit an error
           }
         }
         if (flags & P_NUM)