Merge remote-tracking branch 'upstream/master' into aucmd_textputpost

8 files changed, 421 insertions, 337 deletions

diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c
index 4598d48c4a..bba771f8a5 100644
--- a/src/nvim/lua/converter.c
+++ b/src/nvim/lua/converter.c
@@ -16,8 +16,8 @@
 #include "nvim/eval/typval_defs.h"
 #include "nvim/eval/typval_encode.h"
 #include "nvim/eval/userfunc.h"
-#include "nvim/func_attr.h"
-#include "nvim/gettext.h"
+#include "nvim/gettext_defs.h"
+#include "nvim/highlight_group.h"
 #include "nvim/lua/converter.h"
 #include "nvim/lua/executor.h"
 #include "nvim/macros_defs.h"
@@ -149,7 +149,7 @@ static LuaTableProps nlua_traverse_table(lua_State *const lstate)
     }
   } else {
     if (tsize == 0
-        || (tsize == ret.maxidx
+        || (tsize <= ret.maxidx
             && other_keys_num == 0
             && ret.string_keys_num == 0)) {
       ret.type = kObjectTypeArray;
@@ -171,11 +171,12 @@ static LuaTableProps nlua_traverse_table(lua_State *const lstate)
 
 /// Helper structure for nlua_pop_typval
 typedef struct {
-  typval_T *tv;  ///< Location where conversion result is saved.
-  bool container;  ///< True if tv is a container.
-  bool special;  ///< If true then tv is a _VAL part of special dictionary
-                 ///< that represents mapping.
-  int idx;  ///< Container index (used to detect self-referencing structures).
+  typval_T *tv;     ///< Location where conversion result is saved.
+  size_t list_len;  ///< Maximum length when tv is a list.
+  bool container;   ///< True if tv is a container.
+  bool special;     ///< If true then tv is a _VAL part of special dictionary
+                    ///< that represents mapping.
+  int idx;          ///< Container index (used to detect self-referencing structures).
 } TVPopStackItem;
 
 /// Convert lua object to Vimscript typval_T
@@ -193,7 +194,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
   const int initial_size = lua_gettop(lstate);
   kvec_withinit_t(TVPopStackItem, 2) stack = KV_INITIAL_VALUE;
   kvi_init(stack);
-  kvi_push(stack, ((TVPopStackItem) { ret_tv, false, false, 0 }));
+  kvi_push(stack, ((TVPopStackItem){ .tv = ret_tv }));
   while (ret && kv_size(stack)) {
     if (!lua_checkstack(lstate, lua_gettop(lstate) + 3)) {
       semsg(_("E1502: Lua failed to grow stack to %i"), lua_gettop(lstate) + 3);
@@ -232,19 +233,14 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
             });
             kvi_push(stack, cur);
             tv_list_append_list(cur.tv->vval.v_list, kv_pair);
-            cur = (TVPopStackItem) {
-              .tv = TV_LIST_ITEM_TV(tv_list_last(kv_pair)),
-              .container = false,
-              .special = false,
-              .idx = 0,
-            };
+            cur = (TVPopStackItem){ .tv = TV_LIST_ITEM_TV(tv_list_last(kv_pair)) };
           } else {
             dictitem_T *const di = tv_dict_item_alloc_len(s, len);
             if (tv_dict_add(cur.tv->vval.v_dict, di) == FAIL) {
               abort();
             }
             kvi_push(stack, cur);
-            cur = (TVPopStackItem) { &di->di_tv, false, false, 0 };
+            cur = (TVPopStackItem){ .tv = &di->di_tv };
           }
         } else {
           lua_pop(lstate, 1);
@@ -252,23 +248,18 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
         }
       } else {
         assert(cur.tv->v_type == VAR_LIST);
-        lua_rawgeti(lstate, -1, tv_list_len(cur.tv->vval.v_list) + 1);
-        if (lua_isnil(lstate, -1)) {
-          lua_pop(lstate, 2);
+        if ((size_t)tv_list_len(cur.tv->vval.v_list) == cur.list_len) {
+          lua_pop(lstate, 1);
           continue;
         }
+        lua_rawgeti(lstate, -1, tv_list_len(cur.tv->vval.v_list) + 1);
         // Not populated yet, need to create list item to push.
         tv_list_append_owned_tv(cur.tv->vval.v_list, (typval_T) {
           .v_type = VAR_UNKNOWN,
         });
         kvi_push(stack, cur);
         // TODO(ZyX-I): Use indexes, here list item *will* be reallocated.
-        cur = (TVPopStackItem) {
-          .tv = TV_LIST_ITEM_TV(tv_list_last(cur.tv->vval.v_list)),
-          .container = false,
-          .special = false,
-          .idx = 0,
-        };
+        cur = (TVPopStackItem){ .tv = TV_LIST_ITEM_TV(tv_list_last(cur.tv->vval.v_list)) };
       }
     }
     assert(!cur.container);
@@ -332,6 +323,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
         cur.tv->vval.v_list = tv_list_alloc((ptrdiff_t)table_props.maxidx);
         cur.tv->vval.v_list->lua_table_ref = table_ref;
         tv_list_ref(cur.tv->vval.v_list);
+        cur.list_len = table_props.maxidx;
         if (table_props.maxidx != 0) {
           cur.container = true;
           cur.idx = lua_gettop(lstate);
@@ -355,6 +347,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
             cur.tv = &val_di->di_tv;
             cur.tv->vval.v_list->lua_table_ref = table_ref;
             assert(cur.tv->v_type == VAR_LIST);
+            cur.list_len = table_props.string_keys_num;
           } else {
             cur.tv->v_type = VAR_DICT;
             cur.tv->vval.v_dict = tv_dict_alloc();
@@ -372,9 +365,8 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
         cur.tv->vval.v_float = (float_T)table_props.val;
         break;
       case kObjectTypeNil:
-        emsg(_("E5100: Cannot convert given lua table: table "
-               "should either have a sequence of positive integer keys "
-               "or contain only string keys"));
+        emsg(_("E5100: Cannot convert given lua table: table should "
+               "contain either only integer keys or only string keys"));
         ret = false;
         break;
       default:
@@ -727,7 +719,7 @@ void nlua_push_Dictionary(lua_State *lstate, const Dictionary dict, bool special
   }
   for (size_t i = 0; i < dict.size; i++) {
     nlua_push_String(lstate, dict.items[i].key, special);
-    nlua_push_Object(lstate, dict.items[i].value, special);
+    nlua_push_Object(lstate, &dict.items[i].value, special);
     lua_rawset(lstate, -3);
   }
 }
@@ -740,7 +732,7 @@ void nlua_push_Array(lua_State *lstate, const Array array, bool special)
 {
   lua_createtable(lstate, (int)array.size, 0);
   for (size_t i = 0; i < array.size; i++) {
-    nlua_push_Object(lstate, array.items[i], special);
+    nlua_push_Object(lstate, &array.items[i], special);
     lua_rawseti(lstate, -2, (int)i + 1);
   }
 }
@@ -761,10 +753,10 @@ GENERATE_INDEX_FUNCTION(Tabpage)
 /// Convert given Object to lua value
 ///
 /// Leaves converted value on top of the stack.
-void nlua_push_Object(lua_State *lstate, const Object obj, bool special)
+void nlua_push_Object(lua_State *lstate, Object *obj, bool special)
   FUNC_ATTR_NONNULL_ALL
 {
-  switch (obj.type) {
+  switch (obj->type) {
   case kObjectTypeNil:
     if (special) {
       lua_pushnil(lstate);
@@ -773,12 +765,14 @@ void nlua_push_Object(lua_State *lstate, const Object obj, bool special)
     }
     break;
   case kObjectTypeLuaRef: {
-    nlua_pushref(lstate, obj.data.luaref);
+    nlua_pushref(lstate, obj->data.luaref);
+    api_free_luaref(obj->data.luaref);
+    obj->data.luaref = LUA_NOREF;
     break;
   }
 #define ADD_TYPE(type, data_key) \
   case kObjectType##type: { \
-      nlua_push_##type(lstate, obj.data.data_key, special); \
+      nlua_push_##type(lstate, obj->data.data_key, special); \
       break; \
   }
     ADD_TYPE(Boolean,      boolean)
@@ -790,7 +784,7 @@ void nlua_push_Object(lua_State *lstate, const Object obj, bool special)
 #undef ADD_TYPE
 #define ADD_REMOTE_TYPE(type) \
   case kObjectType##type: { \
-      nlua_push_##type(lstate, (type)obj.data.integer, special); \
+      nlua_push_##type(lstate, (type)obj->data.integer, special); \
       break; \
   }
     ADD_REMOTE_TYPE(Buffer)
@@ -803,8 +797,8 @@ void nlua_push_Object(lua_State *lstate, const Object obj, bool special)
 /// Convert lua value to string
 ///
 /// Always pops one value from the stack.
-String nlua_pop_String(lua_State *lstate, Error *err)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+String nlua_pop_String(lua_State *lstate, Arena *arena, Error *err)
+  FUNC_ATTR_NONNULL_ARG(1, 3) FUNC_ATTR_WARN_UNUSED_RESULT
 {
   if (lua_type(lstate, -1) != LUA_TSTRING) {
     lua_pop(lstate, 1);
@@ -815,7 +809,10 @@ String nlua_pop_String(lua_State *lstate, Error *err)
 
   ret.data = (char *)lua_tolstring(lstate, -1, &(ret.size));
   assert(ret.data != NULL);
-  ret.data = xmemdupz(ret.data, ret.size);
+  // TODO(bfredl): it would be "nice" to just use the memory of the lua string
+  // directly, although ensuring the lifetime of such strings is a bit tricky
+  // (an API call could invoke nested lua, which triggers GC, and kaboom?)
+  ret.data = arena_memdupz(arena, ret.data, ret.size);
   lua_pop(lstate, 1);
 
   return ret;
@@ -824,8 +821,8 @@ String nlua_pop_String(lua_State *lstate, Error *err)
 /// Convert lua value to integer
 ///
 /// Always pops one value from the stack.
-Integer nlua_pop_Integer(lua_State *lstate, Error *err)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+Integer nlua_pop_Integer(lua_State *lstate, Arena *arena, Error *err)
+  FUNC_ATTR_NONNULL_ARG(1, 3) FUNC_ATTR_WARN_UNUSED_RESULT
 {
   if (lua_type(lstate, -1) != LUA_TNUMBER) {
     lua_pop(lstate, 1);
@@ -848,8 +845,8 @@ Integer nlua_pop_Integer(lua_State *lstate, Error *err)
 /// thus `err` is never set as any lua value can be co-erced into a lua bool
 ///
 /// Always pops one value from the stack.
-Boolean nlua_pop_Boolean(lua_State *lstate, Error *err)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+Boolean nlua_pop_Boolean(lua_State *lstate, Arena *arena, Error *err)
+  FUNC_ATTR_NONNULL_ARG(1, 3) FUNC_ATTR_WARN_UNUSED_RESULT
 {
   const Boolean ret = lua_toboolean(lstate, -1);
   lua_pop(lstate, 1);
@@ -923,7 +920,7 @@ static inline LuaTableProps nlua_check_type(lua_State *const lstate, Error *cons
 /// Convert lua table to float
 ///
 /// Always pops one value from the stack.
-Float nlua_pop_Float(lua_State *lstate, Error *err)
+Float nlua_pop_Float(lua_State *lstate, Arena *arena, Error *err)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
   if (lua_type(lstate, -1) == LUA_TNUMBER) {
@@ -947,29 +944,29 @@ Float nlua_pop_Float(lua_State *lstate, Error *err)
 /// @param[in]  table_props  nlua_traverse_table() output.
 /// @param[out]  err  Location where error will be saved.
 static Array nlua_pop_Array_unchecked(lua_State *const lstate, const LuaTableProps table_props,
-                                      Error *const err)
+                                      Arena *arena, Error *const err)
 {
-  Array ret = { .size = table_props.maxidx, .items = NULL };
+  Array ret = arena_array(arena, table_props.maxidx);
 
-  if (ret.size == 0) {
+  if (table_props.maxidx == 0) {
     lua_pop(lstate, 1);
     return ret;
   }
 
-  ret.items = xcalloc(ret.size, sizeof(*ret.items));
-  for (size_t i = 1; i <= ret.size; i++) {
+  for (size_t i = 1; i <= table_props.maxidx; i++) {
     Object val;
 
     lua_rawgeti(lstate, -1, (int)i);
 
-    val = nlua_pop_Object(lstate, false, err);
+    val = nlua_pop_Object(lstate, false, arena, err);
     if (ERROR_SET(err)) {
-      ret.size = i - 1;
       lua_pop(lstate, 1);
-      api_free_array(ret);
+      if (!arena) {
+        api_free_array(ret);
+      }
       return (Array) { .size = 0, .items = NULL };
     }
-    ret.items[i - 1] = val;
+    ADD_C(ret, val);
   }
   lua_pop(lstate, 1);
 
@@ -979,15 +976,14 @@ static Array nlua_pop_Array_unchecked(lua_State *const lstate, const LuaTablePro
 /// Convert lua table to array
 ///
 /// Always pops one value from the stack.
-Array nlua_pop_Array(lua_State *lstate, Error *err)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+Array nlua_pop_Array(lua_State *lstate, Arena *arena, Error *err)
+  FUNC_ATTR_NONNULL_ARG(1, 3) FUNC_ATTR_WARN_UNUSED_RESULT
 {
-  const LuaTableProps table_props = nlua_check_type(lstate, err,
-                                                    kObjectTypeArray);
+  const LuaTableProps table_props = nlua_check_type(lstate, err, kObjectTypeArray);
   if (table_props.type != kObjectTypeArray) {
     return (Array) { .size = 0, .items = NULL };
   }
-  return nlua_pop_Array_unchecked(lstate, table_props, err);
+  return nlua_pop_Array_unchecked(lstate, table_props, arena, err);
 }
 
 /// Convert lua table to dictionary
@@ -999,30 +995,30 @@ Array nlua_pop_Array(lua_State *lstate, Error *err)
 /// @param[in]  table_props  nlua_traverse_table() output.
 /// @param[out]  err  Location where error will be saved.
 static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate, const LuaTableProps table_props,
-                                                bool ref, Error *err)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+                                                bool ref, Arena *arena, Error *err)
+  FUNC_ATTR_NONNULL_ARG(1, 5) FUNC_ATTR_WARN_UNUSED_RESULT
 {
-  Dictionary ret = { .size = table_props.string_keys_num, .items = NULL };
+  Dictionary ret = arena_dict(arena, table_props.string_keys_num);
 
-  if (ret.size == 0) {
+  if (table_props.string_keys_num == 0) {
     lua_pop(lstate, 1);
     return ret;
   }
-  ret.items = xcalloc(ret.size, sizeof(*ret.items));
 
   lua_pushnil(lstate);
-  for (size_t i = 0; lua_next(lstate, -2) && i < ret.size;) {
+  for (size_t i = 0; lua_next(lstate, -2) && i < table_props.string_keys_num;) {
     // stack: dict, key, value
 
     if (lua_type(lstate, -2) == LUA_TSTRING) {
       lua_pushvalue(lstate, -2);
       // stack: dict, key, value, key
 
-      ret.items[i].key = nlua_pop_String(lstate, err);
+      String key = nlua_pop_String(lstate, arena, err);
       // stack: dict, key, value
 
       if (!ERROR_SET(err)) {
-        ret.items[i].value = nlua_pop_Object(lstate, ref, err);
+        Object value = nlua_pop_Object(lstate, ref, arena, err);
+        kv_push_c(ret, ((KeyValuePair) { .key = key, .value = value }));
         // stack: dict, key
       } else {
         lua_pop(lstate, 1);
@@ -1030,8 +1026,9 @@ static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate, const LuaTabl
       }
 
       if (ERROR_SET(err)) {
-        ret.size = i;
-        api_free_dictionary(ret);
+        if (!arena) {
+          api_free_dictionary(ret);
+        }
         lua_pop(lstate, 2);
         // stack:
         return (Dictionary) { .size = 0, .items = NULL };
@@ -1050,8 +1047,8 @@ static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate, const LuaTabl
 /// Convert lua table to dictionary
 ///
 /// Always pops one value from the stack.
-Dictionary nlua_pop_Dictionary(lua_State *lstate, bool ref, Error *err)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+Dictionary nlua_pop_Dictionary(lua_State *lstate, bool ref, Arena *arena, Error *err)
+  FUNC_ATTR_NONNULL_ARG(1, 4) FUNC_ATTR_WARN_UNUSED_RESULT
 {
   const LuaTableProps table_props = nlua_check_type(lstate, err,
                                                     kObjectTypeDictionary);
@@ -1060,7 +1057,7 @@ Dictionary nlua_pop_Dictionary(lua_State *lstate, bool ref, Error *err)
     return (Dictionary) { .size = 0, .items = NULL };
   }
 
-  return nlua_pop_Dictionary_unchecked(lstate, table_props, ref, err);
+  return nlua_pop_Dictionary_unchecked(lstate, table_props, ref, arena, err);
 }
 
 /// Helper structure for nlua_pop_Object
@@ -1072,13 +1069,14 @@ typedef struct {
 /// Convert lua table to object
 ///
 /// Always pops one value from the stack.
-Object nlua_pop_Object(lua_State *const lstate, bool ref, Error *const err)
+Object nlua_pop_Object(lua_State *const lstate, bool ref, Arena *arena, Error *const err)
+  FUNC_ATTR_NONNULL_ARG(1, 4) FUNC_ATTR_WARN_UNUSED_RESULT
 {
   Object ret = NIL;
   const int initial_size = lua_gettop(lstate);
   kvec_withinit_t(ObjPopStackItem, 2) stack = KV_INITIAL_VALUE;
   kvi_init(stack);
-  kvi_push(stack, ((ObjPopStackItem) { &ret, false }));
+  kvi_push(stack, ((ObjPopStackItem){ .obj = &ret }));
   while (!ERROR_SET(err) && kv_size(stack)) {
     ObjPopStackItem cur = kv_pop(stack);
     if (cur.container) {
@@ -1088,8 +1086,7 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Error *const err)
       }
       if (cur.obj->type == kObjectTypeDictionary) {
         // stack: …, dict, key
-        if (cur.obj->data.dictionary.size
-            == cur.obj->data.dictionary.capacity) {
+        if (cur.obj->data.dictionary.size == cur.obj->data.dictionary.capacity) {
           lua_pop(lstate, 2);
           continue;
         }
@@ -1108,15 +1105,9 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Error *const err)
           size_t len;
           const char *s = lua_tolstring(lstate, -2, &len);
           const size_t idx = cur.obj->data.dictionary.size++;
-          cur.obj->data.dictionary.items[idx].key = (String) {
-            .data = xmemdupz(s, len),
-            .size = len,
-          };
+          cur.obj->data.dictionary.items[idx].key = CBUF_TO_ARENA_STR(arena, s, len);
           kvi_push(stack, cur);
-          cur = (ObjPopStackItem) {
-            .obj = &cur.obj->data.dictionary.items[idx].value,
-            .container = false,
-          };
+          cur = (ObjPopStackItem){ .obj = &cur.obj->data.dictionary.items[idx].value };
         } else {
           // stack: …, dict
           lua_pop(lstate, 1);
@@ -1130,15 +1121,8 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Error *const err)
         }
         const size_t idx = cur.obj->data.array.size++;
         lua_rawgeti(lstate, -1, (int)idx + 1);
-        if (lua_isnil(lstate, -1)) {
-          lua_pop(lstate, 2);
-          continue;
-        }
         kvi_push(stack, cur);
-        cur = (ObjPopStackItem) {
-          .obj = &cur.obj->data.array.items[idx],
-          .container = false,
-        };
+        cur = (ObjPopStackItem){ .obj = &cur.obj->data.array.items[idx] };
       }
     }
     assert(!cur.container);
@@ -1152,7 +1136,7 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Error *const err)
     case LUA_TSTRING: {
       size_t len;
       const char *s = lua_tolstring(lstate, -1, &len);
-      *cur.obj = STRING_OBJ(((String) { .data = xmemdupz(s, len), .size = len }));
+      *cur.obj = STRING_OBJ(CBUF_TO_ARENA_STR(arena, s, len));
       break;
     }
     case LUA_TNUMBER: {
@@ -1170,23 +1154,17 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Error *const err)
 
       switch (table_props.type) {
       case kObjectTypeArray:
-        *cur.obj = ARRAY_OBJ(((Array) { .items = NULL, .size = 0, .capacity = 0 }));
+        *cur.obj = ARRAY_OBJ(((Array)ARRAY_DICT_INIT));
         if (table_props.maxidx != 0) {
-          cur.obj->data.array.items =
-            xcalloc(table_props.maxidx,
-                    sizeof(cur.obj->data.array.items[0]));
-          cur.obj->data.array.capacity = table_props.maxidx;
+          cur.obj->data.array = arena_array(arena, table_props.maxidx);
           cur.container = true;
           kvi_push(stack, cur);
         }
         break;
       case kObjectTypeDictionary:
-        *cur.obj = DICTIONARY_OBJ(((Dictionary) { .items = NULL, .size = 0, .capacity = 0 }));
+        *cur.obj = DICTIONARY_OBJ(((Dictionary)ARRAY_DICT_INIT));
         if (table_props.string_keys_num != 0) {
-          cur.obj->data.dictionary.items =
-            xcalloc(table_props.string_keys_num,
-                    sizeof(cur.obj->data.dictionary.items[0]));
-          cur.obj->data.dictionary.capacity = table_props.string_keys_num;
+          cur.obj->data.dictionary = arena_dict(arena, table_props.string_keys_num);
           cur.container = true;
           kvi_push(stack, cur);
           lua_pushnil(lstate);
@@ -1238,7 +1216,9 @@ type_error:
   }
   kvi_destroy(stack);
   if (ERROR_SET(err)) {
-    api_free_object(ret);
+    if (!arena) {
+      api_free_object(ret);
+    }
     ret = NIL;
     lua_pop(lstate, lua_gettop(lstate) - initial_size + 1);
   }
@@ -1246,20 +1226,20 @@ type_error:
   return ret;
 }
 
-LuaRef nlua_pop_LuaRef(lua_State *const lstate, Error *err)
+LuaRef nlua_pop_LuaRef(lua_State *const lstate, Arena *arena, Error *err)
 {
   LuaRef rv = nlua_ref_global(lstate, -1);
   lua_pop(lstate, 1);
   return rv;
 }
 
-handle_T nlua_pop_handle(lua_State *lstate, Error *err)
+handle_T nlua_pop_handle(lua_State *lstate, Arena *arena, Error *err)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
   handle_T ret;
   if (lua_type(lstate, -1) != LUA_TNUMBER) {
     api_set_error(err, kErrorTypeValidation, "Expected Lua number");
-    ret = (handle_T) - 1;
+    ret = (handle_T)(-1);
   } else {
     ret = (handle_T)lua_tonumber(lstate, -1);
   }
@@ -1315,7 +1295,8 @@ void nlua_init_types(lua_State *const lstate)
 }
 
 // lua specific variant of api_dict_to_keydict
-void nlua_pop_keydict(lua_State *L, void *retval, FieldHashfn hashy, char **err_opt, Error *err)
+void nlua_pop_keydict(lua_State *L, void *retval, FieldHashfn hashy, char **err_opt, Arena *arena,
+                      Error *err)
 {
   if (!lua_istable(L, -1)) {
     api_set_error(err, kErrorTypeValidation, "Expected Lua table");
@@ -1342,24 +1323,31 @@ void nlua_pop_keydict(lua_State *L, void *retval, FieldHashfn hashy, char **err_
     char *mem = ((char *)retval + field->ptr_off);
 
     if (field->type == kObjectTypeNil) {
-      *(Object *)mem = nlua_pop_Object(L, true, err);
+      *(Object *)mem = nlua_pop_Object(L, true, arena, err);
     } else if (field->type == kObjectTypeInteger) {
-      *(Integer *)mem = nlua_pop_Integer(L, err);
+      if (field->is_hlgroup && lua_type(L, -1) == LUA_TSTRING) {
+        size_t name_len;
+        const char *name = lua_tolstring(L, -1, &name_len);
+        lua_pop(L, 1);
+        *(Integer *)mem = name_len > 0 ? syn_check_group(name, name_len) : 0;
+      } else {
+        *(Integer *)mem = nlua_pop_Integer(L, arena, err);
+      }
     } else if (field->type == kObjectTypeBoolean) {
       *(Boolean *)mem = nlua_pop_Boolean_strict(L, err);
     } else if (field->type == kObjectTypeString) {
-      *(String *)mem = nlua_pop_String(L, err);
+      *(String *)mem = nlua_pop_String(L, arena, err);
     } else if (field->type == kObjectTypeFloat) {
-      *(Float *)mem = nlua_pop_Float(L, err);
+      *(Float *)mem = nlua_pop_Float(L, arena, err);
     } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
                || field->type == kObjectTypeTabpage) {
-      *(handle_T *)mem = nlua_pop_handle(L, err);
+      *(handle_T *)mem = nlua_pop_handle(L, arena, err);
     } else if (field->type == kObjectTypeArray) {
-      *(Array *)mem = nlua_pop_Array(L, err);
+      *(Array *)mem = nlua_pop_Array(L, arena, err);
     } else if (field->type == kObjectTypeDictionary) {
-      *(Dictionary *)mem = nlua_pop_Dictionary(L, false, err);
+      *(Dictionary *)mem = nlua_pop_Dictionary(L, false, arena, err);
     } else if (field->type == kObjectTypeLuaRef) {
-      *(LuaRef *)mem = nlua_pop_LuaRef(L, err);
+      *(LuaRef *)mem = nlua_pop_LuaRef(L, arena, err);
     } else {
       abort();
     }
@@ -1372,3 +1360,48 @@ void nlua_pop_keydict(lua_State *L, void *retval, FieldHashfn hashy, char **err_
   lua_pop(L, 1);
   // []
 }
+
+void nlua_push_keydict(lua_State *L, void *value, KeySetLink *table)
+{
+  lua_createtable(L, 0, 0);
+  for (size_t i = 0; table[i].str; i++) {
+    KeySetLink *field = &table[i];
+    bool is_set = true;
+    if (field->opt_index >= 0) {
+      OptKeySet *ks = (OptKeySet *)value;
+      is_set = ks->is_set_ & (1ULL << field->opt_index);
+    }
+
+    if (!is_set) {
+      continue;
+    }
+
+    char *mem = ((char *)value + field->ptr_off);
+
+    lua_pushstring(L, field->str);
+    if (field->type == kObjectTypeNil) {
+      nlua_push_Object(L, (Object *)mem, false);
+    } else if (field->type == kObjectTypeInteger) {
+      lua_pushinteger(L, *(Integer *)mem);
+    } else if (field->type == kObjectTypeBuffer || field->type == kObjectTypeWindow
+               || field->type == kObjectTypeTabpage) {
+      lua_pushinteger(L, *(handle_T *)mem);
+    } else if (field->type == kObjectTypeFloat) {
+      lua_pushnumber(L, *(Float *)mem);
+    } else if (field->type == kObjectTypeBoolean) {
+      lua_pushboolean(L, *(Boolean *)mem);
+    } else if (field->type == kObjectTypeString) {
+      nlua_push_String(L, *(String *)mem, false);
+    } else if (field->type == kObjectTypeArray) {
+      nlua_push_Array(L, *(Array *)mem, false);
+    } else if (field->type == kObjectTypeDictionary) {
+      nlua_push_Dictionary(L, *(Dictionary *)mem, false);
+    } else if (field->type == kObjectTypeLuaRef) {
+      nlua_pushref(L, *(LuaRef *)mem);
+    } else {
+      abort();
+    }
+
+    lua_rawset(L, -3);
+  }
+}
diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c
index 06d16efb05..1a9bd026b5 100644
--- a/src/nvim/lua/executor.c
+++ b/src/nvim/lua/executor.c
@@ -15,7 +15,9 @@
 #include "nvim/api/extmark.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/ui.h"
 #include "nvim/ascii_defs.h"
+#include "nvim/buffer_defs.h"
 #include "nvim/change.h"
 #include "nvim/cmdexpand_defs.h"
 #include "nvim/cursor.h"
@@ -33,8 +35,9 @@
 #include "nvim/ex_cmds_defs.h"
 #include "nvim/ex_getln.h"
 #include "nvim/garray.h"
+#include "nvim/garray_defs.h"
 #include "nvim/getchar.h"
-#include "nvim/gettext.h"
+#include "nvim/gettext_defs.h"
 #include "nvim/globals.h"
 #include "nvim/keycodes.h"
 #include "nvim/lua/converter.h"
@@ -45,22 +48,30 @@
 #include "nvim/main.h"
 #include "nvim/memline.h"
 #include "nvim/memory.h"
+#include "nvim/memory_defs.h"
 #include "nvim/message.h"
 #include "nvim/msgpack_rpc/channel.h"
 #include "nvim/option_vars.h"
 #include "nvim/os/fileio.h"
+#include "nvim/os/fileio_defs.h"
 #include "nvim/os/os.h"
 #include "nvim/path.h"
 #include "nvim/pos_defs.h"
 #include "nvim/profile.h"
 #include "nvim/runtime.h"
+#include "nvim/runtime_defs.h"
 #include "nvim/strings.h"
 #include "nvim/ui.h"
+#include "nvim/ui_defs.h"
 #include "nvim/undo.h"
 #include "nvim/usercmd.h"
 #include "nvim/vim_defs.h"
 #include "nvim/window.h"
 
+#ifndef MSWIN
+# include <pthread.h>
+#endif
+
 static int in_fast_callback = 0;
 static bool in_script = false;
 
@@ -115,7 +126,7 @@ lua_State *get_global_lstate(void)
 /// Convert lua error into a Vim error message
 ///
 /// @param  lstate  Lua interpreter state.
-/// @param[in]  msg  Message base, must contain one `%*s`.
+/// @param[in]  msg  Message base, must contain one `%.*s`.
 void nlua_error(lua_State *const lstate, const char *const msg)
   FUNC_ATTR_NONNULL_ALL
 {
@@ -208,7 +219,7 @@ static int nlua_luv_cfpcall(lua_State *lstate, int nargs, int nresult, int flags
     const char *error = lua_tostring(lstate, -1);
 
     multiqueue_put(main_loop.events, nlua_luv_error_event,
-                   2, xstrdup(error), (intptr_t)kCallback);
+                   xstrdup(error), (void *)(intptr_t)kCallback);
     lua_pop(lstate, 1);  // error message
     retval = -status;
   } else {  // LUA_OK
@@ -254,8 +265,7 @@ static int nlua_luv_thread_common_cfpcall(lua_State *lstate, int nargs, int nres
     if (status == LUA_ERRMEM && !(flags & LUVF_CALLBACK_NOEXIT)) {
       // Terminate this thread, as the main thread may be able to continue
       // execution.
-      os_errmsg(e_outofmem);
-      os_errmsg("\n");
+      fprintf(stderr, "%s\n", e_outofmem);
       lua_close(lstate);
 #ifdef MSWIN
       ExitThread(0);
@@ -266,11 +276,11 @@ static int nlua_luv_thread_common_cfpcall(lua_State *lstate, int nargs, int nres
     const char *error = lua_tostring(lstate, -1);
 
     loop_schedule_deferred(&main_loop,
-                           event_create(nlua_luv_error_event, 2,
+                           event_create(nlua_luv_error_event,
                                         xstrdup(error),
-                                        is_callback
-                                        ? (intptr_t)kThreadCallback
-                                        : (intptr_t)kThread));
+                                        (void *)(intptr_t)(is_callback
+                                                           ? kThreadCallback
+                                                           : kThread)));
     lua_pop(lstate, 1);  // error message
     retval = -status;
   } else {  // LUA_OK
@@ -302,7 +312,10 @@ static int nlua_thr_api_nvim__get_runtime(lua_State *lstate)
   lua_pop(lstate, 1);
 
   Error err = ERROR_INIT;
-  const Array pat = nlua_pop_Array(lstate, &err);
+  // TODO(bfredl): we could use an arena here for both "pat" and "ret", but then
+  // we need a path to not use the freelist but a private block local to the thread.
+  // We do not want mutex contentionery for the main arena freelist.
+  const Array pat = nlua_pop_Array(lstate, NULL, &err);
   if (ERROR_SET(&err)) {
     luaL_where(lstate, 1);
     lua_pushstring(lstate, err.msg);
@@ -379,8 +392,7 @@ static int nlua_schedule(lua_State *const lstate)
 
   LuaRef cb = nlua_ref_global(lstate, 1);
 
-  multiqueue_put(main_loop.events, nlua_schedule_event,
-                 1, (void *)(ptrdiff_t)cb);
+  multiqueue_put(main_loop.events, nlua_schedule_event, (void *)(ptrdiff_t)cb);
   return 0;
 }
 
@@ -643,8 +655,7 @@ static bool nlua_init_packages(lua_State *lstate, bool is_standalone)
   lua_getglobal(lstate, "require");
   lua_pushstring(lstate, "vim._init_packages");
   if (nlua_pcall(lstate, 1, 0)) {
-    os_errmsg(lua_tostring(lstate, -1));
-    os_errmsg("\n");
+    fprintf(stderr, "%s\n", lua_tostring(lstate, -1));
     return false;
   }
 
@@ -818,12 +829,12 @@ void nlua_init(char **argv, int argc, int lua_arg0)
 
   lua_State *lstate = luaL_newstate();
   if (lstate == NULL) {
-    os_errmsg(_("E970: Failed to initialize lua interpreter\n"));
+    fprintf(stderr, _("E970: Failed to initialize lua interpreter\n"));
     os_exit(1);
   }
   luaL_openlibs(lstate);
   if (!nlua_state_init(lstate)) {
-    os_errmsg(_("E970: Failed to initialize builtin lua modules\n"));
+    fprintf(stderr, _("E970: Failed to initialize builtin lua modules\n"));
 #ifdef EXITFREE
     nlua_common_free_all_mem(lstate);
 #endif
@@ -1022,15 +1033,14 @@ static int nlua_print(lua_State *const lstate)
 
   if (is_thread) {
     loop_schedule_deferred(&main_loop,
-                           event_create(nlua_print_event, 2,
+                           event_create(nlua_print_event,
                                         msg_ga.ga_data,
-                                        (intptr_t)msg_ga.ga_len));
+                                        (void *)(intptr_t)msg_ga.ga_len));
   } else if (in_fast_callback) {
     multiqueue_put(main_loop.events, nlua_print_event,
-                   2, msg_ga.ga_data, (intptr_t)msg_ga.ga_len);
+                   msg_ga.ga_data, (void *)(intptr_t)msg_ga.ga_len);
   } else {
-    nlua_print_event((void *[]){ msg_ga.ga_data,
-                                 (void *)(intptr_t)msg_ga.ga_len });
+    nlua_print_event((void *[]){ msg_ga.ga_data, (void *)(intptr_t)msg_ga.ga_len });
   }
   return 0;
 
@@ -1236,13 +1246,13 @@ static int nlua_rpc(lua_State *lstate, bool request)
   const char *name = luaL_checklstring(lstate, 2, &name_len);
   int nargs = lua_gettop(lstate) - 2;
   Error err = ERROR_INIT;
-  Array args = ARRAY_DICT_INIT;
+  Arena arena = ARENA_EMPTY;
 
+  Array args = arena_array(&arena, (size_t)nargs);
   for (int i = 0; i < nargs; i++) {
     lua_pushvalue(lstate, i + 3);
-    ADD(args, nlua_pop_Object(lstate, false, &err));
+    ADD(args, nlua_pop_Object(lstate, false, &arena, &err));
     if (ERROR_SET(&err)) {
-      api_free_array(args);
       goto check_err;
     }
   }
@@ -1251,7 +1261,7 @@ static int nlua_rpc(lua_State *lstate, bool request)
     ArenaMem res_mem = NULL;
     Object result = rpc_send_call(chan_id, name, args, &res_mem, &err);
     if (!ERROR_SET(&err)) {
-      nlua_push_Object(lstate, result, false);
+      nlua_push_Object(lstate, &result, false);
       arena_mem_free(res_mem);
     }
   } else {
@@ -1259,10 +1269,11 @@ static int nlua_rpc(lua_State *lstate, bool request)
       api_set_error(&err, kErrorTypeValidation,
                     "Invalid channel: %" PRIu64, chan_id);
     }
-    api_free_array(args);  // TODO(bfredl): no
   }
 
 check_err:
+  arena_mem_free(arena_finish(&arena));
+
   if (ERROR_SET(&err)) {
     lua_pushstring(lstate, err.msg);
     api_clear_error(&err);
@@ -1535,10 +1546,12 @@ int typval_exec_lua_callable(LuaRef lua_cb, int argcount, typval_T *argvars, typ
 ///
 /// @param[in]  str  String to execute.
 /// @param[in]  args array of ... args
+/// @param[in]  mode Whether and how the the return value should be converted to Object
+/// @param[in] arena  can be NULL, then nested allocations are used
 /// @param[out]  err  Location where error will be saved.
 ///
 /// @return Return value of the execution.
-Object nlua_exec(const String str, const Array args, Error *err)
+Object nlua_exec(const String str, const Array args, LuaRetMode mode, Arena *arena, Error *err)
 {
   lua_State *const lstate = global_lstate;
 
@@ -1551,7 +1564,7 @@ Object nlua_exec(const String str, const Array args, Error *err)
   }
 
   for (size_t i = 0; i < args.size; i++) {
-    nlua_push_Object(lstate, args.items[i], false);
+    nlua_push_Object(lstate, &args.items[i], false);
   }
 
   if (nlua_pcall(lstate, (int)args.size, 1)) {
@@ -1562,7 +1575,7 @@ Object nlua_exec(const String str, const Array args, Error *err)
     return NIL;
   }
 
-  return nlua_pop_Object(lstate, false, err);
+  return nlua_call_pop_retval(lstate, mode, arena, err);
 }
 
 bool nlua_ref_is_function(LuaRef ref)
@@ -1583,12 +1596,12 @@ bool nlua_ref_is_function(LuaRef ref)
 /// @param ref     the reference to call (not consumed)
 /// @param name    if non-NULL, sent to callback as first arg
 ///                if NULL, only args are used
-/// @param retval  if true, convert return value to Object
-///                if false, only check if return value is truthy
+/// @param mode    Whether and how the the return value should be converted to Object
+/// @param arena   can be NULL, then nested allocations are used
 /// @param err     Error details, if any (if NULL, errors are echoed)
-/// @return        Return value of function, if retval was set. Otherwise
-/// BOOLEAN_OBJ(true) or NIL.
-Object nlua_call_ref(LuaRef ref, const char *name, Array args, bool retval, Error *err)
+/// @return        Return value of function, as per mode
+Object nlua_call_ref(LuaRef ref, const char *name, Array args, LuaRetMode mode, Arena *arena,
+                     Error *err)
 {
   lua_State *const lstate = global_lstate;
   nl