refactor(lua): use Arena when converting from lua stack to API args

and for return value of nlua_exec/nlua_call_ref, as this uses
the same family of functions.

NB: the handling of luaref:s is a bit of a mess.
add api_luarefs_free_XX functions as a stop-gap as refactoring
luarefs is a can of worms for another PR:s.

as a minor feature/bug-fix, nvim_buf_call and nvim_win_call now preserves
arbitrary return values.

9 files changed, 91 insertions, 57 deletions

diff --git a/src/nvim/api/autocmd.c b/src/nvim/api/autocmd.c
index 78628c4154..4f67f682f1 100644
--- a/src/nvim/api/autocmd.c
+++ b/src/nvim/api/autocmd.c
@@ -431,7 +431,8 @@ Integer nvim_create_autocmd(uint64_t channel_id, Object event, Dict(create_autoc
       });
 
       cb.type = kCallbackLua;
-      cb.data.luaref = api_new_luaref(callback->data.luaref);
+      cb.data.luaref = callback->data.luaref;
+      callback->data.luaref = LUA_NOREF;
       break;
     case kObjectTypeString:
       cb.type = kCallbackFuncref;
diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c
index 183483b329..993e290b2d 100644
--- a/src/nvim/api/buffer.c
+++ b/src/nvim/api/buffer.c
@@ -1227,8 +1227,7 @@ ArrayOf(Integer, 2) nvim_buf_get_mark(Buffer buffer, String name, Error *err)
 /// @param fun        Function to call inside the buffer (currently Lua callable
 ///                   only)
 /// @param[out] err   Error details, if any
-/// @return           Return value of function. NB: will deepcopy Lua values
-///                   currently, use upvalues to send Lua references in and out.
+/// @return           Return value of function.
 Object nvim_buf_call(Buffer buffer, LuaRef fun, Error *err)
   FUNC_API_SINCE(7)
   FUNC_API_LUA_ONLY
@@ -1242,7 +1241,7 @@ Object nvim_buf_call(Buffer buffer, LuaRef fun, Error *err)
   aucmd_prepbuf(&aco, buf);
 
   Array args = ARRAY_DICT_INIT;
-  Object res = nlua_call_ref(fun, NULL, args, true, err);
+  Object res = nlua_call_ref(fun, NULL, args, kRetLuaref, NULL, err);
 
   aucmd_restbuf(&aco);
   try_end(err);
@@ -1419,7 +1418,7 @@ static void push_linestr(lua_State *lstate, Array *a, const char *s, size_t len,
   } else {
     String str = STRING_INIT;
     if (len > 0) {
-      str = arena_string(arena, cbuf_as_string((char *)s, len));
+      str = CBUF_TO_ARENA_STR(arena, s, len);
       if (replace_nl) {
         // Vim represents NULs as NLs, but this may confuse clients.
         strchrsub(str.data, '\n', '\0');
diff --git a/src/nvim/api/command.c b/src/nvim/api/command.c
index bafc45e543..0ac3d42231 100644
--- a/src/nvim/api/command.c
+++ b/src/nvim/api/command.c
@@ -99,7 +99,7 @@
 Dict(cmd) nvim_parse_cmd(String str, Dict(empty) *opts, Arena *arena, Error *err)
   FUNC_API_SINCE(10) FUNC_API_FAST
 {
-  Dict(cmd) result = { 0 };
+  Dict(cmd) result = KEYDICT_INIT;
 
   // Parse command line
   exarg_T ea;
@@ -514,7 +514,7 @@ String nvim_cmd(uint64_t channel_id, Dict(cmd) *cmd, Dict(cmd_opts) *opts, Error
   VALIDATE_MOD((!ea.forceit || (ea.argt & EX_BANG)), "bang", cmd->cmd.data);
 
   if (HAS_KEY(cmd, cmd, magic)) {
-    Dict(cmd_magic) magic[1] = { 0 };
+    Dict(cmd_magic) magic[1] = KEYDICT_INIT;
     if (!api_dict_to_keydict(magic, KeyDict_cmd_magic_get_field, cmd->magic, err)) {
       goto end;
     }
@@ -532,13 +532,13 @@ String nvim_cmd(uint64_t channel_id, Dict(cmd) *cmd, Dict(cmd_opts) *opts, Error
   }
 
   if (HAS_KEY(cmd, cmd, mods)) {
-    Dict(cmd_mods) mods[1] = { 0 };
+    Dict(cmd_mods) mods[1] = KEYDICT_INIT;
     if (!api_dict_to_keydict(mods, KeyDict_cmd_mods_get_field, cmd->mods, err)) {
       goto end;
     }
 
     if (HAS_KEY(mods, cmd_mods, filter)) {
-      Dict(cmd_mods_filter) filter[1] = { 0 };
+      Dict(cmd_mods_filter) filter[1] = KEYDICT_INIT;
 
       if (!api_dict_to_keydict(&filter, KeyDict_cmd_mods_filter_get_field,
                                mods->filter, err)) {
@@ -1103,7 +1103,8 @@ void create_user_command(uint64_t channel_id, String name, Object command, Dict(
 
   if (opts->complete.type == kObjectTypeLuaRef) {
     context = EXPAND_USER_LUA;
-    compl_luaref = api_new_luaref(opts->complete.data.luaref);
+    compl_luaref = opts->complete.data.luaref;
+    opts->complete.data.luaref = LUA_NOREF;
   } else if (opts->complete.type == kObjectTypeString) {
     VALIDATE_S(OK == parse_compl_arg(opts->complete.data.string.data,
                                      (int)opts->complete.data.string.size, &context, &argt,
@@ -1123,7 +1124,8 @@ void create_user_command(uint64_t channel_id, String name, Object command, Dict(
     });
 
     argt |= EX_PREVIEW;
-    preview_luaref = api_new_luaref(opts->preview.data.luaref);
+    preview_luaref = opts->preview.data.luaref;
+    opts->preview.data.luaref = LUA_NOREF;
   }
 
   switch (command.type) {
diff --git a/src/nvim/api/deprecated.c b/src/nvim/api/deprecated.c
index dccaeb6922..c9a6036b8f 100644
--- a/src/nvim/api/deprecated.c
+++ b/src/nvim/api/deprecated.c
@@ -51,12 +51,12 @@ String nvim_command_output(uint64_t channel_id, String command, Error *err)
 
 /// @deprecated Use nvim_exec_lua() instead.
 /// @see nvim_exec_lua
-Object nvim_execute_lua(String code, Array args, Error *err)
+Object nvim_execute_lua(String code, Array args, Arena *arena, Error *err)
   FUNC_API_SINCE(3)
   FUNC_API_DEPRECATED_SINCE(7)
   FUNC_API_REMOTE_ONLY
 {
-  return nlua_exec(code, args, err);
+  return nlua_exec(code, args, kRetObject, arena, err);
 }
 
 /// Gets the buffer number
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index 8b45af7c71..7bf0d87603 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -576,7 +576,7 @@ String arena_string(Arena *arena, String str)
   if (str.size) {
     return cbuf_as_string(arena_memdupz(arena, str.data, str.size), str.size);
   } else {
-    return (String)STRING_INIT;
+    return (String){ .data = arena ? "" : xstrdup(""), .size = 0 };
   }
 }
 
@@ -1062,24 +1062,56 @@ Dictionary api_keydict_to_dict(void *value, KeySetLink *table, size_t max_size,
   return rv;
 }
 
-void api_free_keydict(void *dict, KeySetLink *table)
+void api_luarefs_free_object(Object value)
+{
+  // TODO(bfredl): this is more complicated than it needs to be.
+  // we should be able to lock down more specifically where luarefs can be
+  switch (value.type) {
+  case kObjectTypeLuaRef:
+    api_free_luaref(value.data.luaref);
+    break;
+
+  case kObjectTypeArray:
+    api_luarefs_free_array(value.data.array);
+    break;
+
+  case kObjectTypeDictionary:
+    api_luarefs_free_dict(value.data.dictionary);
+    break;
+
+  default:
+    break;
+  }
+}
+
+void api_luarefs_free_keydict(void *dict, KeySetLink *table)
 {
   for (size_t i = 0; table[i].str; i++) {
     char *mem = ((char *)dict + table[i].ptr_off);
     if (table[i].type == kObjectTypeNil) {
-      api_free_object(*(Object *)mem);
-    } else if (table[i].type == kObjectTypeString) {
-      api_free_string(*(String *)mem);
-    } else if (table[i].type == kObjectTypeArray) {
-      api_free_array(*(Array *)mem);
-    } else if (table[i].type == kObjectTypeDictionary) {
-      api_free_dictionary(*(Dictionary *)mem);
+      api_luarefs_free_object(*(Object *)mem);
     } else if (table[i].type == kObjectTypeLuaRef) {
       api_free_luaref(*(LuaRef *)mem);
+    } else if (table[i].type == kObjectTypeDictionary) {
+      api_luarefs_free_dict(*(Dictionary *)mem);
     }
   }
 }
 
+void api_luarefs_free_array(Array value)
+{
+  for (size_t i = 0; i < value.size; i++) {
+    api_luarefs_free_object(value.items[i]);
+  }
+}
+
+void api_luarefs_free_dict(Dictionary value)
+{
+  for (size_t i = 0; i < value.size; i++) {
+    api_luarefs_free_object(value.items[i].value);
+  }
+}
+
 /// Set a named mark
 /// buffer and mark name must be validated already
 /// @param buffer     Buffer to set the mark on
diff --git a/src/nvim/api/private/helpers.h b/src/nvim/api/private/helpers.h
index 9ee812f45c..11abb8f801 100644
--- a/src/nvim/api/private/helpers.h
+++ b/src/nvim/api/private/helpers.h
@@ -34,6 +34,7 @@
 #define CSTR_TO_OBJ(s) STRING_OBJ(cstr_to_string(s))
 #define CSTR_TO_ARENA_STR(arena, s) arena_string(arena, cstr_as_string(s))
 #define CSTR_TO_ARENA_OBJ(arena, s) STRING_OBJ(CSTR_TO_ARENA_STR(arena, s))
+#define CBUF_TO_ARENA_STR(arena, s, len) arena_string(arena, cbuf_as_string((char *)(s), len))
 
 #define BUFFER_OBJ(s) ((Object) { \
     .type = kObjectTypeBuffer, \
@@ -119,6 +120,8 @@
 #define api_init_array = ARRAY_DICT_INIT
 #define api_init_dictionary = ARRAY_DICT_INIT
 
+#define KEYDICT_INIT { 0 }
+
 #define api_free_boolean(value)
 #define api_free_integer(value)
 #define api_free_float(value)
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 78c2561bbd..769537ac98 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -496,11 +496,12 @@ String nvim_replace_termcodes(String str, Boolean from_part, Boolean do_lt, Bool
 ///                   or executing the Lua code.
 ///
 /// @return           Return value of Lua code if present or NIL.
-Object nvim_exec_lua(String code, Array args, Error *err)
+Object nvim_exec_lua(String code, Array args, Arena *arena, Error *err)
   FUNC_API_SINCE(7)
   FUNC_API_REMOTE_ONLY
 {
-  return nlua_exec(code, args, err);
+  // TODO(bfredl): convert directly from msgpack to lua and then back again
+  return nlua_exec(code, args, kRetObject, arena, err);
 }
 
 /// Notify the user with a message
@@ -512,7 +513,7 @@ Object nvim_exec_lua(String code, Array args, Error *err)
 /// @param log_level  The log level
 /// @param opts       Reserved for future use.
 /// @param[out] err   Error details, if any
-Object nvim_notify(String msg, Integer log_level, Dictionary opts, Error *err)
+Object nvim_notify(String msg, Integer log_level, Dictionary opts, Arena *arena, Error *err)
   FUNC_API_SINCE(7)
 {
   MAXSIZE_TEMP_ARRAY(args, 3);
@@ -520,7 +521,7 @@ Object nvim_notify(String msg, Integer log_level, Dictionary opts, Error *err)
   ADD_C(args, INTEGER_OBJ(log_level));
   ADD_C(args, DICTIONARY_OBJ(opts));
 
-  return NLUA_EXEC_STATIC("return vim.notify(...)", args, err);
+  return NLUA_EXEC_STATIC("return vim.notify(...)", args, kRetObject, arena, err);
 }
 
 /// Calculates the number of display cells occupied by `text`.
@@ -603,7 +604,8 @@ String nvim__get_lib_dir(void)
 /// @param all whether to return all matches or only the first
 /// @param opts is_lua: only search Lua subdirs
 /// @return list of absolute paths to the found files
-ArrayOf(String) nvim__get_runtime(Array pat, Boolean all, Dict(runtime) *opts, Error *err)
+ArrayOf(String) nvim__get_runtime(Array pat, Boolean all, Dict(runtime) *opts, Arena *arena,
+                                  Error *err)
   FUNC_API_SINCE(8)
   FUNC_API_FAST
 {
@@ -613,7 +615,7 @@ ArrayOf(String) nvim__get_runtime(Array pat, Boolean all, Dict(runtime) *opts, E
     return (Array)ARRAY_DICT_INIT;
   }
 
-  ArrayOf(String) res = runtime_get_named(opts->is_lua, pat, all);
+  ArrayOf(String) res = runtime_get_named(opts->is_lua, pat, all, arena);
 
   if (opts->do_source) {
     for (size_t i = 0; i < res.size; i++) {
@@ -1068,7 +1070,7 @@ static void term_write(const char *buf, size_t size, void *data)
   ADD_C(args, BUFFER_OBJ(terminal_buf(chan->term)));
   ADD_C(args, STRING_OBJ(((String){ .data = (char *)buf, .size = size })));
   textlock++;
-  nlua_call_ref(cb, "input", args, false, NULL);
+  nlua_call_ref(cb, "input", args, kRetNilBool, NULL, NULL);
   textlock--;
 }
 
@@ -1189,7 +1191,7 @@ void nvim_set_current_tabpage(Tabpage tabpage, Error *err)
 /// @return
 ///     - true: Client may continue pasting.
 ///     - false: Client must cancel the paste.
-Boolean nvim_paste(String data, Boolean crlf, Integer phase, Error *err)
+Boolean nvim_paste(String data, Boolean crlf, Integer phase, Arena *arena, Error *err)
   FUNC_API_SINCE(6)
   FUNC_API_TEXTLOCK_ALLOW_CMDWIN
 {
@@ -1199,19 +1201,18 @@ Boolean nvim_paste(String data, Boolean crlf, Integer phase, Error *err)
   VALIDATE_INT((phase >= -1 && phase <= 3), "phase", phase, {
     return false;
   });
-  Array args = ARRAY_DICT_INIT;
-  Object rv = OBJECT_INIT;
+  Array lines = ARRAY_DICT_INIT;
   if (phase == -1 || phase == 1) {  // Start of paste-stream.
     draining = false;
   } else if (draining) {
     // Skip remaining chunks.  Report error only once per "stream".
     goto theend;
   }
-  Array lines = string_to_array(data, crlf);
-  ADD(args, ARRAY_OBJ(lines));
-  ADD(args, INTEGER_OBJ(phase));
-  rv = nvim_exec_lua(STATIC_CSTR_AS_STRING("return vim.paste(...)"), args,
-                     err);
+  lines = string_to_array(data, crlf);
+  MAXSIZE_TEMP_ARRAY(args, 2);
+  ADD_C(args, ARRAY_OBJ(lines));
+  ADD_C(args, INTEGER_OBJ(phase));
+  Object rv = NLUA_EXEC_STATIC("return vim.paste(...)", args, kRetNilBool, arena, err);
   if (ERROR_SET(err)) {
     draining = true;
     goto theend;
@@ -1238,8 +1239,7 @@ Boolean nvim_paste(String data, Boolean crlf, Integer phase, Error *err)
     AppendCharToRedobuff(ESC);  // Dot-repeat.
   }
 theend:
-  api_free_object(rv);
-  api_free_array(args);
+  api_free_array(lines);
   if (cancel || phase == -1 || phase == 3) {  // End of paste-stream.
     draining = false;
   }
@@ -1875,7 +1875,7 @@ Array nvim_list_uis(Arena *arena)
 /// Gets the immediate children of process `pid`.
 ///
 /// @return Array of child process ids, empty if process not found.
-Array nvim_get_proc_children(Integer pid, Error *err)
+Array nvim_get_proc_children(Integer pid, Arena *arena, Error *err)
   FUNC_API_SINCE(4)
 {
   Array rvobj = ARRAY_DICT_INIT;
@@ -1892,7 +1892,7 @@ Array nvim_get_proc_children(Integer pid, Error *err)
     DLOG("fallback to vim._os_proc_children()");
     MAXSIZE_TEMP_ARRAY(a, 1);
     ADD(a, INTEGER_OBJ(pid));
-    Object o = NLUA_EXEC_STATIC("return vim._os_proc_children(...)", a, err);
+    Object o = NLUA_EXEC_STATIC("return vim._os_proc_children(...)", a, kRetObject, arena, err);
     if (o.type == kObjectTypeArray) {
       rvobj = o.data.array;
     } else if (!ERROR_SET(err)) {
@@ -1900,11 +1900,11 @@ Array nvim_get_proc_children(Integer pid, Error *err)
                     "Failed to get process children. pid=%" PRId64 " error=%d",
                     pid, rv);
     }
-    goto end;
-  }
-
-  for (size_t i = 0; i < proc_count; i++) {
-    ADD(rvobj, INTEGER_OBJ(proc_list[i]));
+  } else {
+    rvobj = arena_array(arena, proc_count);
+    for (size_t i = 0; i < proc_count; i++) {
+      ADD(rvobj, INTEGER_OBJ(proc_list[i]));
+    }
   }
 
 end:
@@ -1915,19 +1915,17 @@ end:
 /// Gets info describing process `pid`.
 ///
 /// @return Map of process properties, or NIL if process not found.
-Object nvim_get_proc(Integer pid, Error *err)
+Object nvim_get_proc(Integer pid, Arena *arena, Error *err)
   FUNC_API_SINCE(4)
 {
-  Object rvobj = OBJECT_INIT;
-  rvobj.data.dictionary = (Dictionary)ARRAY_DICT_INIT;
-  rvobj.type = kObjectTypeDictionary;
+  Object rvobj = NIL;
 
   VALIDATE_INT((pid > 0 && pid <= INT_MAX), "pid", pid, {
     return NIL;
   });
 
 #ifdef MSWIN
-  rvobj.data.dictionary = os_proc_info((int)pid);
+  rvobj = DICTIONARY_OBJ(os_proc_info((int)pid));
   if (rvobj.data.dictionary.size == 0) {  // Process not found.
     return NIL;
   }
@@ -1935,11 +1933,11 @@ Object nvim_get_proc(Integer pid, Error *err)
   // Cross-platform process info APIs are miserable, so use `ps` instead.
   MAXSIZE_TEMP_ARRAY(a, 1);
   ADD(a, INTEGER_OBJ(pid));
-  Object o = NLUA_EXEC_STATIC("return vim._os_proc_info(...)", a, err);
+  Object o = NLUA_EXEC_STATIC("return vim._os_proc_info(...)", a, kRetObject, arena, err);
   if (o.type == kObjectTypeArray && o.data.array.size == 0) {
     return NIL;  // Process not found.
   } else if (o.type == kObjectTypeDictionary) {
-    rvobj.data.dictionary = o.data.dictionary;
+    rvobj = o;
   } else if (!ERROR_SET(err)) {
     api_set_error(err, kErrorTypeException,
                   "Failed to get process info. pid=%" PRId64, pid);
diff --git a/src/nvim/api/win_config.c b/src/nvim/api/win_config.c
index 52372b838e..e76db82c61 100644
--- a/src/nvim/api/win_config.c
+++ b/src/nvim/api/win_config.c
@@ -600,7 +600,7 @@ Dict(win_config) nvim_win_get_config(Window window, Arena *arena, Error *err)
   /// Keep in sync with WinSplit in buffer_defs.h
   static const char *const win_split_str[] = { "left", "right", "above", "below" };
 
-  Dict(win_config) rv = { 0 };
+  Dict(win_config) rv = KEYDICT_INIT;
 
   win_T *wp = find_window_by_handle(window, err);
   if (!wp) {
diff --git a/src/nvim/api/window.c b/src/nvim/api/window.c
index 4ac7e47832..93c9dfa049 100644
--- a/src/nvim/api/window.c
+++ b/src/nvim/api/window.c
@@ -421,8 +421,7 @@ void nvim_win_close(Window window, Boolean force, Error *err)
 /// @param fun        Function to call inside the window (currently Lua callable
 ///                   only)
 /// @param[out] err   Error details, if any
-/// @return           Return value of function. NB: will deepcopy Lua values
-///                   currently, use upvalues to send Lua references in and out.
+/// @return           Return value of function.
 Object nvim_win_call(Window window, LuaRef fun, Error *err)
   FUNC_API_SINCE(7)
   FUNC_API_LUA_ONLY
@@ -438,7 +437,7 @@ Object nvim_win_call(Window window, LuaRef fun, Error *err)
   win_execute_T win_execute_args;
   if (win_execute_before(&win_execute_args, win, tabpage)) {
     Array args = ARRAY_DICT_INIT;
-    res = nlua_call_ref(fun, NULL, args, true, err);
+    res = nlua_call_ref(fun, NULL, args, kRetLuaref, NULL, err);
   }
   win_execute_after(&win_execute_args);
   try_end(err);