Merge remote-tracking branch 'upstream/master' into libcallnr

7 files changed, 2127 insertions, 130 deletions

diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c
index 9665655e74..32e804d213 100644
--- a/src/nvim/lua/converter.c
+++ b/src/nvim/lua/converter.c
@@ -19,6 +19,7 @@
 #include "nvim/globals.h"
 #include "nvim/message.h"
 #include "nvim/eval/typval.h"
+#include "nvim/eval/userfunc.h"
 #include "nvim/ascii.h"
 #include "nvim/macros.h"
 
@@ -50,6 +51,7 @@ typedef struct {
 #define LUA_PUSH_STATIC_STRING(lstate, s) \
     lua_pushlstring(lstate, s, sizeof(s) - 1)
 
+
 static LuaTableProps nlua_traverse_table(lua_State *const lstate)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
@@ -156,6 +158,13 @@ static LuaTableProps nlua_traverse_table(lua_State *const lstate)
             && other_keys_num == 0
             && ret.string_keys_num == 0)) {
       ret.type = kObjectTypeArray;
+      if (tsize == 0 && lua_getmetatable(lstate, -1)) {
+        nlua_pushref(lstate, nlua_empty_dict_ref);
+        if (lua_rawequal(lstate, -2, -1)) {
+          ret.type = kObjectTypeDictionary;
+        }
+        lua_pop(lstate, 2);
+      }
     } else if (ret.string_keys_num == tsize) {
       ret.type = kObjectTypeDictionary;
     } else {
@@ -279,10 +288,10 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
         break;
       }
       case LUA_TBOOLEAN: {
-        cur.tv->v_type = VAR_SPECIAL;
-        cur.tv->vval.v_special = (lua_toboolean(lstate, -1)
-                                  ? kSpecialVarTrue
-                                  : kSpecialVarFalse);
+        cur.tv->v_type = VAR_BOOL;
+        cur.tv->vval.v_bool = (lua_toboolean(lstate, -1)
+                               ? kBoolVarTrue
+                               : kBoolVarFalse);
         break;
       }
       case LUA_TSTRING: {
@@ -307,6 +316,13 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
         break;
       }
       case LUA_TTABLE: {
+        // Only need to track table refs if we have a metatable associated.
+        LuaRef table_ref = LUA_NOREF;
+        if (lua_getmetatable(lstate, -1)) {
+          lua_pop(lstate, 1);
+          table_ref = nlua_ref(lstate, -1);
+        }
+
         const LuaTableProps table_props = nlua_traverse_table(lstate);
 
         for (size_t i = 0; i < kv_size(stack); i++) {
@@ -322,6 +338,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
           case kObjectTypeArray: {
             cur.tv->v_type = VAR_LIST;
             cur.tv->vval.v_list = tv_list_alloc((ptrdiff_t)table_props.maxidx);
+            cur.tv->vval.v_list->lua_table_ref = table_ref;
             tv_list_ref(cur.tv->vval.v_list);
             if (table_props.maxidx != 0) {
               cur.container = true;
@@ -335,6 +352,7 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
               cur.tv->v_type = VAR_DICT;
               cur.tv->vval.v_dict = tv_dict_alloc();
               cur.tv->vval.v_dict->dv_refcount++;
+              cur.tv->vval.v_dict->lua_table_ref = table_ref;
             } else {
               cur.special = table_props.has_string_with_nul;
               if (table_props.has_string_with_nul) {
@@ -345,11 +363,13 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
                                                         S_LEN("_VAL"));
                 assert(val_di != NULL);
                 cur.tv = &val_di->di_tv;
+                cur.tv->vval.v_list->lua_table_ref = table_ref;
                 assert(cur.tv->v_type == VAR_LIST);
               } else {
                 cur.tv->v_type = VAR_DICT;
                 cur.tv->vval.v_dict = tv_dict_alloc();
                 cur.tv->vval.v_dict->dv_refcount++;
+                cur.tv->vval.v_dict->lua_table_ref = table_ref;
               }
               cur.container = true;
               cur.idx = lua_gettop(lstate);
@@ -377,6 +397,33 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
 nlua_pop_typval_table_processing_end:
         break;
       }
+      case LUA_TFUNCTION: {
+        LuaCFunctionState *state = xmalloc(sizeof(LuaCFunctionState));
+        state->lua_callable.func_ref = nlua_ref(lstate, -1);
+        state->lua_callable.table_ref = LUA_NOREF;
+
+        char_u *name = register_cfunc(
+            &nlua_CFunction_func_call,
+            &nlua_CFunction_func_free,
+            state);
+
+        cur.tv->v_type = VAR_FUNC;
+        cur.tv->vval.v_string = vim_strsave(name);
+        break;
+      }
+      case LUA_TUSERDATA: {
+        nlua_pushref(lstate, nlua_nil_ref);
+        bool is_nil = lua_rawequal(lstate, -2, -1);
+        lua_pop(lstate, 1);
+        if (is_nil) {
+          cur.tv->v_type = VAR_SPECIAL;
+          cur.tv->vval.v_special = kSpecialVarNull;
+        } else {
+          EMSG(_("E5101: Cannot convert given lua type"));
+          ret = false;
+        }
+        break;
+      }
       default: {
         EMSG(_("E5101: Cannot convert given lua type"));
         ret = false;
@@ -401,10 +448,18 @@ nlua_pop_typval_table_processing_end:
   return ret;
 }
 
+static bool typval_conv_special = false;
+
 #define TYPVAL_ENCODE_ALLOW_SPECIALS true
 
 #define TYPVAL_ENCODE_CONV_NIL(tv) \
-    lua_pushnil(lstate)
+    do { \
+      if (typval_conv_special) { \
+        lua_pushnil(lstate); \
+      } else { \
+        nlua_pushref(lstate, nlua_nil_ref); \
+      } \
+    } while (0)
 
 #define TYPVAL_ENCODE_CONV_BOOL(tv, num) \
     lua_pushboolean(lstate, (bool)(num))
@@ -439,7 +494,15 @@ nlua_pop_typval_table_processing_end:
     lua_createtable(lstate, 0, 0)
 
 #define TYPVAL_ENCODE_CONV_EMPTY_DICT(tv, dict) \
-    nlua_create_typed_table(lstate, 0, 0, kObjectTypeDictionary)
+    do { \
+      if (typval_conv_special) { \
+        nlua_create_typed_table(lstate, 0, 0, kObjectTypeDictionary); \
+      } else { \
+        lua_createtable(lstate, 0, 0); \
+        nlua_pushref(lstate, nlua_empty_dict_ref); \
+        lua_setmetatable(lstate, -2); \
+      } \
+    } while (0)
 
 #define TYPVAL_ENCODE_CONV_LIST_START(tv, len) \
     do { \
@@ -548,9 +611,11 @@ nlua_pop_typval_table_processing_end:
 /// @param[in]  tv  typval_T to convert.
 ///
 /// @return true in case of success, false otherwise.
-bool nlua_push_typval(lua_State *lstate, typval_T *const tv)
+bool nlua_push_typval(lua_State *lstate, typval_T *const tv, bool special)
 {
+  typval_conv_special = special;
   const int initial_size = lua_gettop(lstate);
+
   if (!lua_checkstack(lstate, initial_size + 2)) {
     emsgf(_("E1502: Lua failed to grow stack to %i"), initial_size + 4);
     return false;
@@ -666,6 +731,10 @@ void nlua_push_Dictionary(lua_State *lstate, const Dictionary dict,
     nlua_create_typed_table(lstate, 0, 0, kObjectTypeDictionary);
   } else {
     lua_createtable(lstate, 0, (int)dict.size);
+    if (dict.size == 0 && !special) {
+      nlua_pushref(lstate, nlua_empty_dict_ref);
+      lua_setmetatable(lstate, -2);
+    }
   }
   for (size_t i = 0; i < dict.size; i++) {
     nlua_push_String(lstate, dict.items[i].key, special);
@@ -708,7 +777,11 @@ void nlua_push_Object(lua_State *lstate, const Object obj, bool special)
 {
   switch (obj.type) {
     case kObjectTypeNil: {
-      lua_pushnil(lstate);
+      if (special) {
+        lua_pushnil(lstate);
+      } else {
+        nlua_pushref(lstate, nlua_nil_ref);
+      }
       break;
     }
     case kObjectTypeLuaRef: {
@@ -1142,6 +1215,19 @@ Object nlua_pop_Object(lua_State *const lstate, bool ref, Error *const err)
         break;
       }
 
+      case LUA_TUSERDATA: {
+        nlua_pushref(lstate, nlua_nil_ref);
+        bool is_nil = lua_rawequal(lstate, -2, -1);
+        lua_pop(lstate, 1);
+        if (is_nil) {
+          *cur.obj = NIL;
+        } else {
+          api_set_error(err, kErrorTypeValidation,
+                        "Cannot convert userdata");
+        }
+        break;
+      }
+
       default: {
 type_error:
         api_set_error(err, kErrorTypeValidation,
@@ -1179,7 +1265,7 @@ GENERATE_INDEX_FUNCTION(Tabpage)
 
 #undef GENERATE_INDEX_FUNCTION
 
-/// Record some auxilary values in vim module
+/// Record some auxiliary values in vim module
 ///
 /// Assumes that module table is on top of the stack.
 ///
diff --git a/src/nvim/lua/converter.h b/src/nvim/lua/converter.h
index 542c56ea3e..8601a32418 100644
--- a/src/nvim/lua/converter.h
+++ b/src/nvim/lua/converter.h
@@ -9,6 +9,15 @@
 #include "nvim/func_attr.h"
 #include "nvim/eval.h"
 
+typedef struct {
+  LuaRef func_ref;
+  LuaRef table_ref;
+} LuaCallable;
+
+typedef struct {
+  LuaCallable lua_callable;
+} LuaCFunctionState;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "lua/converter.h.generated.h"
 #endif
diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c
index f51aa3c6d4..0d5622f1e7 100644
--- a/src/nvim/lua/executor.c
+++ b/src/nvim/lua/executor.c
@@ -11,26 +11,33 @@
 #include "nvim/func_attr.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/private/handle.h"
 #include "nvim/api/vim.h"
+#include "nvim/msgpack_rpc/channel.h"
 #include "nvim/vim.h"
 #include "nvim/ex_getln.h"
 #include "nvim/ex_cmds2.h"
 #include "nvim/message.h"
 #include "nvim/memline.h"
 #include "nvim/buffer_defs.h"
+#include "nvim/regexp.h"
 #include "nvim/macros.h"
 #include "nvim/screen.h"
 #include "nvim/cursor.h"
 #include "nvim/undo.h"
 #include "nvim/ascii.h"
 #include "nvim/change.h"
+#include "nvim/eval/userfunc.h"
+#include "nvim/event/time.h"
+#include "nvim/event/loop.h"
 
 #ifdef WIN32
 #include "nvim/os/os.h"
 #endif
 
-#include "nvim/lua/executor.h"
 #include "nvim/lua/converter.h"
+#include "nvim/lua/executor.h"
+#include "nvim/lua/treesitter.h"
 
 #include "luv/luv.h"
 
@@ -46,8 +53,14 @@ typedef struct {
 # include "lua/executor.c.generated.h"
 #endif
 
-/// Name of the run code for use in messages
-#define NLUA_EVAL_NAME "<VimL compiled string>"
+#define PUSH_ALL_TYPVALS(lstate, args, argcount, special) \
+  for (int i = 0; i < argcount; i++) { \
+    if (args[i].v_type == VAR_UNKNOWN) { \
+      lua_pushnil(lstate); \
+    } else { \
+      nlua_push_typval(lstate, &args[i], special); \
+    } \
+  }
 
 /// Convert lua error into a Vim error message
 ///
@@ -245,6 +258,109 @@ static int nlua_schedule(lua_State *const lstate)
   return 0;
 }
 
+static struct luaL_Reg regex_meta[] = {
+  { "__gc", regex_gc },
+  { "__tostring", regex_tostring },
+  { "match_str", regex_match_str },
+  { "match_line", regex_match_line },
+  { NULL, NULL }
+};
+
+// Dummy timer callback. Used by f_wait().
+static void dummy_timer_due_cb(TimeWatcher *tw, void *data)
+{
+}
+
+// Dummy timer close callback. Used by f_wait().
+static void dummy_timer_close_cb(TimeWatcher *tw, void *data)
+{
+  xfree(tw);
+}
+
+static bool nlua_wait_condition(lua_State *lstate, int *status,
+                                bool *callback_result)
+{
+  lua_pushvalue(lstate, 2);
+  *status = lua_pcall(lstate, 0, 1, 0);
+  if (*status) {
+    return true;  // break on error, but keep error on stack
+  }
+  *callback_result = lua_toboolean(lstate, -1);
+  lua_pop(lstate, 1);
+  return *callback_result;  // break if true
+}
+
+/// "vim.wait(timeout, condition[, interval])" function
+static int nlua_wait(lua_State *lstate)
+  FUNC_ATTR_NONNULL_ALL
+{
+  intptr_t timeout = luaL_checkinteger(lstate, 1);
+  if (timeout < 0) {
+    return luaL_error(lstate, "timeout must be > 0");
+  }
+
+  // Check if condition can be called.
+  bool is_function = (lua_type(lstate, 2) == LUA_TFUNCTION);
+
+  // Check if condition is callable table
+  if (!is_function && luaL_getmetafield(lstate, 2, "__call") != 0) {
+    is_function = (lua_type(lstate, -1) == LUA_TFUNCTION);
+    lua_pop(lstate, 1);
+  }
+
+  if (!is_function) {
+    lua_pushliteral(lstate, "vim.wait: condition must be a function");
+    return lua_error(lstate);
+  }
+
+  intptr_t interval = 200;
+  if (lua_gettop(lstate) >= 3) {
+    interval = luaL_checkinteger(lstate, 3);
+    if (interval < 0) {
+      return luaL_error(lstate, "interval must be > 0");
+    }
+  }
+
+  TimeWatcher *tw = xmalloc(sizeof(TimeWatcher));
+
+  // Start dummy timer.
+  time_watcher_init(&main_loop, tw, NULL);
+  tw->events = main_loop.events;
+  tw->blockable = true;
+  time_watcher_start(tw, dummy_timer_due_cb,
+                     (uint64_t)interval, (uint64_t)interval);
+
+  int pcall_status = 0;
+  bool callback_result = false;
+
+  LOOP_PROCESS_EVENTS_UNTIL(
+      &main_loop,
+      main_loop.events,
+      (int)timeout,
+      nlua_wait_condition(lstate, &pcall_status, &callback_result) || got_int);
+
+  // Stop dummy timer
+  time_watcher_stop(tw);
+  time_watcher_close(tw, dummy_timer_close_cb);
+
+  if (pcall_status) {
+    return lua_error(lstate);
+  } else if (callback_result) {
+    lua_pushboolean(lstate, 1);
+    lua_pushnil(lstate);
+  } else if (got_int) {
+    got_int = false;
+    vgetc();
+    lua_pushboolean(lstate, 0);
+    lua_pushinteger(lstate, -2);
+  } else {
+    lua_pushboolean(lstate, 0);
+    lua_pushinteger(lstate, -1);
+  }
+
+  return 2;
+}
+
 /// Initialize lua interpreter state
 ///
 /// Called by lua interpreter itself to initialize state.
@@ -269,12 +385,7 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
 #endif
 
   // vim
-  const char *code = (char *)&vim_module[0];
-  if (luaL_loadbuffer(lstate, code, strlen(code), "@vim.lua")
-      || lua_pcall(lstate, 0, LUA_MULTRET, 0)) {
-    nlua_error(lstate, _("E5106: Error while creating vim module: %.*s"));
-    return 1;
-  }
+  lua_newtable(lstate);
   // vim.api
   nlua_add_api_functions(lstate);
   // vim.types, vim.type_idx, vim.val_idx
@@ -294,6 +405,29 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
   // in_fast_event
   lua_pushcfunction(lstate, &nlua_in_fast_event);
   lua_setfield(lstate, -2, "in_fast_event");
+  // call
+  lua_pushcfunction(lstate, &nlua_call);
+  lua_setfield(lstate, -2, "call");
+  // regex
+  lua_pushcfunction(lstate, &nlua_regex);
+  lua_setfield(lstate, -2, "regex");
+  luaL_newmetatable(lstate, "nvim_regex");
+  luaL_register(lstate, NULL, regex_meta);
+  lua_pushvalue(lstate, -1);  // [meta, meta]
+  lua_setfield(lstate, -2, "__index");  // [meta]
+  lua_pop(lstate, 1);  // don't use metatable now
+
+  // rpcrequest
+  lua_pushcfunction(lstate, &nlua_rpcrequest);
+  lua_setfield(lstate, -2, "rpcrequest");
+
+  // rpcnotify
+  lua_pushcfunction(lstate, &nlua_rpcnotify);
+  lua_setfield(lstate, -2, "rpcnotify");
+
+  // wait
+  lua_pushcfunction(lstate, &nlua_wait);
+  lua_setfield(lstate, -2, "wait");
 
   // vim.loop
   luv_set_loop(lstate, &main_loop.uv);
@@ -310,7 +444,45 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
   lua_setfield(lstate, -2, "luv");
   lua_pop(lstate, 3);
 
+  // vim.NIL
+  lua_newuserdata(lstate, 0);
+  lua_createtable(lstate, 0, 0);
+  lua_pushcfunction(lstate, &nlua_nil_tostring);
+  lua_setfield(lstate, -2, "__tostring");
+  lua_setmetatable(lstate, -2);
+  nlua_nil_ref = nlua_ref(lstate, -1);
+  lua_setfield(lstate, -2, "NIL");
+
+  // vim._empty_dict_mt
+  lua_createtable(lstate, 0, 0);
+  lua_pushcfunction(lstate, &nlua_empty_dict_tostring);
+  lua_setfield(lstate, -2, "__tostring");
+  nlua_empty_dict_ref = nlua_ref(lstate, -1);
+  lua_setfield(lstate, -2, "_empty_dict_mt");
+
+  // internal vim._treesitter... API
+  nlua_add_treesitter(lstate);
+
   lua_setglobal(lstate, "vim");
+
+  {
+    const char *code = (char *)&shared_module[0];
+    if (luaL_loadbuffer(lstate, code, strlen(code), "@shared.lua")
+        || lua_pcall(lstate, 0, 0, 0)) {
+      nlua_error(lstate, _("E5106: Error while creating shared module: %.*s"));
+      return 1;
+    }
+  }
+
+  {
+    const char *code = (char *)&vim_module[0];
+    if (luaL_loadbuffer(lstate, code, strlen(code), "@vim.lua")
+        || lua_pcall(lstate, 0, 0, 0)) {
+      nlua_error(lstate, _("E5106: Error while creating vim module: %.*s"));
+      return 1;
+    }
+  }
+
   return 0;
 }
 
@@ -371,29 +543,6 @@ static lua_State *nlua_enter(void)
   return lstate;
 }
 
-/// Execute lua string
-///
-/// @param[in]  str  String to execute.
-/// @param[out]  ret_tv  Location where result will be saved.
-///
-/// @return Result of the execution.
-void executor_exec_lua(const String str, typval_T *const ret_tv)
-  FUNC_ATTR_NONNULL_ALL
-{
-  lua_State *const lstate = nlua_enter();
-
-  if (luaL_loadbuffer(lstate, str.data, str.size, NLUA_EVAL_NAME)) {
-    nlua_error(lstate, _("E5104: Error while creating lua chunk: %.*s"));
-    return;
-  }
-  if (lua_pcall(lstate, 0, 1, 0)) {
-    nlua_error(lstate, _("E5105: Error while calling lua chunk: %.*s"));
-    return;
-  }
-
-  nlua_pop_typval(lstate, ret_tv);
-}
-
 static void nlua_print_event(void **argv)
 {
   char *str = argv[0];
@@ -508,7 +657,7 @@ int nlua_debug(lua_State *lstate)
   for (;;) {
     lua_settop(lstate, 0);
     typval_T input;
-    get_user_input(input_args, &input, false);
+    get_user_input(input_args, &input, false, false);
     msg_putchar('\n');  // Avoid outputting on input line.
     if (input.v_type != VAR_STRING
         || input.vval.v_string == NULL
@@ -534,6 +683,132 @@ int nlua_in_fast_event(lua_State *lstate)
   return 1;
 }
 
+int nlua_call(lua_State *lstate)
+{
+  Error err = ERROR_INIT;
+  size_t name_len;
+  const char_u *name = (const char_u *)luaL_checklstring(lstate, 1, &name_len);
+  if (!nlua_is_deferred_safe(lstate)) {
+    return luaL_error(lstate, e_luv_api_disabled, "vimL function");
+  }
+
+  int nargs = lua_gettop(lstate)-1;
+  if (nargs > MAX_FUNC_ARGS) {
+    return luaL_error(lstate, "Function called with too many arguments");
+  }
+
+  typval_T vim_args[MAX_FUNC_ARGS + 1];
+  int i = 0;  // also used for freeing the variables
+  for (; i < nargs; i++) {
+    lua_pushvalue(lstate, (int)i+2);
+    if (!nlua_pop_typval(lstate, &vim_args[i])) {
+      api_set_error(&err, kErrorTypeException,
+                    "error converting argument %d", i+1);
+      goto free_vim_args;
+    }
+  }
+
+  TRY_WRAP({
+    // TODO(bfredl): this should be simplified in error handling refactor
+    force_abort = false;
+    suppress_errthrow = false;
+    current_exception = NULL;
+    did_emsg = false;
+
+    try_start();
+    typval_T rettv;
+    int dummy;
+    // call_func() retval is deceptive, ignore it.  Instead we set `msg_list`
+    // (TRY_WRAP) to capture abort-causing non-exception errors.
+    (void)call_func(name, (int)name_len, &rettv, nargs,
+                    vim_args, NULL,
+                    curwin->w_cursor.lnum, curwin->w_cursor.lnum,
+                    &dummy, true, NULL, NULL);
+    if (!try_end(&err)) {
+      nlua_push_typval(lstate, &rettv, false);
+    }
+    tv_clear(&rettv);
+  });
+
+free_vim_args:
+  while (i > 0) {
+    tv_clear(&vim_args[--i]);
+  }
+  if (ERROR_SET(&err)) {
+    lua_pushstring(lstate, err.msg);
+    api_clear_error(&err);
+    return lua_error(lstate);
+  }
+  return 1;
+}
+
+static int nlua_rpcrequest(lua_State *lstate)
+{
+  if (!nlua_is_deferred_safe(lstate)) {
+    return luaL_error(lstate, e_luv_api_disabled, "rpcrequest");
+  }
+  return nlua_rpc(lstate, true);
+}
+
+static int nlua_rpcnotify(lua_State *lstate)
+{
+  return nlua_rpc(lstate, false);
+}
+
+static int nlua_rpc(lua_State *lstate, bool request)
+{
+  size_t name_len;
+  uint64_t chan_id = (uint64_t)luaL_checkinteger(lstate, 1);
+  const char *name = luaL_checklstring(lstate, 2, &name_len);
+  int nargs = lua_gettop(lstate)-2;
+  Error err = ERROR_INIT;
+  Array args = ARRAY_DICT_INIT;
+
+  for (int i = 0; i < nargs; i++) {
+    lua_pushvalue(lstate, (int)i+3);
+    ADD(args, nlua_pop_Object(lstate, false, &err));
+    if (ERROR_SET(&err)) {
+      api_free_array(args);
+      goto check_err;
+    }
+  }
+
+  if (request) {
+    Object result = rpc_send_call(chan_id, name, args, &err);
+    if (!ERROR_SET(&err)) {
+      nlua_push_Object(lstate, result, false);
+      api_free_object(result);
+    }
+  } else {
+    if (!rpc_send_event(chan_id, name, args)) {
+      api_set_error(&err, kErrorTypeValidation,
+                    "Invalid channel: %"PRIu64, chan_id);
+    }
+  }
+
+check_err:
+  if (ERROR_SET(&err)) {
+    lua_pushstring(lstate, err.msg);
+    api_clear_error(&err);
+    return lua_error(lstate);
+  }
+
+  return request ? 1 : 0;
+}
+
+static int nlua_nil_tostring(lua_State *lstate)
+{
+  lua_pushstring(lstate, "vim.NIL");
+  return 1;
+}
+
+static int nlua_empty_dict_tostring(lua_State *lstate)
+{
+  lua_pushstring(lstate, "vim.empty_dict()");
+  return 1;
+}
+
+
 #ifdef WIN32
 /// os.getenv: override os.getenv to maintain coherency. #9681
 ///
@@ -568,12 +843,25 @@ void executor_free_luaref(LuaRef ref)
   nlua_unref(lstate, ref);
 }
 
-/// push a value referenced in the regirstry
+/// push a value referenced in the registry
 void nlua_pushref(lua_State *lstate, LuaRef ref)
 {
   lua_rawgeti(lstate, LUA_REGISTRYINDEX, ref);
 }
 
+/// Gets a new reference to an object stored at original_ref
+///
+/// NOTE: It does not copy the value, it creates a new ref to the lua object.
+///       Leaves the stack unchanged.
+LuaRef nlua_newref(lua_State *lstate, LuaRef original_ref)
+{
+  nlua_pushref(lstate, original_ref);
+  LuaRef new_ref = nlua_ref(lstate, -1);
+  lua_pop(lstate, 1);
+
+  return new_ref;
+}
+
 /// Evaluate lua string
 ///
 /// Used for luaeval().
@@ -587,10 +875,6 @@ void executor_eval_lua(const String str, typval_T *const arg,
                        typval_T *const ret_tv)
   FUNC_ATTR_NONNULL_ALL
 {
-  lua_State *const lstate = nlua_enter();
-
-  garray_T str_ga;
-  ga_init(&str_ga, 1, 80);
 #define EVALHEADER "local _A=select(1,...) return ("
   const size_t lcmd_len = sizeof(EVALHEADER) - 1 + str.size + 1;
   char *lcmd;
@@ -603,35 +887,118 @@ void executor_eval_lua(const String str, typval_T *const arg,
   memcpy(lcmd + sizeof(EVALHEADER) - 1, str.data, str.size);
   lcmd[lcmd_len - 1] = ')';
 #undef EVALHEADER
-  if (luaL_loadbuffer(lstate, lcmd, lcmd_len, NLUA_EVAL_NAME)) {
-    nlua_error(lstate,
-               _("E5107: Error while creating lua chunk for luaeval(): %.*s"));
-    if (lcmd != (char *)IObuff) {
-      xfree(lcmd);
-    }
-    return;
-  }
+  typval_exec_lua(lcmd, lcmd_len, "luaeval()", arg, 1, true, ret_tv);
+
   if (lcmd != (char *)IObuff) {
     xfree(lcmd);
   }
+}
 
-  if (arg->v_type == VAR_UNKNOWN) {
-    lua_pushnil(lstate);
+void executor_call_lua(const char *str, size_t len, typval_T *const args,
+                       int argcount, typval_T *ret_tv)
+  FUNC_ATTR_NONNULL_ALL
+{
+#define CALLHEADER "return "
+#define CALLSUFFIX "(...)"
+  const size_t lcmd_len = sizeof(CALLHEADER) - 1 + len + sizeof(CALLSUFFIX) - 1;
+  char *lcmd;
+  if (lcmd_len < IOSIZE) {
+    lcmd = (char *)IObuff;
   } else {
-    nlua_push_typval(lstate, arg);
+    lcmd = xmalloc(lcmd_len);
+  }
+  memcpy(lcmd, CALLHEADER, sizeof(CALLHEADER) - 1);
+  memcpy(lcmd + sizeof(CALLHEADER) - 1, str, len);
+  memcpy(lcmd + sizeof(CALLHEADER) - 1 + len, CALLSUFFIX,
+         sizeof(CALLSUFFIX) - 1);
+#undef CALLHEADER
+#undef CALLSUFFIX
+
+  typval_exec_lua(lcmd, lcmd_len, "v:lua", args, argcount, false, ret_tv);
+
+  if (lcmd != (char *)IObuff) {
+    xfree(lcmd);
+  }
+}
+
+static void typval_exec_lua(const char *lcmd, size_t lcmd_len, const char *name,
+                            typval_T *const args, int argcount, bool special,
+                            typval_T *ret_tv)
+{
+  if (check_restricted() || check_secure()) {
+    if (ret_tv) {
+      ret_tv->v_type = VAR_NUMBER;
+      ret_tv->vval.v_number = 0;
+    }
+    return;
+  }
+
+  lua_State *const lstate = nlua_enter();
+  if (luaL_loadbuffer(lstate, lcmd, lcmd_len, name)) {
+    nlua_error(lstate, _("E5107: Error loading lua %.*s"));
+    return;
   }
-  if (lua_pcall(lstate, 1, 1, 0)) {
-    nlua_error(lstate,
-               _("E5108: Error while calling lua chunk for luaeval(): %.*s"));
+
+  PUSH_ALL_TYPVALS(lstate, args, argcount, special);
+
+  if (lua_pcall(lstate, argcount, ret_tv ? 1 : 0, 0)) {
+    nlua_error(lstate, _("E5108: Error executing lua %.*s"));
     return;
   }
 
-  nlua_pop_typval(lstate, ret_tv);
+  if (ret_tv) {
+    nlua_pop_typval(lstate, ret_tv);
+  }
+}
+
+/// Call a LuaCallable given some typvals
+///
+/// Used to call any lua callable passed from Lua into VimL
+///
+/// @param[in]  lstate Lua State
+/// @param[in]  lua_cb Lua Callable
+/// @param[in]  argcount Count of typval arguments
+/// @param[in]  argvars Typval Arguments
+/// @param[out] rettv The return value from the called function.
+int typval_exec_lua_callable(
+    lua_State *lstate,
+    LuaCallable lua_cb,
+    int argcount,
+    typval_T *argvars,
+    typval_T *rettv
+)
+{
+  int offset = 0;
+  LuaRef cb = lua_cb.func_ref;
+
+  if (cb == LUA_NOREF) {
+    // This shouldn't happen.
+    luaL_error(lstate, "Invalid function passed to VimL");
+    return ERROR_OTHER;
+  }
+
+  nlua_pushref(lstate, cb);
+
+  if (lua_cb.table_ref != LUA_NOREF) {
+    offset += 1;
+    nlua_pushref(lstate, lua_cb.table_ref);
+  }
+
+  PUSH_ALL_TYPVALS(lstate, argvars, argcount, false);
+
+  if (lua_pcall(lstate, argcount + offset, 1, 0)) {
+    nlua_print(lstate);
+    return ERROR_OTHER;
+  }
+
+  nlua_pop_typval(lstate, rettv);
+
+  return ERROR_NONE;
 }
 
-/// Execute lua string
+/// Execute Lua string
 ///
-/// Used for nvim_execute_lua().
+/// Used for nvim_exec_lua().
 ///
 /// @param[in]  str  String to execute.
 /// @param[in]  args array of ... args
@@ -666,7 +1033,7 @@ Object executor_exec_lua_api(const String str, const Array args, Error *err)
 }
 
 Object executor_exec_lua_cb(LuaRef ref, const char *name, Array args,
-                            bool retval)
+                            bool retval, Error *err)
 {
   lua_State *const lstate = nlua_enter();
   nlua_pushref(lstate, ref);
@@ -676,16 +1043,24 @@ Object executor_exec_lua_cb(LuaRef ref, const char *name, Array args,
   }
 
   if (lua_pcall(lstate, (int)args.size+1, retval ? 1 : 0, 0)) {
-    // TODO(bfredl): callbacks:s might not always be msg-safe, for instance
-    // lua callbacks for redraw events. Later on let the caller deal with the
-    // error instead.
-    nlua_error(lstate, _("Error executing lua callback: %.*s"));
+    // if err is passed, the caller will deal with the error.
+    if (err) {
+      size_t len;
+      const char *errstr = lua_tolstring(lstate, -1, &len);
+      api_set_error(err, kErrorTypeException,
+                    "Error executing lua: %.*s", (int)len, errstr);
+    } else {
+      nlua_error(lstate, _("Error executing lua callback: %.*s"));
+    }
     return NIL;
   }
-  Error err = ERROR_INIT;
 
   if (retval) {
-    return nlua_pop_Object(lstate, false, &err);
+    Error dummy = ERROR_INIT;
+    if (err == NULL) {
+      err = &dummy;
+    }
+    return nlua_pop_Object(lstate, false, err);
   } else {
     return NIL;
   }
@@ -712,9 +1087,8 @@ void ex_lua(exarg_T *const eap)
     xfree(code);
     return;
   }
-  typval_T tv = { .v_type = VAR_UNKNOWN };
-  executor_exec_lua((String) { .data = code, .size = len }, &tv);
-  tv_clear(&tv);
+  typval_exec_lua(code, len, ":lua", NULL, 0, false, NULL);
+
   xfree(code);
 }
 
@@ -752,8 +1126,8 @@ void ex_luado(exarg_T *const eap)
 #undef DOSTART
 #undef DOEND
 
-  if (luaL_loadbuffer(lstate, lcmd, lcmd_len, NLUA_EVAL_NAME)) {
-    nlua_error(lstate, _("E5109: Error while creating lua chunk: %.*s"));
+  if (luaL_loadbuffer(lstate, lcmd, lcmd_len, ":luado")) {
+    nlua_error(lstate, _("E5109: Error loading lua: %.*s"));
     if (lcmd_len >= IOSIZE) {
       xfree(lcmd);
     }
@@ -763,7 +1137,7 @@ void ex_luado(exarg_T *const eap)
     xfree(lcmd);
   }
   if (lua_pcall(lstate, 0, 1, 0)) {
-    nlua_error(lstate, _("E5110: Error while creating lua function: %.*s"));
+    nlua_error(lstate, _("E5110: Error executing lua: %.*s"));
     return;
   }
   for (linenr_T l = eap->line1; l <= eap->line2; l++) {
@@ -774,7 +1148,7 @@ void ex_luado(exarg_T *const eap)
     lua_pushstring(lstate, (const char *)ml_get_buf(curbuf, l, false));
     lua_pushnumber(lstate, (lua_Number)l);
     if (lua_pcall(lstate, 2, 1, 0)) {
-      nlua_error(lstate, _("E5111: Error while calling lua function: %.*s"));
+      nlua_error(lstate, _("E5111: Error calling lua: %.*s"));
       break;
     }
     if (lua_isstring(lstate, -1)) {
@@ -816,3 +1190,270 @@ void ex_luafile(exarg_T *const eap)
     return;
   }
 }
+
+static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
+{
+  tslua_init(lstate);
+
+  lua_pushcfunction(lstate, tslua_push_parser);
+  lua_setfield(lstate, -2, "_create_ts_parser");
+
+  lua_pushcfunction(lstate, tslua_add_language);
+  lua_setfield(lstate, -2, "_ts_add_language");
+
+  lua_pushcfunction(lstate, tslua_has_language);
+  lua_setfield(lstate, -2, "_ts_has_language");
+
+  lua_pushcfunction(lstate, tslua_inspect_lang);
+  lua_setfield(lstate, -2, "_ts_inspect_language");
+
+  lua_pushcfunction(lstate, ts_lua_parse_query);
+  lua_setfield(lstate, -2, "_ts_parse_query");
+}
+
+static int nlua_regex(lua_State *lstate)
+{
+  Error err = ERROR_INIT;
+  const char *text = luaL_checkstring(lstate, 1);
+  regprog_T *prog = NULL;
+
+  TRY_WRAP({
+    try_start();
+    prog = vim_regcomp((char_u *)text, RE_AUTO | RE_MAGIC | RE_STRICT);
+    try_end(&err);
+  });
+
+  if (ERROR_SET(&err)) {
+    return luaL_error(lstate, "couldn't parse regex: %s", err.msg);
+  }
+  assert(prog);
+
+  regprog_T **p = lua_newuserdata(lstate, sizeof(regprog_T *));
+  *p = prog;
+
+  lua_getfield(lstate, LUA_REGISTRYINDEX, "nvim_regex");  // [udata, meta]
+  lua_setmetatable(lstate, -2);  // [udata]
+  return 1;
+}
+
+static regprog_T **regex_check(lua_State *L)
+{
+  return luaL_checkudata(L, 1, "nvim_regex");
+}
+
+
+static int regex_gc(lua_State *lstate)
+{
+  regprog_T **prog = regex_check(lstate);
+  vim_regfree(*prog);
+  return 0;
+}
+
+static int regex_tostring(lua_State *lstate)
+{
+  lua_pushstring(lstate, "<regex>");
+  return 1;
+}
+
+static int regex_match(lua_State *lstate, regprog_T **prog, char_u *str)
+{
+  regmatch_T rm;
+  rm.regprog = *prog;
+  rm.rm_ic = false;
+  bool match = vim_regexec(&rm, str, 0);
+  *prog = rm.regprog;
+
+  if (match) {
+    lua_pushinteger(lstate, (lua_Integer)(rm.startp[0]-str));
+    lua_pushinteger(lstate, (lua_Integer)(rm.endp[0]-str));
+    return 2;
+  }
+  return 0;
+}
+
+static int regex_match_str(lua_State *lstate)
+{
+  regprog_T **prog = regex_check(lstate);
+  const char *str = luaL_checkstring(lstate, 2);
+  int nret = regex_match(lstate, prog, (char_u *)str);
+
+  if (!*prog) {
+    return luaL_error(lstate, "regex: internal error");
+  }
+
+  return nret;
+}
+
+static int regex_match_line(lua_State *lstate)
+{
+  regprog_T **prog = regex_check(lstate);
+
+  int narg = lua_gettop(lstate);
+  if (narg < 3) {
+    return luaL_error(lstate, "not enough args");
+  }
+
+  long bufnr = luaL_checkinteger(lstate, 2);
+  long rownr = luaL_checkinteger(lstate, 3);
+  long start = 0, end = -1;
+  if (narg >= 4) {
+