lua: Add ability to pass lua functions directly to vimL

8 files changed, 265 insertions, 20 deletions

diff --git a/src/nvim/eval/typval.h b/src/nvim/eval/typval.h
index 343dd205ff..cb13bec50d 100644
--- a/src/nvim/eval/typval.h
+++ b/src/nvim/eval/typval.h
@@ -271,6 +271,11 @@ typedef struct {
 /// Number of fixed variables used for arguments
 #define FIXVAR_CNT 12
 
+/// Callback interface for C function reference
+typedef int (*cfunc_T)(int argcount, typval_T *argvars, typval_T *rettv, void *state);  // NOLINT
+/// Callback to clear cfunc_T
+typedef void (*cfunc_free_T)(void *state);
+
 // Structure to hold info for a function that is currently being executed.
 typedef struct funccall_S funccall_T;
 
@@ -307,6 +312,10 @@ struct ufunc {
   garray_T     uf_lines;         ///< function lines
   int          uf_profiling;     ///< true when func is being profiled
   int          uf_prof_initialized;
+  // Managing cfuncs
+  cfunc_T      uf_cb;            ///< C function extension callback
+  cfunc_free_T uf_cb_free;       ///< C function extesion free callback
+  void        *uf_cb_state;      ///< State of C function extension.
   // Profiling the function as a whole.
   int          uf_tm_count;      ///< nr of calls
   proftime_T   uf_tm_total;      ///< time spent in function + children
diff --git a/src/nvim/eval/userfunc.c b/src/nvim/eval/userfunc.c
index 4d658498c1..ca2f4a32ca 100644
--- a/src/nvim/eval/userfunc.c
+++ b/src/nvim/eval/userfunc.c
@@ -32,6 +32,7 @@
 #define FC_DELETED  0x10          // :delfunction used while uf_refcount > 0
 #define FC_REMOVED  0x20          // function redefined while uf_refcount > 0
 #define FC_SANDBOX  0x40          // function defined in the sandbox
+#define FC_CFUNC    0x80          // C function extension
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 #include "eval/userfunc.c.generated.h"
@@ -162,6 +163,17 @@ static void register_closure(ufunc_T *fp)
     [current_funccal->fc_funcs.ga_len++] = fp;
 }
 
+
+/// Get a name for a lambda.  Returned in static memory.
+char_u * get_lambda_name(void)
+{
+    static char_u   name[30];
+    static int      lambda_no = 0;
+
+    snprintf((char *)name, sizeof(name), "<lambda>%d", ++lambda_no);
+    return name;
+}
+
 /// Parse a lambda expression and get a Funcref from "*arg".
 ///
 /// @return OK or FAIL.  Returns NOTDONE for dict or {expr}.
@@ -175,7 +187,6 @@ int get_lambda_tv(char_u **arg, typval_T *rettv, bool evaluate)
   int        ret;
   char_u     *start = skipwhite(*arg + 1);
   char_u     *s, *e;
-  static int lambda_no = 0;
   bool       *old_eval_lavars = eval_lavars_used;
   bool       eval_lavars = false;
 
@@ -219,11 +230,9 @@ int get_lambda_tv(char_u **arg, typval_T *rettv, bool evaluate)
   if (evaluate) {
     int len, flags = 0;
     char_u *p;
-    char_u name[20];
     garray_T newlines;
 
-    lambda_no++;
-    snprintf((char *)name, sizeof(name), "<lambda>%d", lambda_no);
+    char_u *name = get_lambda_name();
 
     fp = xcalloc(1, offsetof(ufunc_T, uf_name) + STRLEN(name) + 1);
     pt = xcalloc(1, sizeof(partial_T));
@@ -700,6 +709,11 @@ static void func_clear_items(ufunc_T *fp)
   ga_clear_strings(&(fp->uf_args));
   ga_clear_strings(&(fp->uf_lines));
 
+  if (fp->uf_cb_free != NULL) {
+    fp->uf_cb_free(fp->uf_cb_state);
+    fp->uf_cb_free = NULL;
+  }
+
   XFREE_CLEAR(fp->uf_tml_count);
   XFREE_CLEAR(fp->uf_tml_total);
   XFREE_CLEAR(fp->uf_tml_self);
@@ -1408,6 +1422,9 @@ call_func(
 
       if (fp != NULL && (fp->uf_flags & FC_DELETED)) {
         error = ERROR_DELETED;
+      } else if (fp != NULL && (fp->uf_flags & FC_CFUNC)) {
+        cfunc_T cb = fp->uf_cb;
+        error = (*cb)(argcount, argvars, rettv, fp->uf_cb_state);
       } else if (fp != NULL) {
         if (argv_func != NULL) {
           // postponed filling in the arguments, do it now
@@ -3435,3 +3452,30 @@ bool set_ref_in_func(char_u *name, ufunc_T *fp_in, int copyID)
   xfree(tofree);
   return abort;
 }
+
+/// Registers a C extension user function.
+char_u *register_cfunc(cfunc_T cb, cfunc_free_T cb_free, void *state)
+{
+  char_u *name = get_lambda_name();
+  ufunc_T *fp = NULL;
+  int flags = FC_CFUNC;
+
+  fp = xcalloc(1, offsetof(ufunc_T, uf_name) + STRLEN(name) + 1);
+  if (fp == NULL) {
+    return NULL;
+  }
+
+  fp->uf_refcount = 1;
+  fp->uf_varargs = true;
+  fp->uf_flags = flags;
+  fp->uf_calls = 0;
+  fp->uf_script_ctx = current_sctx;
+  fp->uf_cb = cb;
+  fp->uf_cb_free = cb_free;
+  fp->uf_cb_state = state;
+
+  STRCPY(fp->uf_name, name);
+  hash_add(&func_hashtab, UF2HIKEY(fp));
+
+  return name;
+}
diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c
index 69114c967d..9fc4ca7e7e 100644
--- a/src/nvim/lua/converter.c
+++ b/src/nvim/lua/converter.c
@@ -19,6 +19,7 @@
 #include "nvim/globals.h"
 #include "nvim/message.h"
 #include "nvim/eval/typval.h"
+#include "nvim/eval/userfunc.h"
 #include "nvim/ascii.h"
 #include "nvim/macros.h"
 
@@ -50,6 +51,7 @@ typedef struct {
 #define LUA_PUSH_STATIC_STRING(lstate, s) \
     lua_pushlstring(lstate, s, sizeof(s) - 1)
 
+
 static LuaTableProps nlua_traverse_table(lua_State *const lstate)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
@@ -384,6 +386,20 @@ bool nlua_pop_typval(lua_State *lstate, typval_T *ret_tv)
 nlua_pop_typval_table_processing_end:
         break;
       }
+      case LUA_TFUNCTION: {
+        LuaCFunctionState *state = xmalloc(sizeof(LuaCFunctionState));
+
+        state->lua_callable.func_ref = nlua_ref(lstate, -1);
+
+        char_u *name = register_cfunc(
+            &nlua_CFunction_func_call,
+            &nlua_CFunction_func_free,
+            state);
+
+        cur.tv->v_type = VAR_FUNC;
+        cur.tv->vval.v_string = vim_strsave(name);
+        break;
+      }
       case LUA_TUSERDATA: {
         nlua_pushref(lstate, nlua_nil_ref);
         bool is_nil = lua_rawequal(lstate, -2, -1);
diff --git a/src/nvim/lua/converter.h b/src/nvim/lua/converter.h
index 542c56ea3e..e74e3fb084 100644
--- a/src/nvim/lua/converter.h
+++ b/src/nvim/lua/converter.h
@@ -9,6 +9,14 @@
 #include "nvim/func_attr.h"
 #include "nvim/eval.h"
 
+typedef struct {
+    LuaRef func_ref;
+} LuaCallable;
+
+typedef struct {
+    LuaCallable lua_callable;
+} LuaCFunctionState;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "lua/converter.h.generated.h"
 #endif
diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c
index 4b47b34d8a..5e924a9f90 100644
--- a/src/nvim/lua/executor.c
+++ b/src/nvim/lua/executor.c
@@ -35,8 +35,8 @@
 #include "nvim/os/os.h"
 #endif
 
-#include "nvim/lua/executor.h"
 #include "nvim/lua/converter.h"
+#include "nvim/lua/executor.h"
 #include "nvim/lua/treesitter.h"
 
 #include "luv/luv.h"
@@ -833,7 +833,7 @@ void executor_free_luaref(LuaRef ref)
   nlua_unref(lstate, ref);
 }
 
-/// push a value referenced in the regirstry
+/// push a value referenced in the registry
 void nlua_pushref(lua_State *lstate, LuaRef ref)
 {
   lua_rawgeti(lstate, LUA_REGISTRYINDEX, ref);
@@ -933,6 +933,33 @@ static void typval_exec_lua(const char *lcmd, size_t lcmd_len, const char *name,
   }
 }
 
+/// Call a LuaCallable given some typvals
+int typval_exec_lua_callable(
+    lua_State *lstate,
+    LuaCallable lua_cb,
+    int argcount,
+    typval_T *argvars,
+    typval_T *rettv
+)
+{
+    LuaRef cb = lua_cb.func_ref;
+
+    nlua_pushref(lstate, cb);
+
+    for (int i = 0; i < argcount; i++) {
+        nlua_push_typval(lstate, &argvars[i], false);
+    }
+
+    if (lua_pcall(lstate, argcount, 1, 0)) {
+        luaL_error(lstate, "nlua_CFunction_func_call failed.");
+        return ERROR_OTHER;
+    }
+
+    nlua_pop_typval(lstate, rettv);
+
+    return ERROR_NONE;
+}
+
 /// Execute Lua string
 ///
 /// Used for nvim_exec_lua().
@@ -1280,3 +1307,31 @@ static int regex_match_line(lua_State *lstate)
 
   return nret;
 }
+
+int nlua_CFunction_func_call(
+    int argcount,
+    typval_T *argvars,
+    typval_T *rettv,
+    void *state)
+{
+    lua_State *const lstate = nlua_enter();
+    LuaCFunctionState *funcstate = (LuaCFunctionState *)state;
+
+    return typval_exec_lua_callable(
+        lstate,
+        funcstate->lua_callable,
+        argcount,
+        argvars,
+        rettv);
+}
+/// Required functions for lua c functions as VimL callbacks
+void nlua_CFunction_func_free(void *state)
+{
+    lua_State *const lstate = nlua_enter();
+    LuaCFunctionState *funcstate = (LuaCFunctionState *)state;
+
+    nlua_unref(lstate, funcstate->lua_callable.func_ref);
+    xfree(funcstate);
+}
+
+
diff --git a/src/nvim/lua/executor.h b/src/nvim/lua/executor.h
index 3259fc0fa1..6599b44584 100644
--- a/src/nvim/lua/executor.h
+++ b/src/nvim/lua/executor.h
@@ -8,6 +8,7 @@
 #include "nvim/func_attr.h"
 #include "nvim/eval/typval.h"
 #include "nvim/ex_cmds_defs.h"
+#include "nvim/lua/converter.h"
 
 // Generated by msgpack-gen.lua
 void nlua_add_api_functions(lua_State *lstate) REAL_FATTR_NONNULL_ALL;
diff --git a/src/nvim/lua/vim.lua b/src/nvim/lua/vim.lua
index 18256242e8..820b237c4f 100644
--- a/src/nvim/lua/vim.lua
+++ b/src/nvim/lua/vim.lua
@@ -272,6 +272,10 @@ vim.fn = setmetatable({}, {
   end
 })
 
+vim.funcref = function(viml_func_name)
+  return vim.fn[viml_func_name]
+end
+
 -- These are for loading runtime modules lazily since they aren't available in
 -- the nvim binary as specified in executor.c
 local function __index(t, key)
diff --git a/test/functional/lua/luaeval_spec.lua b/test/functional/lua/luaeval_spec.lua
index 61c8e5c02e..4faacecd86 100644
--- a/test/functional/lua/luaeval_spec.lua
+++ b/test/functional/lua/luaeval_spec.lua
@@ -2,7 +2,6 @@
 local helpers = require('test.functional.helpers')(after_each)
 local Screen = require('test.functional.ui.screen')
 
-local redir_exec = helpers.redir_exec
 local pcall_err = helpers.pcall_err
 local exc_exec = helpers.exc_exec
 local exec_lua = helpers.exec_lua
@@ -188,23 +187,132 @@ describe('luaeval()', function()
   it('issues an error in some cases', function()
     eq("Vim(call):E5100: Cannot convert given lua table: table should either have a sequence of positive integer keys or contain only string keys",
        exc_exec('call luaeval("{1, foo=2}")'))
-    eq("Vim(call):E5101: Cannot convert given lua type",
-       exc_exec('call luaeval("vim.api.nvim_buf_get_lines")'))
+
     startswith("Vim(call):E5107: Error loading lua [string \"luaeval()\"]:",
                exc_exec('call luaeval("1, 2, 3")'))
     startswith("Vim(call):E5108: Error executing lua [string \"luaeval()\"]:",
                exc_exec('call luaeval("(nil)()")'))
-    eq("Vim(call):E5101: Cannot convert given lua type",
-       exc_exec('call luaeval("{42, vim.api}")'))
-    eq("Vim(call):E5101: Cannot convert given lua type",
-       exc_exec('call luaeval("{foo=42, baz=vim.api}")'))
-
-    -- The following should not crash: conversion error happens inside
-    eq("Vim(call):E5101: Cannot convert given lua type",
-       exc_exec('call luaeval("vim.api")'))
-    -- The following should not show internal error
-    eq("\nE5101: Cannot convert given lua type\n0",
-       redir_exec('echo luaeval("vim.api")'))
+
+  end)
+
+  it('should handle sending lua functions to viml', function()
+    eq(true, exec_lua [[
+      can_pass_lua_callback_to_vim_from_lua_result = nil
+
+      vim.fn.call(function()
+        can_pass_lua_callback_to_vim_from_lua_result = true
+      end, {})
+
+      return can_pass_lua_callback_to_vim_from_lua_result
+    ]])
+  end)
+
+  it('run functions even in timers', function()
+    eq(true, exec_lua [[
+      can_pass_lua_callback_to_vim_from_lua_result = nil
+
+      vim.fn.timer_start(50, function()
+        can_pass_lua_callback_to_vim_from_lua_result = true
+      end)
+
+      vim.wait(1000, function()
+        return can_pass_lua_callback_to_vim_from_lua_result
+      end)
+
+      return can_pass_lua_callback_to_vim_from_lua_result
+    ]])
+  end)
+
+  it('can run named functions more than once', function()
+    eq(5, exec_lua [[
+      count_of_vals = 0
+
+      vim.fn.timer_start(5, function()
+        count_of_vals = count_of_vals + 1
+      end, {['repeat'] = 5})
+
+      vim.fn.wait(1000, function()
+        return count_of_vals >= 5
+      end)
+
+      return count_of_vals
+    ]])
+  end)
+
+  it('can handle clashing names', function()
+    eq(1, exec_lua [[
+      local f_loc = function() return 1 end
+
+      local result = nil
+      vim.fn.timer_start(100, function()
+        result = f_loc()
+      end)
+
+      local f_loc = function() return 2 end
+      vim.wait(1000, function() return result ~= nil end)
+
+      return result
+    ]])
+  end)
+
+  it('should handle passing functions around', function()
+    command [[
+      function VimCanCallLuaCallbacks(Concat, Cb)
+        let message = a:Concat("Hello Vim", "I'm Lua")
+        call a:Cb(message)
+      endfunction
+    ]]
+
+    eq("Hello Vim I'm Lua", exec_lua [[
+      can_pass_lua_callback_to_vim_from_lua_result = ""
+
+      vim.fn.VimCanCallLuaCallbacks(
+        function(greeting, message) return greeting .. " " .. message end,
+        function(message) can_pass_lua_callback_to_vim_from_lua_result = message end
+      )
+
+      return can_pass_lua_callback_to_vim_from_lua_result
+    ]])
+  end)
+
+  it('should handle funcrefs', function()
+    command [[
+      function VimCanCallLuaCallbacks(Concat, Cb)
+        let message = a:Concat("Hello Vim", "I'm Lua")
+        call a:Cb(message)
+      endfunction
+    ]]
+
+    eq("Hello Vim I'm Lua", exec_lua [[
+      can_pass_lua_callback_to_vim_from_lua_result = ""
+
+      vim.funcref('VimCanCallLuaCallbacks')(
+        function(greeting, message) return greeting .. " " .. message end,
+        function(message) can_pass_lua_callback_to_vim_from_lua_result = message end
+      )
+
+      return can_pass_lua_callback_to_vim_from_lua_result
+    ]])
+  end)
+
+  -- TODO(tjdevries): Need to figure
+  pending('should work with metatables using __call', function()
+    eq(true, exec_lua [[
+      local this_is_local_variable = false
+      local callable_table = setmetatable({}, {
+        __call = function(...)
+          this_is_local_variable = true
+        end
+      })
+
+      vim.fn.timer_start(5, callable_table)
+
+      vim.wait(1000, function()
+        return this_is_local_variable
+      end)
+
+      return this_is_local_variable
+    ]])
   end)
 
   it('correctly converts containers with type_idx', function()