api: allow nvim_buf_attach from lua using callbacks

20 files changed, 301 insertions, 59 deletions

diff --git a/runtime/doc/api.txt b/runtime/doc/api.txt
index 22ad8e0633..e127ccae0c 100644
--- a/runtime/doc/api.txt
+++ b/runtime/doc/api.txt
@@ -199,6 +199,21 @@ paste a block of 6 lines, emits: >
 User reloads the buffer with ":edit", emits: >
   nvim_buf_detach_event[{buf}]
 
+                                                        *api-buffer-updates-lua*
+In-process lua plugins can also recieve buffer updates, in the form of lua
+callbacks. These callbacks are called frequently in various contexts, buffer
+contents or window layout should not be changed inside these |textlock|.
+
+|nvim_buf_attach| will take keyword args for the callbacks. "on_lines" will
+receive parameters ("lines", {buf}, {changedtick}, {firstline}, {lastline}, {new_lastline}).
+Unlike remote channels the text contents are not passed. The new text can be
+accessed inside the callback as
+`vim.api.nvim_buf_get_lines(buf, firstline, new_lastline, true)`
+"on_changedtick" is invoked when |b:changedtick| was incremented but no text
+was changed. The parameters recieved are ("changedtick", {buf}, {changedtick}).
+
+
+
 ==============================================================================
 Buffer highlighting					       *api-highlights*
 
diff --git a/src/clint.py b/src/clint.py
index 3e48ead7bf..862fdbc06b 100755
--- a/src/clint.py
+++ b/src/clint.py
@@ -2539,6 +2539,8 @@ def CheckSpacing(filename, clean_lines, linenum, nesting_state, error):
                    r'(?<!\bkbtree_t)'
                    r'(?<!\bkbitr_t)'
                    r'(?<!\bPMap)'
+                   r'(?<!\bArrayOf)'
+                   r'(?<!\bDictionaryOf)'
                    r'\((?:const )?(?:struct )?[a-zA-Z_]\w*(?: *\*(?:const)?)*\)'
                    r' +'
                    r'-?(?:\*+|&)?(?:\w+|\+\+|--|\()', cast_line)
diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c
index 06d7c1810c..81b3851c53 100644
--- a/src/nvim/api/buffer.c
+++ b/src/nvim/api/buffer.c
@@ -7,6 +7,7 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <limits.h>
+#include <lauxlib.h>
 
 #include "nvim/api/buffer.h"
 #include "nvim/api/private/helpers.h"
@@ -98,37 +99,62 @@ String buffer_get_line(Buffer buffer, Integer index, Error *err)
   return rv;
 }
 
-/// Activates buffer-update events on the channel.
+/// Activates buffer-update events on a channel, or as lua callbacks.
 ///
 /// @param channel_id
 /// @param buffer Buffer handle, or 0 for current buffer
 /// @param send_buffer Set to true if the initial notification should contain
 ///        the whole buffer. If so, the first notification will be a
 ///        `nvim_buf_lines_event`. Otherwise, the first notification will be
-///        a `nvim_buf_changedtick_event`
-/// @param  opts  Optional parameters. Reserved for future use.
+///        a `nvim_buf_changedtick_event`. Not used for lua callbacks.
+/// @param  opts  Optional parameters.
+///               `on_lines`: lua callback received on change.
+///               `on_changedtick`: lua callback received on changedtick
+///                                 increment without text change.
+///               See |api-buffer-updates-lua| for more information
 /// @param[out] err Error details, if any
 /// @return False when updates couldn't be enabled because the buffer isn't
 ///         loaded or `opts` contained an invalid key; otherwise True.
+///         TODO: LUA_API_NO_EVAL
 Boolean nvim_buf_attach(uint64_t channel_id,
                         Buffer buffer,
                         Boolean send_buffer,
-                        Dictionary opts,
+                        DictionaryOf(LuaRef) opts,
                         Error *err)
-  FUNC_API_SINCE(4) FUNC_API_REMOTE_ONLY
+  FUNC_API_SINCE(4)
 {
-  if (opts.size > 0) {
-      api_set_error(err, kErrorTypeValidation, "dict isn't empty");
-      return false;
-  }
-
   buf_T *buf = find_buffer_by_handle(buffer, err);
 
   if (!buf) {
     return false;
   }
 
-  return buf_updates_register(buf, channel_id, send_buffer);
+  bool is_lua = (channel_id == LUA_INTERNAL_CALL);
+  BufUpdateCallbacks cb = BUF_UPDATE_CALLBACKS_INIT;
+  for (size_t i = 0; i < opts.size; i++) {
+    String k = opts.items[i].key;
+    Object *v = &opts.items[i].value;
+    if (is_lua && strequal("on_lines", k.data)) {
+      if (v->type != kObjectTypeLuaRef) {
+        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+        return false;
+      }
+      cb.on_lines = v->data.luaref;
+      v->data.integer = LUA_NOREF;
+    } else if (is_lua && strequal("on_changedtick", k.data)) {
+      if (v->type != kObjectTypeLuaRef) {
+        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+        return false;
+      }
+      cb.on_changedtick = v->data.luaref;
+      v->data.integer = LUA_NOREF;
+    } else {
+      api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
+      return false;
+    }
+  }
+
+  return buf_updates_register(buf, channel_id, cb, send_buffer);
 }
 
 /// Deactivates buffer-update events on the channel.
@@ -307,7 +333,7 @@ void buffer_set_line_slice(Buffer buffer,
                            Integer end,
                            Boolean include_start,
                            Boolean include_end,
-                           ArrayOf(String) replacement,  // NOLINT
+                           ArrayOf(String) replacement,
                            Error *err)
 {
   start = convert_index(start) + !include_start;
@@ -340,7 +366,7 @@ void nvim_buf_set_lines(uint64_t channel_id,
                         Integer start,
                         Integer end,
                         Boolean strict_indexing,
-                        ArrayOf(String) replacement,  // NOLINT
+                        ArrayOf(String) replacement,
                         Error *err)
   FUNC_API_SINCE(1)
 {
diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index 978c55691b..f0d48bf145 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -104,6 +104,7 @@ typedef enum {
   kObjectTypeString,
   kObjectTypeArray,
   kObjectTypeDictionary,
+  kObjectTypeLuaRef,
   // EXT types, cannot be split or reordered, see #EXT_OBJECT_TYPE_SHIFT
   kObjectTypeBuffer,
   kObjectTypeWindow,
@@ -119,6 +120,7 @@ struct object {
     String string;
     Array array;
     Dictionary dictionary;
+    LuaRef luaref;
   } data;
 };
 
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index 521ec11906..6b05d1ac0a 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -11,6 +11,7 @@
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/handle.h"
 #include "nvim/msgpack_rpc/helpers.h"
+#include "nvim/lua/executor.h"
 #include "nvim/ascii.h"
 #include "nvim/assert.h"
 #include "nvim/vim.h"
@@ -1147,6 +1148,10 @@ void api_free_object(Object value)
       api_free_dictionary(value.data.dictionary);
       break;
 
+    case kObjectTypeLuaRef:
+      executor_free_luaref(value.data.luaref);
+      break;
+
     default:
       abort();
   }
diff --git a/src/nvim/api/private/helpers.h b/src/nvim/api/private/helpers.h
index cc74824402..0ea7667428 100644
--- a/src/nvim/api/private/helpers.h
+++ b/src/nvim/api/private/helpers.h
@@ -48,6 +48,10 @@
     .type = kObjectTypeDictionary, \
     .data.dictionary = d })
 
+#define LUAREF_OBJ(r) ((Object) { \
+    .type = kObjectTypeLuaRef, \
+    .data.luaref = r })
+
 #define NIL ((Object) {.type = kObjectTypeNil})
 
 #define PUT(dict, k, v) \
diff --git a/src/nvim/buffer.c b/src/nvim/buffer.c
index 0c14656b33..52dc359716 100644
--- a/src/nvim/buffer.c
+++ b/src/nvim/buffer.c
@@ -1836,6 +1836,8 @@ buf_T * buflist_new(char_u *ffname, char_u *sfname, linenr_T lnum, int flags)
   buf->b_p_bl = (flags & BLN_LISTED) ? true : false;    // init 'buflisted'
   kv_destroy(buf->update_channels);
   kv_init(buf->update_channels);
+  kv_destroy(buf->update_callbacks);
+  kv_init(buf->update_callbacks);
   if (!(flags & BLN_DUMMY)) {
     // Tricky: these autocommands may change the buffer list.  They could also
     // split the window with re-using the one empty buffer. This may result in
diff --git a/src/nvim/buffer_defs.h b/src/nvim/buffer_defs.h
index 255aeb82b6..117a9183a4 100644
--- a/src/nvim/buffer_defs.h
+++ b/src/nvim/buffer_defs.h
@@ -453,6 +453,12 @@ typedef struct {
 /// Primary exists so that literals of relevant type can be made.
 typedef TV_DICTITEM_STRUCT(sizeof("changedtick")) ChangedtickDictItem;
 
+typedef struct {
+  LuaRef on_lines;
+  LuaRef on_changedtick;
+} BufUpdateCallbacks;
+#define BUF_UPDATE_CALLBACKS_INIT { LUA_NOREF, LUA_NOREF }
+
 #define BUF_HAS_QF_ENTRY 1
 #define BUF_HAS_LL_ENTRY 2
 
@@ -796,6 +802,7 @@ struct file_buffer {
   // array of channelids which have asked to receive updates for this
   // buffer.
   kvec_t(uint64_t) update_channels;
+  kvec_t(BufUpdateCallbacks) update_callbacks;
 
   int b_diff_failed;    // internal diff failed for this buffer
 };
diff --git a/src/nvim/buffer_updates.c b/src/nvim/buffer_updates.c
index 9d9c998a68..2515e3f8aa 100644
--- a/src/nvim/buffer_updates.c
+++ b/src/nvim/buffer_updates.c
@@ -5,19 +5,30 @@
 #include "nvim/memline.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/msgpack_rpc/channel.h"
+#include "nvim/lua/executor.h"
 #include "nvim/assert.h"
 #include "nvim/buffer.h"
 
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "buffer_updates.c.generated.h"
+#endif
+
 // Register a channel. Return True if the channel was added, or already added.
 // Return False if the channel couldn't be added because the buffer is
 // unloaded.
-bool buf_updates_register(buf_T *buf, uint64_t channel_id, bool send_buffer)
+bool buf_updates_register(buf_T *buf, uint64_t channel_id,
+                          BufUpdateCallbacks cb, bool send_buffer)
 {
   // must fail if the buffer isn't loaded
   if (buf->b_ml.ml_mfp == NULL) {
     return false;
   }
 
+  if (channel_id == LUA_INTERNAL_CALL) {
+    kv_push(buf->update_callbacks, cb);
+    return true;
+  }
+
   // count how many channels are currently watching the buffer
   size_t size = kv_size(buf->update_channels);
   if (size) {
@@ -69,6 +80,11 @@ bool buf_updates_register(buf_T *buf, uint64_t channel_id, bool send_buffer)
   return true;
 }
 
+bool buf_updates_active(buf_T *buf)
+{
+    return kv_size(buf->update_channels) || kv_size(buf->update_callbacks);
+}
+
 void buf_updates_send_end(buf_T *buf, uint64_t channelid)
 {
     Array args = ARRAY_DICT_INIT;
@@ -125,6 +141,12 @@ void buf_updates_unregister_all(buf_T *buf)
     kv_destroy(buf->update_channels);
     kv_init(buf->update_channels);
   }
+
+  for (size_t i = 0; i < kv_size(buf->update_callbacks); i++) {
+    free_update_callbacks(kv_A(buf->update_callbacks, i));
+  }
+  kv_destroy(buf->update_callbacks);
+  kv_init(buf->update_callbacks);
 }
 
 void buf_updates_send_changes(buf_T *buf,
@@ -133,6 +155,10 @@ void buf_updates_send_changes(buf_T *buf,
                               int64_t num_removed,
                               bool send_tick)
 {
+  if (!buf_updates_active(buf)) {
+    return;
+  }
+
   // if one the channels doesn't work, put its ID here so we can remove it later
   uint64_t badchannelid = 0;
 
@@ -183,6 +209,47 @@ void buf_updates_send_changes(buf_T *buf,
     ELOG("Disabling buffer updates for dead channel %"PRIu64, badchannelid);
     buf_updates_unregister(buf, badchannelid);
   }
+
+  // notify each of the active channels
+  size_t j = 0;
+  for (size_t i = 0; i < kv_size(buf->update_callbacks); i++) {
+    BufUpdateCallbacks cb = kv_A(buf->update_callbacks, i);
+    bool keep = true;
+    if (cb.on_lines != LUA_NOREF) {
+      Array args = ARRAY_DICT_INIT;
+      Object items[5];
+      args.size = 5;
+      args.items = items;
+
+      // the first argument is always the buffer handle
+      args.items[0] = BUFFER_OBJ(buf->handle);
+
+      // next argument is b:changedtick
+      args.items[1] = send_tick ? INTEGER_OBJ(buf_get_changedtick(buf)) : NIL;
+
+      // the first line that changed (zero-indexed)
+      args.items[2] = INTEGER_OBJ(firstline - 1);
+
+      // the last line that was changed
+      args.items[3] = INTEGER_OBJ(firstline - 1 + num_removed);
+
+      // the last line in the updated range
+      args.items[4] = INTEGER_OBJ(firstline - 1 + num_added);
+
+      textlock++;
+      Object res = executor_exec_lua_cb(cb.on_lines, "lines", args);
+      textlock--;
+
+      if (res.type == kObjectTypeBoolean && res.data.boolean == true) {
+        free_update_callbacks(cb);
+        keep = false;
+      }
+    }
+    if (keep) {
+      kv_A(buf->update_callbacks, j++) = kv_A(buf->update_callbacks, i);
+    }
+  }
+  kv_size(buf->update_callbacks) = j;
 }
 
 void buf_updates_changedtick(buf_T *buf)
@@ -192,6 +259,36 @@ void buf_updates_changedtick(buf_T *buf)
     uint64_t channel_id = kv_A(buf->update_channels, i);
     buf_updates_changedtick_single(buf, channel_id);
   }
+  size_t j = 0;
+  for (size_t i = 0; i < kv_size(buf->update_callbacks); i++) {
+    BufUpdateCallbacks cb = kv_A(buf->update_callbacks, i);
+    bool keep = true;
+    if (cb.on_changedtick != LUA_NOREF) {
+      Array args = ARRAY_DICT_INIT;
+      Object items[2];
+      args.size = 2;
+      args.items = items;
+
+      // the first argument is always the buffer handle
+      args.items[0] = BUFFER_OBJ(buf->handle);
+
+      // next argument is b:changedtick
+      args.items[1] = INTEGER_OBJ(buf_get_changedtick(buf));
+
+      textlock++;
+      Object res = executor_exec_lua_cb(cb.on_changedtick, "changedtick", args);
+      textlock--;
+
+      if (res.type == kObjectTypeBoolean && res.data.boolean == true) {
+        free_update_callbacks(cb);
+        keep = false;
+      }
+    }
+    if (keep) {
+      kv_A(buf->update_callbacks, j++) = kv_A(buf->update_callbacks, i);
+    }
+  }
+  kv_size(buf->update_callbacks) = j;
 }
 
 void buf_updates_changedtick_single(buf_T *buf, uint64_t channel_id)
@@ -209,3 +306,9 @@ void buf_updates_changedtick_single(buf_T *buf, uint64_t channel_id)
     // don't try and clean up dead channels here
     rpc_send_event(channel_id, "nvim_buf_changedtick_event", args);
 }
+
+static void free_update_callbacks(BufUpdateCallbacks cb)
+{
+  executor_free_luaref(cb.on_lines);
+  executor_free_luaref(cb.on_changedtick);
+}
diff --git a/src/nvim/ex_cmds.c b/src/nvim/ex_cmds.c
index 8722c03204..1dd76553d4 100644
--- a/src/nvim/ex_cmds.c
+++ b/src/nvim/ex_cmds.c
@@ -900,9 +900,7 @@ int do_move(linenr_T line1, linenr_T line2, linenr_T dest)
   changed_lines(last_line - num_lines + 1, 0, last_line + 1, -extra, false);
 
   // send update regarding the new lines that were added
-  if (kv_size(curbuf->update_channels)) {
-    buf_updates_send_changes(curbuf, dest + 1, num_lines, 0, true);
-  }
+  buf_updates_send_changes(curbuf, dest + 1, num_lines, 0, true);
 
   /*
    * Now we delete the original text -- webb
@@ -939,9 +937,7 @@ int do_move(linenr_T line1, linenr_T line2, linenr_T dest)
   }
 
   // send nvim_buf_lines_event regarding lines that were deleted
-  if (kv_size(curbuf->update_channels)) {
-    buf_updates_send_changes(curbuf, line1 + extra, 0, num_lines, true);
-  }
+  buf_updates_send_changes(curbuf, line1 + extra, 0, num_lines, true);
 
   return OK;
 }
@@ -4074,12 +4070,10 @@ skip:
     i = curbuf->b_ml.ml_line_count - old_line_count;
     changed_lines(first_line, 0, last_line - i, i, false);
 
-    if (kv_size(curbuf->update_channels)) {
-      int64_t num_added = last_line - first_line;
-      int64_t num_removed = num_added - i;
-      buf_updates_send_changes(curbuf, first_line, num_added, num_removed,
-                               do_buf_event);
-    }
+    int64_t num_added = last_line - first_line;
+    int64_t num_removed = num_added - i;
+    buf_updates_send_changes(curbuf, first_line, num_added, num_removed,
+                             do_buf_event);
   }
 
   xfree(sub_firstline);   /* may have to free allocated copy of the line */
diff --git a/src/nvim/fold.c b/src/nvim/fold.c
index 39975308d7..72d8c14468 100644
--- a/src/nvim/fold.c
+++ b/src/nvim/fold.c
@@ -737,15 +737,13 @@ void deleteFold(
     changed_lines(first_lnum, (colnr_T)0, last_lnum, 0L, false);
 
     // send one nvim_buf_lines_event at the end
-    if (kv_size(curbuf->update_channels)) {
-      // last_lnum is the line *after* the last line of the outermost fold
-      // that was modified. Note also that deleting a fold might only require
-      // the modification of the *first* line of the fold, but we send through a
-      // notification that includes every line that was part of the fold
-      int64_t num_changed = last_lnum - first_lnum;
-      buf_updates_send_changes(curbuf, first_lnum, num_changed,
-                               num_changed, true);
-    }
+    // last_lnum is the line *after* the last line of the outermost fold
+    // that was modified. Note also that deleting a fold might only require
+    // the modification of the *first* line of the fold, but we send through a
+    // notification that includes every line that was part of the fold
+    int64_t num_changed = last_lnum - first_lnum;
+    buf_updates_send_changes(curbuf, first_lnum, num_changed,
+                             num_changed, true);
   }
 }
 
@@ -1584,13 +1582,11 @@ static void foldCreateMarkers(linenr_T start, linenr_T end)
    * changed when the start marker is inserted and the end isn't. */
   changed_lines(start, (colnr_T)0, end, 0L, false);
 
-  if (kv_size(curbuf->update_channels)) {
-    // Note: foldAddMarker() may not actually change start and/or end if
-    // u_save() is unable to save the buffer line, but we send the
-    // nvim_buf_lines_event anyway since it won't do any harm.
-    int64_t num_changed = 1 + end - start;
-    buf_updates_send_changes(curbuf, start, num_changed, num_changed, true);
-  }
+  // Note: foldAddMarker() may not actually change start and/or end if
+  // u_save() is unable to save the buffer line, but we send the
+  // nvim_buf_lines_event anyway since it won't do any harm.
+  int64_t num_changed = 1 + end - start;
+  buf_updates_send_changes(curbuf, start, num_changed, num_changed, true);
 }
 
 /* foldAddMarker() {{{2 */
diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
index 3703b76973..f52d05a4a5 100644
--- a/src/nvim/generators/gen_api_dispatch.lua
+++ b/src/nvim/generators/gen_api_dispatch.lua
@@ -135,7 +135,7 @@ for i,f in ipairs(shallowcopy(functions)) do
 end
 
 -- don't expose internal attributes like "impl_name" in public metadata
-exported_attributes = {'name', 'parameters', 'return_type', 'method',
+exported_attributes = {'name', 'return_type', 'method',
                        'since', 'deprecated_since'}
 exported_functions = {}
 for _,f in ipairs(functions) do
@@ -144,6 +144,13 @@ for _,f in ipairs(functions) do
     for _,attr in ipairs(exported_attributes) do
       f_exported[attr] = f[attr]
     end
+    f_exported.parameters = {}
+    for i,param in ipairs(f.parameters) do
+      if param[1] == "DictionaryOf(LuaRef)" then
+        param = {"Dictionary", param[2]}
+      end
+      f_exported.parameters[i] = param
+    end
     exported_functions[#exported_functions+1] = f_exported
   end
 end
@@ -371,14 +378,18 @@ local function process_function(fn)
     param = fn.parameters[j]
     cparam = string.format('arg%u', j)
     param_type = real_type(param[1])
-    lc_param_type = param_type:lower()
+    lc_param_type = real_type(param[1]):lower()
+    extra = ((param_type == "Object" or param_type == "Dictionary") and "false, ") or ""
+    if param[1] == "DictionaryOf(LuaRef)" then
+      extra = "true, "
+    end
     write_shifted_output(output, string.format([[
-    const %s %s = nlua_pop_%s(lstate, &err);
+    const %s %s = nlua_pop_%s(lstate, %s&err);
 
     if (ERROR_SET(&err)) {
       goto exit_%u;
     }
-    ]], param[1], cparam, param_type, #fn.parameters - j))
+    ]], param[1], cparam, param_type, extra, #fn.parameters - j))
     free_code[#free_code + 1] = ('api_free_%s(%s);'):format(
       lc_param_type, cparam)
     cparams = cparam .. ', ' .. cparams
diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c
index 5da6d2c0a0..3729e42b99 100644
--- a/src/nvim/lua/converter.c
+++ b/src/nvim/lua/converter.c
@@ -706,6 +706,10 @@ void nlua_push_Object(lua_State *lstate, const Object obj)
       lua_pushnil(lstate);
       break;
     }
+    case kObjectTypeLuaRef: {
+      nlua_pushref(lstate, obj.data.luaref);
+      break;
+    }
 #define ADD_TYPE(type, data_key) \
     case kObjectType##type: { \
       nlua_push_##type(lstate, obj.data.data_key); \
@@ -862,7 +866,7 @@ static Array nlua_pop_Array_unchecked(lua_State *const lstate,
 
     lua_rawgeti(lstate, -1, (int)i);
 
-    val = nlua_pop_Object(lstate, err);
+    val = nlua_pop_Object(lstate, false, err);
     if (ERROR_SET(err)) {
       ret.size = i - 1;
       lua_pop(lstate, 1);
@@ -900,6 +904,7 @@ Array nlua_pop_Array(lua_State *lstate, Error *err)
 /// @param[out]  err  Location where error will be saved.
 static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate,
                                                 const LuaTableProps table_props,
+                                                bool ref,
                                                 Error *err)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
@@ -923,7 +928,7 @@ static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate,
       // stack: dict, key, value
 
       if (!ERROR_SET(err)) {
-        ret.items[i].value = nlua_pop_Object(lstate, err);
+        ret.items[i].value = nlua_pop_Object(lstate, ref, err);
         // stack: dict, key
       } else {
         lua_pop(lstate, 1);
@@ -951,7 +956,7 @@ static Dictionary nlua_pop_Dictionary_unchecked(lua_State *lstate,
 /// Convert lua table to dictionary
 ///
 /// Always pops one value from the stack.
-Dictionary nlua_pop_Dictionary(lua_State *lstate, Error *err)
+Dictionary nlua_pop_Dictionary(lua_State *lstate, bool ref, Error *err)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
 {
   const LuaTableProps table_props = nlua_check_type(lstate, err,
@@ -961,7 +966,7 @@ Dictionary nlua_pop_Dictionary(lua_State *lstate, Error *err)
     return (Dictionary) { .size = 0, .items = NULL };
   }
 
-  return nlua_pop_Dictionary_unchecked(lstate, table_props, err);
+  return nlua_pop_Dictionary_unchecked(lstate, table_props, ref, err);
 }
 
 /// Helper structure for nlua_pop_Object
@@ -973,7 +978,7 @@ typedef struct {
 /// Convert lua table to object
 ///
 /// Always pops one value from the stack.
-Object nlua_pop_Object(lua_State *const lstate, Error *const err)
+Object nlua_pop_Object(lua_State *const lstate, bool ref, Error *const err)
 {
   Object ret = NIL;
   const int initial_size = lua_gettop(lstate);
@@ -1122,7 +1127,18 @@ Object nlua_pop_Object(lua_State *const lstate, Error *const err)
         }
         break;
       }
+
+      case LUA_TFUNCTION: {
+        if (ref) {
+          *cur.obj = LUAREF_OBJ(nlua_ref(lstate, -1));
+        } else {
+          goto type_error;
+        }
+        break;
+      }
+
       default: {
+type_error:
         api_set_error(err, kErrorTypeValidation,
                       "Cannot convert given lua type");
         break;
diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c
index 4e94c10283..fa8a67ff39 100644
--- a/src/nvim/lua/executor.c
+++ b/src/nvim/lua/executor.c
@@ -363,6 +363,33 @@ static int nlua_getenv(lua_State *lstate)
 }
 #endif
 
+/// add the value to the registry
+LuaRef nlua_ref(lua_State *lstate, int index)
+{
+  lua_pushvalue(lstate, index);
+  return luaL_ref(lstate, LUA_REGISTRYINDEX);
+}
+
+/// remove the value from the registry
+void nlua_unref(lua_State *lstate, LuaRef ref)
+{
+  if (ref > 0) {
+    luaL_unref(lstate, LUA_REGISTRYINDEX, ref);
+  }
+}
+
+void executor_free_luaref(LuaRef ref)
+{
+  lua_State *const lstate = nlua_enter();
+  nlua_unref(lstate, ref);
+}
+
+/// push a value referenced in the regirstry
+void nlua_pushref(lua_State *lstate, LuaRef ref)
+{
+  lua_rawgeti(lstate, LUA_REGISTRYINDEX, ref);
+}
+
 /// Evaluate lua string
 ///
 /// Used for luaeval().
@@ -451,9 +478,29 @@ Object executor_exec_lua_api(const String str, const Array args, Error *err)
     return NIL;
   }
 
-  return nlua_pop_Object(lstate, err);
+  return nlua_pop_Object(lstate, false, err);
 }
 
+Object executor_exec_lua_cb(LuaRef ref, const char *name, Array args)
+{
+  lua_State *const lstate = nlua_enter();
+  nlua_pushref(lstate, ref);
+  lua_pushstring(lstate, name);
+  for (size_t i = 0; i < args.size; i++) {
+    nlua_push_Object(lstate, args.items[i]);
+  }
+
+  if (lua_pcall(lstate, (int)args.size+1, 1, 0)) {
+    // TODO(bfredl): callbacks:s might not always be msg-safe, for instance
+    // lua callbacks for redraw events. Later on let the caller deal with the
+    // error instead.
+    nlua_error(lstate, _("Error executing lua callback: %.*s"));
+    return NIL;
+  }
+  Error err = ERROR_INIT;
+
+  return nlua_pop_Object(lstate, false, &err);
+}
 
 /// Run lua string
 ///
diff --git a/src/nvim/lua/executor.h b/src/nvim/lua/executor.h
index 0cbf290f64..8d356a5600 100644
--- a/src/nvim/lua/executor.h
+++ b/src/nvim/lua/executor.h
@@ -2,6 +2,7 @@
 #define NVIM_LUA_EXECUTOR_H
 
 #include <lua.h>
+#include <lauxlib.h>
 
 #include "nvim/api/private/defs.h"
 #include "nvim/func_attr.h"
diff --git a/src/nvim/misc1.c b/src/nvim/misc1.c
index 4ef0103c4f..45e03681eb 100644
--- a/src/nvim/misc1.c
+++ b/src/nvim/misc1.c
@@ -1847,9 +1847,7 @@ void changed_bytes(linenr_T lnum, colnr_T col)
   changedOneline(curbuf, lnum);
   changed_common(lnum, col, lnum + 1, 0L);
   // notify any channels that are watching
-  if (kv_size(curbuf->update_channels)) {
-    buf_updates_send_changes(curbuf, lnum, 1, 1, true);
-  }
+  buf_updates_send_changes(curbuf, lnum, 1, 1, true);
 
   /* Diff highlighting in other diff windows may need to be updated too. */
   if (curwin->w_p_diff) {
@@ -1973,7 +1971,7 @@ changed_lines(
 
   changed_common(lnum, col, lnume, xtra);
 
-  if (do_buf_event && kv_size(curbuf->update_channels)) {
+  if (do_buf_event) {
     int64_t num_added = (int64_t)(lnume + xtra - lnum);
     int64_t num_removed = lnume - lnum;
     buf_updates_send_changes(curbuf, lnum, num_added, num_removed, true);
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index 3925dc546a..3f768dcc0c 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -253,7 +253,8 @@ bool msgpack_rpc_to_object(const msgpack_object *const obj, Object *const arg)
           case kObjectTypeFloat:
           case kObjectTypeString:
           case kObjectTypeArray:
-          case kObjectTypeDictionary: {
+          case kObjectTypeDictionary:
+          case kObjectTypeLuaRef: {
             break;
           }
         }
@@ -387,6 +388,13 @@ void msgpack_rpc_from_object(const Object result, msgpack_packer *const res)
         msgpack_pack_nil(res);
         break;
       }
+      case kObjectTypeLuaRef: {
+        // TODO(bfredl): could also be an error. Though kObjectTypeLuaRef
+        // should only appear when the caller has opted in to handle references,
+        // see nlua_pop_Object.
+        msgpack_pack_nil(res);
+        break;
+      }
       case kObjectTypeBoolean: {
         msgpack_rpc_from_boolean(cur.aobj->data.boolean, res);
         break;
diff --git a/src/nvim/types.h b/src/nvim/types.h
index f803b45e27..5bcc0c3e1b 100644
--- a/src/nvim/types.h
+++ b/src/nvim/types.h
@@ -16,6 +16,11 @@ typedef uint32_t u8char_T;
 // Opaque handle used by API clients to refer to various objects in vim
 typedef int handle_T;
 
+// Opaque handle to a lua value. Must be free with `executor_free_luaref` when
+// not needed anymore! LUA_NOREF represents missing reference, i e to indicate
+// absent callback etc.
+typedef int LuaRef;
+
 typedef struct expand expand_T;
 
 #endif  // NVIM_TYPES_H
diff --git a/src/nvim/undo.c b/src/nvim/undo.c
index 8c90c4bf30..116bdd02b8 100644
--- a/src/nvim/undo.c
+++ b/src/nvim/undo.c
@@ -2299,7 +2299,7 @@ static void u_undoredo(int undo, bool do_buf_event)
   // because the calls to changed()/unchanged() above will bump changedtick
   // again, we need to send a nvim_buf_lines_event with just the new value of
   // b:changedtick
-  if (do_buf_event && kv_size(curbuf->update_channels)) {
+  if (do_buf_event) {
     buf_updates_changedtick(curbuf);
   }
 
diff --git a/test/functional/api/buffer_updates_spec.lua b/test/functional/api/buffer_updates_spec.lua
index b894d2facd..89d0c6f000 100644
--- a/test/functional/api/buffer_updates_spec.lua
+++ b/test/functional/api/buffer_updates_spec.lua
@@ -760,7 +760,7 @@ describe('API: buffer events:', function()
   it('returns a proper error on nonempty options dict', function()
     clear()
     local b = editoriginal(false)
-    expect_err("dict isn't empty", buffer, 'attach', b, false, {builtin="asfd"})
+    expect_err("unexpected key: builtin", buffer, 'attach', b, false, {builtin="asfd"})
   end)
 
   it('nvim_buf_attach returns response after delay #8634', function()