Merge pull request #12851 from bfredl/luahl

luahl: still WIP but better

9 files changed, 237 insertions, 141 deletions

diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c
index 15065760b3..21f7c8e931 100644
--- a/src/nvim/api/buffer.c
+++ b/src/nvim/api/buffer.c
@@ -244,78 +244,6 @@ Boolean nvim_buf_detach(uint64_t channel_id,
   return true;
 }
 
-static void buf_clear_luahl(buf_T *buf, bool force)
-{
-  if (buf->b_luahl || force) {
-    api_free_luaref(buf->b_luahl_start);
-    api_free_luaref(buf->b_luahl_window);
-    api_free_luaref(buf->b_luahl_line);
-    api_free_luaref(buf->b_luahl_end);
-  }
-  buf->b_luahl_start = LUA_NOREF;
-  buf->b_luahl_window = LUA_NOREF;
-  buf->b_luahl_line = LUA_NOREF;
-  buf->b_luahl_end = LUA_NOREF;
-}
-
-/// Unstabilized interface for defining syntax hl in lua.
-///
-/// This is not yet safe for general use, lua callbacks will need to
-/// be restricted, like textlock and probably other stuff.
-///
-/// The API on_line/nvim__put_attr is quite raw and not intended to be the
-/// final shape. Ideally this should operate on chunks larger than a single
-/// line to reduce interpreter overhead, and generate annotation objects
-/// (bufhl/virttext) on the fly but using the same representation.
-void nvim__buf_set_luahl(uint64_t channel_id, Buffer buffer,
-                         DictionaryOf(LuaRef) opts, Error *err)
-  FUNC_API_LUA_ONLY
-{
-  buf_T *buf = find_buffer_by_handle(buffer, err);
-
-  if (!buf) {
-    return;
-  }
-
-  redraw_buf_later(buf, NOT_VALID);
-  buf_clear_luahl(buf, false);
-
-  for (size_t i = 0; i < opts.size; i++) {
-    String k = opts.items[i].key;
-    Object *v = &opts.items[i].value;
-    if (strequal("on_start", k.data)) {
-      if (v->type != kObjectTypeLuaRef) {
-        api_set_error(err, kErrorTypeValidation, "callback is not a function");
-        goto error;
-      }
-      buf->b_luahl_start = v->data.luaref;
-      v->data.luaref = LUA_NOREF;
-    } else if (strequal("on_window", k.data)) {
-      if (v->type != kObjectTypeLuaRef) {
-        api_set_error(err, kErrorTypeValidation, "callback is not a function");
-        goto error;
-      }
-      buf->b_luahl_window = v->data.luaref;
-      v->data.luaref = LUA_NOREF;
-    } else if (strequal("on_line", k.data)) {
-      if (v->type != kObjectTypeLuaRef) {
-        api_set_error(err, kErrorTypeValidation, "callback is not a function");
-        goto error;
-      }
-      buf->b_luahl_line = v->data.luaref;
-      v->data.luaref = LUA_NOREF;
-    } else {
-      api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
-      goto error;
-    }
-  }
-  buf->b_luahl = true;
-  return;
-error:
-  buf_clear_luahl(buf, true);
-  buf->b_luahl = false;
-}
-
 void nvim__buf_redraw_range(Buffer buffer, Integer first, Integer last,
                             Error *err)
   FUNC_API_LUA_ONLY
@@ -1664,43 +1592,6 @@ void nvim_buf_clear_highlight(Buffer buffer,
   nvim_buf_clear_namespace(buffer, ns_id, line_start, line_end, err);
 }
 
-static VirtText parse_virt_text(Array chunks, Error *err)
-{
-  VirtText virt_text = KV_INITIAL_VALUE;
-  for (size_t i = 0; i < chunks.size; i++) {
-    if (chunks.items[i].type != kObjectTypeArray) {
-      api_set_error(err, kErrorTypeValidation, "Chunk is not an array");
-      goto free_exit;
-    }
-    Array chunk = chunks.items[i].data.array;
-    if (chunk.size == 0 || chunk.size > 2
-        || chunk.items[0].type != kObjectTypeString
-        || (chunk.size == 2 && chunk.items[1].type != kObjectTypeString)) {
-      api_set_error(err, kErrorTypeValidation,
-                    "Chunk is not an array with one or two strings");
-      goto free_exit;
-    }
-
-    String str = chunk.items[0].data.string;
-    char *text = transstr(str.size > 0 ? str.data : "");  // allocates
-
-    int hl_id = 0;
-    if (chunk.size == 2) {
-      String hl = chunk.items[1].data.string;
-      if (hl.size > 0) {
-        hl_id = syn_check_group((char_u *)hl.data, (int)hl.size);
-      }
-    }
-    kv_push(virt_text, ((VirtTextChunk){ .text = text, .hl_id = hl_id }));
-  }
-
-  return virt_text;
-
-free_exit:
-  clear_virttext(&virt_text);
-  return virt_text;
-}
-
 
 /// Set the virtual text (annotation) for a buffer line.
 ///
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index 13f77d2d85..e0d5862e02 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -15,6 +15,8 @@
 #include "nvim/lua/executor.h"
 #include "nvim/ascii.h"
 #include "nvim/assert.h"
+#include "nvim/charset.h"
+#include "nvim/syntax.h"
 #include "nvim/vim.h"
 #include "nvim/buffer.h"
 #include "nvim/window.h"
@@ -1579,3 +1581,52 @@ bool extmark_get_index_from_obj(buf_T *buf, Integer ns_id, Object obj, int
     return false;
   }
 }
+
+VirtText parse_virt_text(Array chunks, Error *err)
+{
+  VirtText virt_text = KV_INITIAL_VALUE;
+  for (size_t i = 0; i < chunks.size; i++) {
+    if (chunks.items[i].type != kObjectTypeArray) {
+      api_set_error(err, kErrorTypeValidation, "Chunk is not an array");
+      goto free_exit;
+    }
+    Array chunk = chunks.items[i].data.array;
+    if (chunk.size == 0 || chunk.size > 2
+        || chunk.items[0].type != kObjectTypeString
+        || (chunk.size == 2 && chunk.items[1].type != kObjectTypeString)) {
+      api_set_error(err, kErrorTypeValidation,
+                    "Chunk is not an array with one or two strings");
+      goto free_exit;
+    }
+
+    String str = chunk.items[0].data.string;
+    char *text = transstr(str.size > 0 ? str.data : "");  // allocates
+
+    int hl_id = 0;
+    if (chunk.size == 2) {
+      String hl = chunk.items[1].data.string;
+      if (hl.size > 0) {
+        hl_id = syn_check_group((char_u *)hl.data, (int)hl.size);
+      }
+    }
+    kv_push(virt_text, ((VirtTextChunk){ .text = text, .hl_id = hl_id }));
+  }
+
+  return virt_text;
+
+free_exit:
+  clear_virttext(&virt_text);
+  return virt_text;
+}
+
+bool api_is_truthy(Object obj, const char *what, Error *err)
+{
+  if (obj.type == kObjectTypeBoolean) {
+    return obj.data.boolean;
+  } else if (obj.type == kObjectTypeInteger) {
+    return obj.data.integer;  // C semantics: non-zery int is true
+  } else {
+    api_set_error(err, kErrorTypeValidation, "%s is not an boolean", what);
+    return false;
+  }
+}
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 9155ffcfb8..1de1472fc2 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -2610,22 +2610,91 @@ Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Error *err)
 /// interface should probably be derived from a reformed
 /// bufhl/virttext interface with full support for multi-line
 /// ranges etc
-void nvim__put_attr(Integer id, Integer start_row, Integer start_col,
-                    Integer end_row, Integer end_col)
+void nvim__put_attr(Integer line, Integer col, Dictionary opts, Error *err)
   FUNC_API_LUA_ONLY
 {
   if (!lua_attr_active) {
     return;
   }
-  if (id == 0 || syn_get_final_id((int)id) == 0) {
-    return;
+  int line2 = -1, hl_id = 0;
+  colnr_T col2 = 0;
+  VirtText virt_text = KV_INITIAL_VALUE;
+  for (size_t i = 0; i < opts.size; i++) {
+    String k = opts.items[i].key;
+    Object *v = &opts.items[i].value;
+    if (strequal("end_line", k.data)) {
+      if (v->type != kObjectTypeInteger) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_line is not an integer");
+        goto error;
+      }
+      if (v->data.integer < 0) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_line value outside range");
+        goto error;
+      }
+
+      line2 = (int)v->data.integer;
+    } else if (strequal("end_col", k.data)) {
+      if (v->type != kObjectTypeInteger) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_col is not an integer");
+        goto error;
+      }
+      if (v->data.integer < 0 || v->data.integer > MAXCOL) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_col value outside range");
+        goto error;
+      }
+
+      col2 = (colnr_T)v->data.integer;
+    } else if (strequal("hl_group", k.data)) {
+      String hl_group;
+      switch (v->type) {
+        case kObjectTypeString:
+          hl_group = v->data.string;
+          hl_id = syn_check_group(
+              (char_u *)(hl_group.data),
+              (int)hl_group.size);
+          break;
+        case kObjectTypeInteger:
+          hl_id = (int)v->data.integer;
+          break;
+        default:
+          api_set_error(err, kErrorTypeValidation,
+                        "hl_group is not valid.");
+          goto error;
+      }
+    } else if (strequal("virt_text", k.data)) {
+      if (v->type != kObjectTypeArray) {
+        api_set_error(err, kErrorTypeValidation,
+                      "virt_text is not an Array");
+        goto error;
+      }
+      virt_text = parse_virt_text(v->data.array, err);
+      if (ERROR_SET(err)) {
+        goto error;
+      }
+    } else {
+      api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
+      goto error;
+    }
+  }
+  if (col2 && line2 < 0) {
+    line2 = (int)line;
   }
-  int attr = syn_id2attr((int)id);
-  if (attr == 0) {
+
+  int attr = hl_id ? syn_id2attr((int)hl_id) : 0;
+  if (attr == 0 && !kv_size(virt_text)) {
     return;
   }
-  decorations_add_luahl_attr(attr, (int)start_row, (colnr_T)start_col,
-                             (int)end_row, (colnr_T)end_col);
+
+  VirtText *v = xmalloc(sizeof(*v));
+  *v = virt_text;  // LeakSanitizer be sad
+  decorations_add_luahl_attr(attr, (int)line, (colnr_T)col,
+                             (int)line2, (colnr_T)col2, v);
+error:
+  return;
 }
 
 void nvim__screenshot(String path)
@@ -2633,3 +2702,68 @@ void nvim__screenshot(String path)
 {
   ui_call_screenshot(path);
 }
+
+static void clear_luahl(bool force)
+{
+  if (luahl_active || force) {
+    api_free_luaref(luahl_start);
+    api_free_luaref(luahl_win);
+    api_free_luaref(luahl_line);
+    api_free_luaref(luahl_end);
+  }
+  luahl_start = LUA_NOREF;
+  luahl_win = LUA_NOREF;
+  luahl_line = LUA_NOREF;
+  luahl_end = LUA_NOREF;
+  luahl_active = false;
+}
+
+/// Unstabilized interface for defining syntax hl in lua.
+///
+/// This is not yet safe for general use, lua callbacks will need to
+/// be restricted, like textlock and probably other stuff.
+///
+/// The API on_line/nvim__put_attr is quite raw and not intended to be the
+/// final shape. Ideally this should operate on chunks larger than a single
+/// line to reduce interpreter overhead, and generate annotation objects
+/// (bufhl/virttext) on the fly but using the same representation.
+void nvim__set_luahl(DictionaryOf(LuaRef) opts, Error *err)
+  FUNC_API_LUA_ONLY
+{
+  redraw_later(NOT_VALID);
+  clear_luahl(false);
+
+  for (size_t i = 0; i < opts.size; i++) {
+    String k = opts.items[i].key;
+    Object *v = &opts.items[i].value;
+    if (strequal("on_start", k.data)) {
+      if (v->type != kObjectTypeLuaRef) {
+        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+        goto error;
+      }
+      luahl_start = v->data.luaref;
+      v->data.luaref = LUA_NOREF;
+    } else if (strequal("on_win", k.data)) {
+      if (v->type != kObjectTypeLuaRef) {
+        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+        goto error;
+      }
+      luahl_win = v->data.luaref;
+      v->data.luaref = LUA_NOREF;
+    } else if (strequal("on_line", k.data)) {
+      if (v->type != kObjectTypeLuaRef) {
+        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+        goto error;
+      }
+      luahl_line = v->data.luaref;
+      v->data.luaref = LUA_NOREF;
+    } else {
+      api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
+      goto error;
+    }
+  }
+  luahl_active = true;
+  return;
+error:
+  clear_luahl(true);
+}
diff --git a/src/nvim/buffer_defs.h b/src/nvim/buffer_defs.h
index b3c95f9362..bd9cd2f5ec 100644
--- a/src/nvim/buffer_defs.h
+++ b/src/nvim/buffer_defs.h
@@ -842,12 +842,6 @@ struct file_buffer {
   // The number for times the current line has been flushed in the memline.
   int flush_count;
 
-  bool b_luahl;
-  LuaRef b_luahl_start;
-  LuaRef b_luahl_window;
-  LuaRef b_luahl_line;
-  LuaRef b_luahl_end;
-
   int b_diff_failed;    // internal diff failed for this buffer
 };
 
diff --git a/src/nvim/extmark.c b/src/nvim/extmark.c
index 3a04908ccb..ba94f55a50 100644
--- a/src/nvim/extmark.c
+++ b/src/nvim/extmark.c
@@ -844,8 +844,15 @@ VirtText *extmark_find_virttext(buf_T *buf, int row, uint64_t ns_id)
 bool decorations_redraw_reset(buf_T *buf, DecorationRedrawState *state)
 {
   state->row = -1;
+  for (size_t i = 0; i < kv_size(state->active); i++) {
+    HlRange item = kv_A(state->active, i);
+    if (item.virt_text_owned) {
+      clear_virttext(item.virt_text);
+      xfree(item.virt_text);
+    }
+  }
   kv_size(state->active) = 0;
-  return buf->b_extmark_index || buf->b_luahl;
+  return buf->b_extmark_index;
 }
 
 
@@ -889,10 +896,10 @@ bool decorations_redraw_start(buf_T *buf, int top_row,
     HlRange range;
     if (mark.id&MARKTREE_END_FLAG) {
       range = (HlRange){ altpos.row, altpos.col, mark.row, mark.col,
-                         attr_id, vt };
+                         attr_id, vt, false };
     } else {
       range = (HlRange){ mark.row, mark.col, altpos.row,
-                         altpos.col, attr_id, vt };
+                         altpos.col, attr_id, vt, false };
     }
     kv_push(state->active, range);
 
@@ -957,7 +964,7 @@ int decorations_redraw_col(buf_T *buf, int col, DecorationRedrawState *state)
     VirtText *vt = kv_size(decor->virt_text) ? &decor->virt_text : NULL;
     kv_push(state->active, ((HlRange){ mark.row, mark.col,
                                        endpos.row, endpos.col,
-                                       attr_id, vt }));
+                                       attr_id, vt, false }));
 
 next_mark:
     marktree_itr_next(buf->b_marktree, state->itr);
@@ -991,6 +998,9 @@ next_mark:
     }
     if (keep) {
       kv_A(state->active, j++) = kv_A(state->active, i);
+    } else if (item.virt_text_owned) {
+      clear_virttext(item.virt_text);
+      xfree(item.virt_text);
     }
   }
   kv_size(state->active) = j;
diff --git a/src/nvim/extmark.h b/src/nvim/extmark.h
index 534e97a7f4..d394d4d806 100644
--- a/src/nvim/extmark.h
+++ b/src/nvim/extmark.h
@@ -85,6 +85,7 @@ typedef struct {
   int end_col;
   int attr_id;
   VirtText *virt_text;
+  bool virt_text_owned;
 } HlRange;
 
 typedef struct {
diff --git a/src/nvim/globals.h b/src/nvim/globals.h
index 205be4b811..ddb69fc567 100644
--- a/src/nvim/globals.h
+++ b/src/nvim/globals.h
@@ -405,6 +405,12 @@ EXTERN int sys_menu INIT(= false);
 // ('lines' and 'rows') must not be changed.
 EXTERN int updating_screen INIT(= 0);
 
+EXTERN bool luahl_active INIT(= false);
+EXTERN LuaRef luahl_start INIT(= LUA_NOREF);
+EXTERN LuaRef luahl_win INIT(= LUA_NOREF);
+EXTERN LuaRef luahl_line INIT(= LUA_NOREF);
+EXTERN LuaRef luahl_end INIT(= LUA_NOREF);
+
 // All windows are linked in a list. firstwin points to the first entry,
 // lastwin to the last entry (can be the same as firstwin) and curwin to the
 // currently active window.
diff --git a/src/nvim/main.c b/src/nvim/main.c
index 1374c5eb5d..a22df9cc69 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -7,6 +7,8 @@
 #include <string.h>
 #include <stdbool.h>
 
+#include <lua.h>
+#include <lauxlib.h>
 #include <msgpack.h>
 
 #include "nvim/ascii.h"
diff --git a/src/nvim/screen.c b/src/nvim/screen.c
index 3c2e1ccaf5..5ac5306dd9 100644
--- a/src/nvim/screen.c
+++ b/src/nvim/screen.c
@@ -158,6 +158,8 @@ static bool msg_grid_invalid = false;
 
 static bool resizing = false;
 
+static bool do_luahl_line = false;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "screen.c.generated.h"
 #endif
@@ -508,12 +510,12 @@ int update_screen(int type)
       }
 
       buf_T *buf = wp->w_buffer;
-      if (buf->b_luahl && buf->b_luahl_window != LUA_NOREF) {
+      if (luahl_active && luahl_start != LUA_NOREF) {
         Error err = ERROR_INIT;
         FIXED_TEMP_ARRAY(args, 2);
         args.items[0] = BUFFER_OBJ(buf->handle);
         args.items[1] = INTEGER_OBJ(display_tick);
-        nlua_call_ref(buf->b_luahl_start, "start", args, false, &err);
+        nlua_call_ref(luahl_start, "start", args, false, &err);
         if (ERROR_SET(&err)) {
           ELOG("error in luahl start: %s", err.msg);
           api_clear_error(&err);
@@ -639,10 +641,11 @@ bool decorations_active = false;
 
 void decorations_add_luahl_attr(int attr_id,
                                 int start_row, int start_col,
-                                int end_row, int end_col)
+                                int end_row, int end_col, VirtText *virt_text)
 {
   kv_push(decorations.active,
-          ((HlRange){ start_row, start_col, end_row, end_col, attr_id, NULL }));
+          ((HlRange){ start_row, start_col,
+                      end_row, end_col, attr_id, virt_text, true }));
 }
 
 /*
@@ -1238,7 +1241,9 @@ static void win_update(win_T *wp)
 
   decorations_active = decorations_redraw_reset(buf, &decorations);
 
-  if (buf->b_luahl && buf->b_luahl_window != LUA_NOREF) {
+  do_luahl_line = false;
+
+  if (luahl_win != LUA_NOREF) {
     Error err = ERROR_INIT;
     FIXED_TEMP_ARRAY(args, 4);
     linenr_T knownmax = ((wp->w_valid & VALID_BOTLINE)
@@ -1251,7 +1256,13 @@ static void win_update(win_T *wp)
     args.items[3] = INTEGER_OBJ(knownmax);
     // TODO(bfredl): we could allow this callback to change mod_top, mod_bot.
     // For now the "start" callback is expected to use nvim__buf_redraw_range.
-    nlua_call_ref(buf->b_luahl_window, "window", args, false, &err);
+    Object ret = nlua_call_ref(luahl_win, "win", args, true, &err);
+
+    if (!ERROR_SET(&err) && api_is_truthy(ret, "luahl_window retval", &err)) {
+      do_luahl_line = true;
+      decorations_active = true;
+    }
+
     if (ERROR_SET(&err)) {
       ELOG("error in luahl window: %s", err.msg);
       api_clear_error(&err);
@@ -2348,7 +2359,7 @@ win_line (
     }
 
     if (decorations_active) {
-      if (buf->b_luahl && buf->b_luahl_line != LUA_NOREF) {
+      if (do_luahl_line && luahl_line != LUA_NOREF) {
         Error err = ERROR_INIT;
         FIXED_TEMP_ARRAY(args, 3);
         args.items[0] = WINDOW_OBJ(wp->handle);
@@ -2356,14 +2367,10 @@ win_line (
         args.items[2] = INTEGER_OBJ(lnum-1);
         lua_attr_active = true;
         extra_check = true;
-        Object o = nlua_call_ref(buf->b_luahl_line, "line", args, true, &err);
+        nlua_call_ref(luahl_line, "line", args, false, &err);
         lua_attr_active = false;
-        if (o.type == kObjectTypeString) {
-          // TODO(bfredl): this is a bit of a hack. A final API should use an
-          // "unified" interface where luahl can add both bufhl and virttext
-          luatext = o.data.string.data;
-          do_virttext = true;
-        } else if (ERROR_SET(&err)) {
+
+        if (ERROR_SET(&err)) {
           ELOG("error in luahl line: %s", err.msg);
           luatext = err.msg;
           do_virttext = true;