extmarks/bufhl: reimplement using new marktree data structure

Add new "splice" interface for tracking buffer changes at the byte
level. This will later be reused for byte-resolution buffer updates.
(Implementation has been started, but using undocumented "_on_bytes"
option now as interface hasn't been finalized).

Use this interface to improve many edge cases of extmark adjustment.
Changed tests indicate previously incorrect behavior. Adding tests for
more edge cases will be follow-up work (overlaps on_bytes tests)

Don't consider creation/deletion of marks an undoable event by itself.
This behavior was never documented, and imposes  complexity for little gain.

Add nvim__buf_add_decoration temporary API for direct access to the new
implementation. This should be refactored into a proper API for
decorations, probably involving a huge dict.

fixes #11598

1 files changed, 14 insertions, 69 deletions

diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index 36331eee6e..37e31e0807 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -1511,61 +1511,6 @@ ArrayOf(Dictionary) keymap_array(String mode, buf_T *buf)
   return mappings;
 }
 
-// Returns an extmark given an id or a positional index
-// If throw == true then an error will be raised if nothing
-// was found
-// Returns NULL if something went wrong
-Extmark *extmark_from_id_or_pos(Buffer buffer, Integer ns, Object id,
-                                Error *err, bool throw)
-{
-  buf_T *buf = find_buffer_by_handle(buffer, err);
-
-  if (!buf) {
-    return NULL;
-  }
-
-  Extmark *extmark = NULL;
-  if (id.type == kObjectTypeArray) {
-    if (id.data.array.size != 2) {
-      api_set_error(err, kErrorTypeValidation,
-                    _("Position must have 2 elements"));
-      return NULL;
-    }
-    linenr_T row = (linenr_T)id.data.array.items[0].data.integer;
-    colnr_T col = (colnr_T)id.data.array.items[1].data.integer;
-    if (row < 1 || col < 1) {
-      if (throw) {
-      api_set_error(err, kErrorTypeValidation, _("Row and column MUST be > 0"));
-      }
-      return NULL;
-    }
-    extmark = extmark_from_pos(buf, (uint64_t)ns, row, col);
-  } else if (id.type != kObjectTypeInteger) {
-    if (throw) {
-      api_set_error(err, kErrorTypeValidation,
-                    _("Mark id must be an int or [row, col]"));
-    }
-    return NULL;
-  } else if (id.data.integer < 0) {
-    if (throw) {
-      api_set_error(err, kErrorTypeValidation, _("Mark id must be positive"));
-    }
-    return NULL;
-  } else {
-    extmark = extmark_from_id(buf,
-                              (uint64_t)ns,
-                              (uint64_t)id.data.integer);
-  }
-
-  if (!extmark) {
-    if (throw) {
-      api_set_error(err, kErrorTypeValidation, _("Mark doesn't exist"));
-    }
-    return NULL;
-  }
-  return extmark;
-}
-
 // Is the Namespace in use?
 bool ns_initialized(uint64_t ns)
 {
@@ -1584,29 +1529,29 @@ bool ns_initialized(uint64_t ns)
 /// @param[out] colnr extmark column
 ///
 /// @return true if the extmark was found, else false
-bool extmark_get_index_from_obj(buf_T *buf, Integer ns, Object obj, linenr_T
-                                *lnum, colnr_T *colnr, Error *err)
+bool extmark_get_index_from_obj(buf_T *buf, Integer ns_id, Object obj, int
+                                *row, colnr_T *col, Error *err)
 {
   // Check if it is mark id
   if (obj.type == kObjectTypeInteger) {
     Integer id = obj.data.integer;
     if (id == 0) {
-        *lnum = 1;
-        *colnr = 1;
+        *row = 0;
+        *col = 0;
         return true;
     } else if (id == -1) {
-        *lnum = MAXLNUM;
-        *colnr = MAXCOL;
+        *row = MAXLNUM;
+        *col = MAXCOL;
         return true;
     } else if (id < 0) {
       api_set_error(err, kErrorTypeValidation, _("Mark id must be positive"));
       return false;
     }
 
-    Extmark *extmark = extmark_from_id(buf, (uint64_t)ns, (uint64_t)id);
-    if (extmark) {
-      *lnum = extmark->line->lnum;
-      *colnr = extmark->col;
+    ExtmarkInfo extmark = extmark_from_id(buf, (uint64_t)ns_id, (uint64_t)id);
+    if (extmark.row >= 0) {
+      *row = extmark.row;
+      *col = extmark.col;
       return true;
     } else {
       api_set_error(err, kErrorTypeValidation, _("No mark with requested id"));
@@ -1623,10 +1568,10 @@ bool extmark_get_index_from_obj(buf_T *buf, Integer ns, Object obj, linenr_T
                     _("Position must have 2 integer elements"));
       return false;
     }
-    Integer line = pos.items[0].data.integer;
-    Integer col = pos.items[1].data.integer;
-    *lnum = (linenr_T)(line >= 0 ? line + 1 : MAXLNUM);
-    *colnr = (colnr_T)(col >= 0 ? col + 1 : MAXCOL);
+    Integer pos_row = pos.items[0].data.integer;
+    Integer pos_col = pos.items[1].data.integer;
+    *row = (int)(pos_row >= 0 ? pos_row  : MAXLNUM);
+    *col = (colnr_T)(pos_col >= 0 ? pos_col : MAXCOL);
     return true;
   } else {
     api_set_error(err, kErrorTypeValidation,