refactor(decorations): break up Decoration struct into smaller pieces

Remove the monolithic Decoration struct. Before this change, each extmark
could either represent just a hl_id + priority value as a inline
decoration, or it would take a pointer to this monolitic 112 byte struct
which has to be allocated.

This change separates the decorations into two pieces: DecorSignHighlight
for signs, highlights and simple set-flag decorations (like spell,
ui-watched), and DecorVirtText for virtual text and lines.

The main separation here is whether they are expected to allocate more
memory. Currently this is not really true as sign text has to be an
allocated string, but the plan is to get rid of this eventually (it can
just be an array of two schar_T:s). Further refactors are expected to
improve the representation of each decoration kind individually. The
goal of this particular PR is to get things started by cutting the
Gordian knot which was the monolithic struct Decoration.

Now, each extmark can either contain chained indicies/pointers to
these kinds of objects, or it can fit a subset of DecorSignHighlight
inline.

The point of this change is not only to make decorations smaller in
memory. In fact, the main motivation is to later allow them to grow
_larger_, but on a dynamic, on demand fashion. As a simple example, it
would be possible to augment highlights to take a list of multiple
`hl_group`:s, which then would trivially map to a chain of multiple
DecorSignHighlight entries.

One small feature improvement included with this refactor itself, is
that the restriction that extmarks cannot be removed inside a decoration
provider has been lifted. These are instead safely lifetime extended
on a "to free" list until the current iteration of screen drawing is done.

NB: flags is a mess. but DecorLevel is useless, this slightly less so

1 files changed, 131 insertions, 148 deletions

diff --git a/src/nvim/api/extmark.c b/src/nvim/api/extmark.c
index 31fc01403a..8a2cde8372 100644
--- a/src/nvim/api/extmark.c
+++ b/src/nvim/api/extmark.c
@@ -103,15 +103,6 @@ bool ns_initialized(uint32_t ns)
   return ns < (uint32_t)next_namespace_id;
 }
 
-static Object hl_group_name(int hl_id, bool hl_name)
-{
-  if (hl_name) {
-    return CSTR_TO_OBJ(syn_id2name(hl_id));
-  } else {
-    return INTEGER_OBJ(hl_id);
-  }
-}
-
 Array virt_text_to_array(VirtText vt, bool hl_name)
 {
   Array chunks = ARRAY_DICT_INIT;
@@ -176,85 +167,9 @@ static Array extmark_to_array(MTPair extmark, bool id, bool add_dict, bool hl_na
       PUT(dict, "invalid", BOOLEAN_OBJ(true));
     }
 
-    // pretend this is a pointer for a short while, Decoration will be factored away very soon
-    const Decoration decor[1] = { get_decor(start) };
-    if (decor->hl_id) {
-      PUT(dict, "hl_group", hl_group_name(decor->hl_id, hl_name));
-      PUT(dict, "hl_eol", BOOLEAN_OBJ(decor->hl_eol));
-    }
-    if (decor->hl_mode) {
-      PUT(dict, "hl_mode", CSTR_TO_OBJ(hl_mode_str[decor->hl_mode]));
-    }
-
-    if (kv_size(decor->virt_text)) {
-      Array chunks = virt_text_to_array(decor->virt_text, hl_name);
-      PUT(dict, "virt_text", ARRAY_OBJ(chunks));
-      PUT(dict, "virt_text_hide", BOOLEAN_OBJ(decor->virt_text_hide));
-      if (decor->virt_text_pos == kVTWinCol) {
-        PUT(dict, "virt_text_win_col", INTEGER_OBJ(decor->col));
-      }
-      PUT(dict, "virt_text_pos",
-          CSTR_TO_OBJ(virt_text_pos_str[decor->virt_text_pos]));
-    }
-
-    if (decor->ui_watched) {
-      PUT(dict, "ui_watched", BOOLEAN_OBJ(true));
-    }
-
-    if (kv_size(decor->virt_lines)) {
-      Array all_chunks = ARRAY_DICT_INIT;
-      bool virt_lines_leftcol = false;
-      for (size_t i = 0; i < kv_size(decor->virt_lines); i++) {
-        virt_lines_leftcol = kv_A(decor->virt_lines, i).left_col;
-        Array chunks = virt_text_to_array(kv_A(decor->virt_lines, i).line, hl_name);
-        ADD(all_chunks, ARRAY_OBJ(chunks));
-      }
-      PUT(dict, "virt_lines", ARRAY_OBJ(all_chunks));
-      PUT(dict, "virt_lines_above", BOOLEAN_OBJ(decor->virt_lines_above));
-      PUT(dict, "virt_lines_leftcol", BOOLEAN_OBJ(virt_lines_leftcol));
-    }
-
-    if (decor->sign_text) {
-      PUT(dict, "sign_text", CSTR_TO_OBJ(decor->sign_text));
-    }
-
-    // uncrustify:off
-
-    struct { char *name; const int val; } hls[] = {
-      { "sign_hl_group"      , decor->sign_hl_id       },
-      { "number_hl_group"    , decor->number_hl_id     },
-      { "line_hl_group"      , decor->line_hl_id       },
-      { "cursorline_hl_group", decor->cursorline_hl_id },
-      { NULL, 0 },
-    };
-
-    // uncrustify:on
-
-    for (int j = 0; hls[j].name; j++) {
-      if (hls[j].val) {
-        PUT(dict, hls[j].name, hl_group_name(hls[j].val, hl_name));
-      }
-    }
-
-    if (decor->sign_text
-        || decor->hl_id
-        || kv_size(decor->virt_text)
-        || decor->ui_watched) {
-      PUT(dict, "priority", INTEGER_OBJ(decor->priority));
-    }
-
-    if (decor->conceal) {
-      String name = cstr_to_string((char *)&decor->conceal_char);
-      PUT(dict, "conceal", STRING_OBJ(name));
-    }
-
-    if (decor->spell != kNone) {
-      PUT(dict, "spell", BOOLEAN_OBJ(decor->spell == kTrue));
-    }
+    decor_to_dict_legacy(&dict, mt_decor(start), hl_name);
 
-    if (dict.size) {
-      ADD(rv, DICTIONARY_OBJ(dict));
-    }
+    ADD(rv, DICTIONARY_OBJ(dict));
   }
 
   return rv;
@@ -581,8 +496,14 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
                              Dict(set_extmark) *opts, Error *err)
   FUNC_API_SINCE(7)
 {
-  Decoration decor = DECORATION_INIT;
-  bool has_decor = false;
+  DecorHighlightInline hl = DECOR_HIGHLIGHT_INLINE_INIT;
+  // TODO(bfredl): in principle signs with max one (1) hl group and max 4 bytes of text.
+  // should be a candidate for inlining as well.
+  DecorSignHighlight sign = DECOR_SIGN_HIGHLIGHT_INIT;
+  DecorVirtText virt_text = DECOR_VIRT_TEXT_INIT;
+  DecorVirtText virt_lines = DECOR_VIRT_LINES_INIT;
+  bool has_hl = false;
+  String conceal_char_large = STRING_INIT;
 
   buf_T *buf = find_buffer_by_handle(buffer, err);
   if (!buf) {
@@ -643,11 +564,11 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
     Object *opt;
     int *dest;
   } hls[] = {
-    { "hl_group"           , &opts->hl_group           , &decor.hl_id            },
-    { "sign_hl_group"      , &opts->sign_hl_group      , &decor.sign_hl_id       },
-    { "number_hl_group"    , &opts->number_hl_group    , &decor.number_hl_id     },
-    { "line_hl_group"      , &opts->line_hl_group      , &decor.line_hl_id       },
-    { "cursorline_hl_group", &opts->cursorline_hl_group, &decor.cursorline_hl_id },
+    { "hl_group"           , &opts->hl_group           , &hl.hl_id            },
+    { "sign_hl_group"      , &opts->sign_hl_group      , &sign.hl_id       },
+    { "number_hl_group"    , &opts->number_hl_group    , &sign.number_hl_id     },
+    { "line_hl_group"      , &opts->line_hl_group      , &sign.line_hl_id       },
+    { "cursorline_hl_group", &opts->cursorline_hl_group, &sign.cursorline_hl_id },
     { NULL, NULL, NULL },
   };
 
@@ -655,26 +576,33 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
 
   for (int j = 0; hls[j].name && hls[j].dest; j++) {
     if (hls[j].opt->type != kObjectTypeNil) {
+      if (j > 0) {
+        sign.flags |= kSHIsSign;
+      } else {
+        has_hl = true;
+      }
       *hls[j].dest = object_to_hl_id(*hls[j].opt, hls[j].name, err);
       if (ERROR_SET(err)) {
         goto error;
       }
-      has_decor = true;
     }
   }
 
   if (HAS_KEY(opts, set_extmark, conceal)) {
+    hl.flags |= kSHConceal;
+    has_hl = true;
     String c = opts->conceal;
-    decor.conceal = true;
-    if (c.size) {
-      decor.conceal_char = utf_ptr2char(c.data);
+    if (c.size > 0) {
+      if (c.size <= 4) {
+        memcpy(hl.conceal_char, c.data, c.size + (c.size < 4 ? 1 : 0));
+      } else {
+        conceal_char_large = c;
+      }
     }
-    has_decor = true;
   }
 
   if (HAS_KEY(opts, set_extmark, virt_text)) {
-    decor.virt_text = parse_virt_text(opts->virt_text, err, &decor.virt_text_width);
-    has_decor = true;
+    virt_text.data.virt_text = parse_virt_text(opts->virt_text, err, &virt_text.width);
     if (ERROR_SET(err)) {
       goto error;
     }
@@ -683,13 +611,13 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
   if (HAS_KEY(opts, set_extmark, virt_text_pos)) {
     String str = opts->virt_text_pos;
     if (strequal("eol", str.data)) {
-      decor.virt_text_pos = kVTEndOfLine;
+      virt_text.pos = kVPosEndOfLine;
     } else if (strequal("overlay", str.data)) {
-      decor.virt_text_pos = kVTOverlay;
+      virt_text.pos = kVPosOverlay;
     } else if (strequal("right_align", str.data)) {
-      decor.virt_text_pos = kVTRightAlign;
+      virt_text.pos = kVPosRightAlign;
     } else if (strequal("inline", str.data)) {
-      decor.virt_text_pos = kVTInline;
+      virt_text.pos = kVPosInline;
     } else {
       VALIDATE_S(false, "virt_text_pos", str.data, {
         goto error;
@@ -698,26 +626,26 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
   }
 
   if (HAS_KEY(opts, set_extmark, virt_text_win_col)) {
-    decor.col = (int)opts->virt_text_win_col;
-    decor.virt_text_pos = kVTWinCol;
+    virt_text.col = (int)opts->virt_text_win_col;
+    virt_text.pos = kVPosWinCol;
   }
 
-  decor.hl_eol = opts->hl_eol;
-  decor.virt_text_hide = opts->virt_text_hide;
+  hl.flags |= opts->hl_eol ? kSHHlEol : 0;
+  virt_text.flags |= opts->virt_text_hide ? kVTHide : 0;
 
   if (HAS_KEY(opts, set_extmark, hl_mode)) {
     String str = opts->hl_mode;
     if (strequal("replace", str.data)) {
-      decor.hl_mode = kHlModeReplace;
+      virt_text.hl_mode = kHlModeReplace;
     } else if (strequal("combine", str.data)) {
-      decor.hl_mode = kHlModeCombine;
+      virt_text.hl_mode = kHlModeCombine;
     } else if (strequal("blend", str.data)) {
-      if (decor.virt_text_pos == kVTInline) {
+      if (virt_text.pos == kVPosInline) {
         VALIDATE(false, "%s", "cannot use 'blend' hl_mode with inline virtual text", {
           goto error;
         });
       }
-      decor.hl_mode = kHlModeBlend;
+      virt_text.hl_mode = kHlModeBlend;
     } else {
       VALIDATE_S(false, "hl_mode", str.data, {
         goto error;
@@ -735,29 +663,32 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
       });
       int dummig;
       VirtText jtem = parse_virt_text(a.items[j].data.array, err, &dummig);
-      kv_push(decor.virt_lines, ((struct virt_line){ jtem, virt_lines_leftcol }));
+      kv_push(virt_lines.data.virt_lines, ((struct virt_line){ jtem, virt_lines_leftcol }));
       if (ERROR_SET(err)) {
         goto error;
       }
-      has_decor = true;
     }
   }
 
-  decor.virt_lines_above = opts->virt_lines_above;
+  virt_lines.flags |= opts->virt_lines_above ? kVTLinesAbove : 0;
 
   if (HAS_KEY(opts, set_extmark, priority)) {
     VALIDATE_RANGE((opts->priority >= 0 && opts->priority <= UINT16_MAX), "priority", {
       goto error;
     });
-    decor.priority = (DecorPriority)opts->priority;
+    hl.priority = (DecorPriority)opts->priority;
+    sign.priority = (DecorPriority)opts->priority;
+    virt_text.priority = (DecorPriority)opts->priority;
+    virt_lines.priority = (DecorPriority)opts->priority;
   }
 
   if (HAS_KEY(opts, set_extmark, sign_text)) {
-    VALIDATE_S(init_sign_text(NULL, &decor.sign_text, opts->sign_text.data),
+    sign.text.ptr = NULL;
+    VALIDATE_S(init_sign_text(NULL, &sign.text.ptr, opts->sign_text.data),
                "sign_text", "", {
       goto error;
     });
-    has_decor = true;
+    sign.flags |= kSHIsSign;
   }
 
   bool right_gravity = GET_BOOL_OR_TRUE(opts, set_extmark, right_gravity);
@@ -771,16 +702,18 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
 
   size_t len = 0;
 
-  if (!HAS_KEY(opts, set_extmark, spell)) {
-    decor.spell = kNone;
-  } else {
-    decor.spell = opts->spell ? kTrue : kFalse;
-    has_decor = true;
+  if (HAS_KEY(opts, set_extmark, spell)) {
+    hl.flags |= (opts->spell) ? kSHSpellOn : kSHSpellOff;
+    has_hl = true;
   }
 
-  decor.ui_watched = opts->ui_watched;
-  if (decor.ui_watched) {
-    has_decor = true;
+  if (opts->ui_watched) {
+    hl.flags |= kSHUIWatched;
+    if (virt_text.pos == kVPosOverlay) {
+      // TODO(bfredl): in a revised interface this should be the default.
+      hl.flags |= kSHUIWatchedOverlay;
+    }
+    has_hl = true;
   }
 
   VALIDATE_RANGE((line >= 0), "line", {
@@ -829,28 +762,90 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
     col2 = 0;
   }
 
-  // TODO(bfredl): synergize these two branches even more
   if (opts->ephemeral && decor_state.win && decor_state.win->w_buffer == buf) {
-    decor_push_ephemeral((int)line, (int)col, line2, col2, &decor, (uint64_t)ns_id, id);
+    int r = (int)line;
+    int c = (int)col;
+    if (line2 == -1) {
+      line2 = r;
+      col2 = c;
+    }
+
+    if (kv_size(virt_text.data.virt_text)) {
+      decor_range_add_virt(&decor_state, r, c, line2, col2, decor_put_vt(virt_text, NULL), true);
+    }
+    if (kv_size(virt_lines.data.virt_lines)) {
+      decor_range_add_virt(&decor_state, r, c, line2, col2, decor_put_vt(virt_lines, NULL), true);
+    }
+    if (has_hl) {
+      DecorSignHighlight sh = decor_sh_from_inline(hl, conceal_char_large);
+      decor_range_add_sh(&decor_state, r, c, line2, col2, &sh, true, (uint32_t)ns_id, id);
+    }
+    if (sign.flags & kSHIsSign) {
+      decor_range_add_sh(&decor_state, r, c, line2, col2, &sign, true, (uint32_t)ns_id, id);
+    }
   } else {
     if (opts->ephemeral) {
       api_set_error(err, kErrorTypeException, "not yet implemented");
       goto error;
     }
 
+    uint16_t decor_flags = 0;
+
+    DecorVirtText *decor_alloc = NULL;
+    if (kv_size(virt_text.data.virt_text)) {
+      decor_alloc = decor_put_vt(virt_text, decor_alloc);
+      if (virt_text.pos == kVPosInline) {
+        decor_flags |= MT_FLAG_DECOR_VIRT_TEXT_INLINE;
+      }
+    }
+    if (kv_size(virt_lines.data.virt_lines)) {
+      decor_alloc = decor_put_vt(virt_lines, decor_alloc);
+      decor_flags |= MT_FLAG_DECOR_VIRT_LINES;
+    }
+
+    uint32_t decor_indexed = DECOR_ID_INVALID;
+    if (sign.flags & kSHIsSign) {
+      decor_indexed = decor_put_sh(sign);
+      if (sign.text.ptr != NULL) {
+        decor_flags |= MT_FLAG_DECOR_SIGNTEXT;
+      }
+      if (sign.number_hl_id || sign.line_hl_id || sign.cursorline_hl_id) {
+        decor_flags |= MT_FLAG_DECOR_SIGNHL;
+      }
+    }
+
+    DecorInline decor = DECOR_INLINE_INIT;
+    if (decor_alloc || decor_indexed != DECOR_ID_INVALID || conceal_char_large.size) {
+      if (has_hl) {
+        DecorSignHighlight sh = decor_sh_from_inline(hl, conceal_char_large);
+        sh.next = decor_indexed;
+        decor_indexed = decor_put_sh(sh);
+      }
+      decor.ext = true;
+      decor.data.ext = (DecorExt){ .sh_idx = decor_indexed, .vt = decor_alloc };
+    } else {
+      decor.data.hl = hl;
+    }
+
+    if (has_hl) {
+      decor_flags |= MT_FLAG_DECOR_HL;
+    }
+
     extmark_set(buf, (uint32_t)ns_id, &id, (int)line, (colnr_T)col, line2, col2,
-                has_decor ? &decor : NULL, right_gravity, opts->end_right_gravity,
+                decor, decor_flags, right_gravity, opts->end_right_gravity,
                 !GET_BOOL_OR_TRUE(opts, set_extmark, undo_restore),
                 opts->invalidate, err);
     if (ERROR_SET(err)) {
-      goto error;
+      decor_free(decor);
+      return 0;
     }
   }
 
   return (Integer)id;
 
 error:
-  decor_clear(&decor);
+  clear_virttext(&virt_text.data.virt_text);
+  clear_virtlines(&virt_lines.data.virt_lines);
   return 0;
 }
 
@@ -873,11 +868,6 @@ Boolean nvim_buf_del_extmark(Buffer buffer, Integer ns_id, Integer id, Error *er
     return false;
   });
 
-  if (decor_state.running_on_lines) {
-    api_set_error(err, kErrorTypeValidation, "Cannot remove extmarks during on_line callbacks");
-    return false;
-  }
-
   return extmark_del_id(buf, (uint32_t)ns_id, (uint32_t)id);
 }
 
@@ -962,13 +952,11 @@ Integer nvim_buf_add_highlight(Buffer buffer, Integer ns_id, String hl_group, In
     end_line++;
   }
 
-  Decoration decor = DECORATION_INIT;
-  decor.hl_id = hl_id;
+  DecorInline decor = DECOR_INLINE_INIT;
+  decor.data.hl.hl_id = hl_id;
 
-  extmark_set(buf, ns, NULL,
-              (int)line, (colnr_T)col_start,
-              end_line, (colnr_T)col_end,
-              &decor, true, false, false, false, NULL);
+  extmark_set(buf, ns, NULL, (int)line, (colnr_T)col_start, end_line, (colnr_T)col_end,
+              decor, MT_FLAG_DECOR_HL, true, false, false, false, NULL);
   return ns_id;
 }
 
@@ -997,11 +985,6 @@ void nvim_buf_clear_namespace(Buffer buffer, Integer ns_id, Integer line_start,
     return;
   });
 
-  if (decor_state.running_on_lines) {
-    api_set_error(err, kErrorTypeValidation, "Cannot remove extmarks during on_line callbacks");
-    return;
-  }
-
   if (line_end < 0 || line_end > MAXLNUM) {
     line_end = MAXLNUM;
   }