Merge pull request #25826 from bfredl/art_deco

refactor(decorations): break up Decoration struct into smaller pieces

15 files changed, 1085 insertions, 610 deletions

diff --git a/src/nvim/api/deprecated.c b/src/nvim/api/deprecated.c
index 849897b529..b4aa6fe99e 100644
--- a/src/nvim/api/deprecated.c
+++ b/src/nvim/api/deprecated.c
@@ -154,21 +154,25 @@ Integer nvim_buf_set_virtual_text(Buffer buffer, Integer src_id, Integer line, A
     return 0;
   }
 
-  Decoration *existing = decor_find_virttext(buf, (int)line, ns_id);
+  DecorVirtText *existing = decor_find_virttext(buf, (int)line, ns_id);
 
   if (existing) {
-    clear_virttext(&existing->virt_text);
-    existing->virt_text = virt_text;
-    existing->virt_text_width = width;
+    clear_virttext(&existing->data.virt_text);
+    existing->data.virt_text = virt_text;
+    existing->width = width;
     return src_id;
   }
 
-  Decoration decor = DECORATION_INIT;
-  decor.virt_text = virt_text;
-  decor.virt_text_width = width;
-  decor.priority = 0;
+  DecorVirtText *vt = xmalloc(sizeof *vt);
+  *vt = (DecorVirtText)DECOR_VIRT_TEXT_INIT;
+  vt->data.virt_text = virt_text;
+  vt->width = width;
+  vt->priority = 0;
 
-  extmark_set(buf, ns_id, NULL, (int)line, 0, -1, -1, &decor, true, false, false, false, NULL);
+  DecorInline decor = { .ext = true, .data.ext.vt = vt, .data.ext.sh_idx = DECOR_ID_INVALID };
+
+  extmark_set(buf, ns_id, NULL, (int)line, 0, -1, -1, decor, 0, true,
+              false, false, false, NULL);
   return src_id;
 }
 
diff --git a/src/nvim/api/extmark.c b/src/nvim/api/extmark.c
index 31fc01403a..8a2cde8372 100644
--- a/src/nvim/api/extmark.c
+++ b/src/nvim/api/extmark.c
@@ -103,15 +103,6 @@ bool ns_initialized(uint32_t ns)
   return ns < (uint32_t)next_namespace_id;
 }
 
-static Object hl_group_name(int hl_id, bool hl_name)
-{
-  if (hl_name) {
-    return CSTR_TO_OBJ(syn_id2name(hl_id));
-  } else {
-    return INTEGER_OBJ(hl_id);
-  }
-}
-
 Array virt_text_to_array(VirtText vt, bool hl_name)
 {
   Array chunks = ARRAY_DICT_INIT;
@@ -176,85 +167,9 @@ static Array extmark_to_array(MTPair extmark, bool id, bool add_dict, bool hl_na
       PUT(dict, "invalid", BOOLEAN_OBJ(true));
     }
 
-    // pretend this is a pointer for a short while, Decoration will be factored away very soon
-    const Decoration decor[1] = { get_decor(start) };
-    if (decor->hl_id) {
-      PUT(dict, "hl_group", hl_group_name(decor->hl_id, hl_name));
-      PUT(dict, "hl_eol", BOOLEAN_OBJ(decor->hl_eol));
-    }
-    if (decor->hl_mode) {
-      PUT(dict, "hl_mode", CSTR_TO_OBJ(hl_mode_str[decor->hl_mode]));
-    }
-
-    if (kv_size(decor->virt_text)) {
-      Array chunks = virt_text_to_array(decor->virt_text, hl_name);
-      PUT(dict, "virt_text", ARRAY_OBJ(chunks));
-      PUT(dict, "virt_text_hide", BOOLEAN_OBJ(decor->virt_text_hide));
-      if (decor->virt_text_pos == kVTWinCol) {
-        PUT(dict, "virt_text_win_col", INTEGER_OBJ(decor->col));
-      }
-      PUT(dict, "virt_text_pos",
-          CSTR_TO_OBJ(virt_text_pos_str[decor->virt_text_pos]));
-    }
-
-    if (decor->ui_watched) {
-      PUT(dict, "ui_watched", BOOLEAN_OBJ(true));
-    }
-
-    if (kv_size(decor->virt_lines)) {
-      Array all_chunks = ARRAY_DICT_INIT;
-      bool virt_lines_leftcol = false;
-      for (size_t i = 0; i < kv_size(decor->virt_lines); i++) {
-        virt_lines_leftcol = kv_A(decor->virt_lines, i).left_col;
-        Array chunks = virt_text_to_array(kv_A(decor->virt_lines, i).line, hl_name);
-        ADD(all_chunks, ARRAY_OBJ(chunks));
-      }
-      PUT(dict, "virt_lines", ARRAY_OBJ(all_chunks));
-      PUT(dict, "virt_lines_above", BOOLEAN_OBJ(decor->virt_lines_above));
-      PUT(dict, "virt_lines_leftcol", BOOLEAN_OBJ(virt_lines_leftcol));
-    }
-
-    if (decor->sign_text) {
-      PUT(dict, "sign_text", CSTR_TO_OBJ(decor->sign_text));
-    }
-
-    // uncrustify:off
-
-    struct { char *name; const int val; } hls[] = {
-      { "sign_hl_group"      , decor->sign_hl_id       },
-      { "number_hl_group"    , decor->number_hl_id     },
-      { "line_hl_group"      , decor->line_hl_id       },
-      { "cursorline_hl_group", decor->cursorline_hl_id },
-      { NULL, 0 },
-    };
-
-    // uncrustify:on
-
-    for (int j = 0; hls[j].name; j++) {
-      if (hls[j].val) {
-        PUT(dict, hls[j].name, hl_group_name(hls[j].val, hl_name));
-      }
-    }
-
-    if (decor->sign_text
-        || decor->hl_id
-        || kv_size(decor->virt_text)
-        || decor->ui_watched) {
-      PUT(dict, "priority", INTEGER_OBJ(decor->priority));
-    }
-
-    if (decor->conceal) {
-      String name = cstr_to_string((char *)&decor->conceal_char);
-      PUT(dict, "conceal", STRING_OBJ(name));
-    }
-
-    if (decor->spell != kNone) {
-      PUT(dict, "spell", BOOLEAN_OBJ(decor->spell == kTrue));
-    }
+    decor_to_dict_legacy(&dict, mt_decor(start), hl_name);
 
-    if (dict.size) {
-      ADD(rv, DICTIONARY_OBJ(dict));
-    }
+    ADD(rv, DICTIONARY_OBJ(dict));
   }
 
   return rv;
@@ -581,8 +496,14 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
                              Dict(set_extmark) *opts, Error *err)
   FUNC_API_SINCE(7)
 {
-  Decoration decor = DECORATION_INIT;
-  bool has_decor = false;
+  DecorHighlightInline hl = DECOR_HIGHLIGHT_INLINE_INIT;
+  // TODO(bfredl): in principle signs with max one (1) hl group and max 4 bytes of text.
+  // should be a candidate for inlining as well.
+  DecorSignHighlight sign = DECOR_SIGN_HIGHLIGHT_INIT;
+  DecorVirtText virt_text = DECOR_VIRT_TEXT_INIT;
+  DecorVirtText virt_lines = DECOR_VIRT_LINES_INIT;
+  bool has_hl = false;
+  String conceal_char_large = STRING_INIT;
 
   buf_T *buf = find_buffer_by_handle(buffer, err);
   if (!buf) {
@@ -643,11 +564,11 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
     Object *opt;
     int *dest;
   } hls[] = {
-    { "hl_group"           , &opts->hl_group           , &decor.hl_id            },
-    { "sign_hl_group"      , &opts->sign_hl_group      , &decor.sign_hl_id       },
-    { "number_hl_group"    , &opts->number_hl_group    , &decor.number_hl_id     },
-    { "line_hl_group"      , &opts->line_hl_group      , &decor.line_hl_id       },
-    { "cursorline_hl_group", &opts->cursorline_hl_group, &decor.cursorline_hl_id },
+    { "hl_group"           , &opts->hl_group           , &hl.hl_id            },
+    { "sign_hl_group"      , &opts->sign_hl_group      , &sign.hl_id       },
+    { "number_hl_group"    , &opts->number_hl_group    , &sign.number_hl_id     },
+    { "line_hl_group"      , &opts->line_hl_group      , &sign.line_hl_id       },
+    { "cursorline_hl_group", &opts->cursorline_hl_group, &sign.cursorline_hl_id },
     { NULL, NULL, NULL },
   };
 
@@ -655,26 +576,33 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
 
   for (int j = 0; hls[j].name && hls[j].dest; j++) {
     if (hls[j].opt->type != kObjectTypeNil) {
+      if (j > 0) {
+        sign.flags |= kSHIsSign;
+      } else {
+        has_hl = true;
+      }
       *hls[j].dest = object_to_hl_id(*hls[j].opt, hls[j].name, err);
       if (ERROR_SET(err)) {
         goto error;
       }
-      has_decor = true;
     }
   }
 
   if (HAS_KEY(opts, set_extmark, conceal)) {
+    hl.flags |= kSHConceal;
+    has_hl = true;
     String c = opts->conceal;
-    decor.conceal = true;
-    if (c.size) {
-      decor.conceal_char = utf_ptr2char(c.data);
+    if (c.size > 0) {
+      if (c.size <= 4) {
+        memcpy(hl.conceal_char, c.data, c.size + (c.size < 4 ? 1 : 0));
+      } else {
+        conceal_char_large = c;
+      }
     }
-    has_decor = true;
   }
 
   if (HAS_KEY(opts, set_extmark, virt_text)) {
-    decor.virt_text = parse_virt_text(opts->virt_text, err, &decor.virt_text_width);
-    has_decor = true;
+    virt_text.data.virt_text = parse_virt_text(opts->virt_text, err, &virt_text.width);
     if (ERROR_SET(err)) {
       goto error;
     }
@@ -683,13 +611,13 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
   if (HAS_KEY(opts, set_extmark, virt_text_pos)) {
     String str = opts->virt_text_pos;
     if (strequal("eol", str.data)) {
-      decor.virt_text_pos = kVTEndOfLine;
+      virt_text.pos = kVPosEndOfLine;
     } else if (strequal("overlay", str.data)) {
-      decor.virt_text_pos = kVTOverlay;
+      virt_text.pos = kVPosOverlay;
     } else if (strequal("right_align", str.data)) {
-      decor.virt_text_pos = kVTRightAlign;
+      virt_text.pos = kVPosRightAlign;
     } else if (strequal("inline", str.data)) {
-      decor.virt_text_pos = kVTInline;
+      virt_text.pos = kVPosInline;
     } else {
       VALIDATE_S(false, "virt_text_pos", str.data, {
         goto error;
@@ -698,26 +626,26 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
   }
 
   if (HAS_KEY(opts, set_extmark, virt_text_win_col)) {
-    decor.col = (int)opts->virt_text_win_col;
-    decor.virt_text_pos = kVTWinCol;
+    virt_text.col = (int)opts->virt_text_win_col;
+    virt_text.pos = kVPosWinCol;
   }
 
-  decor.hl_eol = opts->hl_eol;
-  decor.virt_text_hide = opts->virt_text_hide;
+  hl.flags |= opts->hl_eol ? kSHHlEol : 0;
+  virt_text.flags |= opts->virt_text_hide ? kVTHide : 0;
 
   if (HAS_KEY(opts, set_extmark, hl_mode)) {
     String str = opts->hl_mode;
     if (strequal("replace", str.data)) {
-      decor.hl_mode = kHlModeReplace;
+      virt_text.hl_mode = kHlModeReplace;
     } else if (strequal("combine", str.data)) {
-      decor.hl_mode = kHlModeCombine;
+      virt_text.hl_mode = kHlModeCombine;
     } else if (strequal("blend", str.data)) {
-      if (decor.virt_text_pos == kVTInline) {
+      if (virt_text.pos == kVPosInline) {
         VALIDATE(false, "%s", "cannot use 'blend' hl_mode with inline virtual text", {
           goto error;
         });
       }
-      decor.hl_mode = kHlModeBlend;
+      virt_text.hl_mode = kHlModeBlend;
     } else {
       VALIDATE_S(false, "hl_mode", str.data, {
         goto error;
@@ -735,29 +663,32 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
       });
       int dummig;
       VirtText jtem = parse_virt_text(a.items[j].data.array, err, &dummig);
-      kv_push(decor.virt_lines, ((struct virt_line){ jtem, virt_lines_leftcol }));
+      kv_push(virt_lines.data.virt_lines, ((struct virt_line){ jtem, virt_lines_leftcol }));
       if (ERROR_SET(err)) {
         goto error;
       }
-      has_decor = true;
     }
   }
 
-  decor.virt_lines_above = opts->virt_lines_above;
+  virt_lines.flags |= opts->virt_lines_above ? kVTLinesAbove : 0;
 
   if (HAS_KEY(opts, set_extmark, priority)) {
     VALIDATE_RANGE((opts->priority >= 0 && opts->priority <= UINT16_MAX), "priority", {
       goto error;
     });
-    decor.priority = (DecorPriority)opts->priority;
+    hl.priority = (DecorPriority)opts->priority;
+    sign.priority = (DecorPriority)opts->priority;
+    virt_text.priority = (DecorPriority)opts->priority;
+    virt_lines.priority = (DecorPriority)opts->priority;
   }
 
   if (HAS_KEY(opts, set_extmark, sign_text)) {
-    VALIDATE_S(init_sign_text(NULL, &decor.sign_text, opts->sign_text.data),
+    sign.text.ptr = NULL;
+    VALIDATE_S(init_sign_text(NULL, &sign.text.ptr, opts->sign_text.data),
                "sign_text", "", {
       goto error;
     });
-    has_decor = true;
+    sign.flags |= kSHIsSign;
   }
 
   bool right_gravity = GET_BOOL_OR_TRUE(opts, set_extmark, right_gravity);
@@ -771,16 +702,18 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
 
   size_t len = 0;
 
-  if (!HAS_KEY(opts, set_extmark, spell)) {
-    decor.spell = kNone;
-  } else {
-    decor.spell = opts->spell ? kTrue : kFalse;
-    has_decor = true;
+  if (HAS_KEY(opts, set_extmark, spell)) {
+    hl.flags |= (opts->spell) ? kSHSpellOn : kSHSpellOff;
+    has_hl = true;
   }
 
-  decor.ui_watched = opts->ui_watched;
-  if (decor.ui_watched) {
-    has_decor = true;
+  if (opts->ui_watched) {
+    hl.flags |= kSHUIWatched;
+    if (virt_text.pos == kVPosOverlay) {
+      // TODO(bfredl): in a revised interface this should be the default.
+      hl.flags |= kSHUIWatchedOverlay;
+    }
+    has_hl = true;
   }
 
   VALIDATE_RANGE((line >= 0), "line", {
@@ -829,28 +762,90 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
     col2 = 0;
   }
 
-  // TODO(bfredl): synergize these two branches even more
   if (opts->ephemeral && decor_state.win && decor_state.win->w_buffer == buf) {
-    decor_push_ephemeral((int)line, (int)col, line2, col2, &decor, (uint64_t)ns_id, id);
+    int r = (int)line;
+    int c = (int)col;
+    if (line2 == -1) {
+      line2 = r;
+      col2 = c;
+    }
+
+    if (kv_size(virt_text.data.virt_text)) {
+      decor_range_add_virt(&decor_state, r, c, line2, col2, decor_put_vt(virt_text, NULL), true);
+    }
+    if (kv_size(virt_lines.data.virt_lines)) {
+      decor_range_add_virt(&decor_state, r, c, line2, col2, decor_put_vt(virt_lines, NULL), true);
+    }
+    if (has_hl) {
+      DecorSignHighlight sh = decor_sh_from_inline(hl, conceal_char_large);
+      decor_range_add_sh(&decor_state, r, c, line2, col2, &sh, true, (uint32_t)ns_id, id);
+    }
+    if (sign.flags & kSHIsSign) {
+      decor_range_add_sh(&decor_state, r, c, line2, col2, &sign, true, (uint32_t)ns_id, id);
+    }
   } else {
     if (opts->ephemeral) {
       api_set_error(err, kErrorTypeException, "not yet implemented");
       goto error;
     }
 
+    uint16_t decor_flags = 0;
+
+    DecorVirtText *decor_alloc = NULL;
+    if (kv_size(virt_text.data.virt_text)) {
+      decor_alloc = decor_put_vt(virt_text, decor_alloc);
+      if (virt_text.pos == kVPosInline) {
+        decor_flags |= MT_FLAG_DECOR_VIRT_TEXT_INLINE;
+      }
+    }
+    if (kv_size(virt_lines.data.virt_lines)) {
+      decor_alloc = decor_put_vt(virt_lines, decor_alloc);
+      decor_flags |= MT_FLAG_DECOR_VIRT_LINES;
+    }
+
+    uint32_t decor_indexed = DECOR_ID_INVALID;
+    if (sign.flags & kSHIsSign) {
+      decor_indexed = decor_put_sh(sign);
+      if (sign.text.ptr != NULL) {
+        decor_flags |= MT_FLAG_DECOR_SIGNTEXT;
+      }
+      if (sign.number_hl_id || sign.line_hl_id || sign.cursorline_hl_id) {
+        decor_flags |= MT_FLAG_DECOR_SIGNHL;
+      }
+    }
+
+    DecorInline decor = DECOR_INLINE_INIT;
+    if (decor_alloc || decor_indexed != DECOR_ID_INVALID || conceal_char_large.size) {
+      if (has_hl) {
+        DecorSignHighlight sh = decor_sh_from_inline(hl, conceal_char_large);
+        sh.next = decor_indexed;
+        decor_indexed = decor_put_sh(sh);
+      }
+      decor.ext = true;
+      decor.data.ext = (DecorExt){ .sh_idx = decor_indexed, .vt = decor_alloc };
+    } else {
+      decor.data.hl = hl;
+    }
+
+    if (has_hl) {
+      decor_flags |= MT_FLAG_DECOR_HL;
+    }
+
     extmark_set(buf, (uint32_t)ns_id, &id, (int)line, (colnr_T)col, line2, col2,
-                has_decor ? &decor : NULL, right_gravity, opts->end_right_gravity,
+                decor, decor_flags, right_gravity, opts->end_right_gravity,
                 !GET_BOOL_OR_TRUE(opts, set_extmark, undo_restore),
                 opts->invalidate, err);
     if (ERROR_SET(err)) {
-      goto error;
+      decor_free(decor);
+      return 0;
     }
   }
 
   return (Integer)id;
 
 error:
-  decor_clear(&decor);
+  clear_virttext(&virt_text.data.virt_text);
+  clear_virtlines(&virt_lines.data.virt_lines);
   return 0;
 }
 
@@ -873,11 +868,6 @@ Boolean nvim_buf_del_extmark(Buffer buffer, Integer ns_id, Integer id, Error *er
     return false;
   });
 
-  if (decor_state.running_on_lines) {
-    api_set_error(err, kErrorTypeValidation, "Cannot remove extmarks during on_line callbacks");
-    return false;
-  }
-
   return extmark_del_id(buf, (uint32_t)ns_id, (uint32_t)id);
 }
 
@@ -962,13 +952,11 @@ Integer nvim_buf_add_highlight(Buffer buffer, Integer ns_id, String hl_group, In
     end_line++;
   }
 
-  Decoration decor = DECORATION_INIT;
-  decor.hl_id = hl_id;
+  DecorInline decor = DECOR_INLINE_INIT;
+  decor.data.hl.hl_id = hl_id;
 
-  extmark_set(buf, ns, NULL,
-              (int)line, (colnr_T)col_start,
-              end_line, (colnr_T)col_end,
-              &decor, true, false, false, false, NULL);
+  extmark_set(buf, ns, NULL, (int)line, (colnr_T)col_start, end_line, (colnr_T)col_end,
+              decor, MT_FLAG_DECOR_HL, true, false, false, false, NULL);
   return ns_id;
 }
 
@@ -997,11 +985,6 @@ void nvim_buf_clear_namespace(Buffer buffer, Integer ns_id, Integer line_start,
     return;
   });
 
-  if (decor_state.running_on_lines) {
-    api_set_error(err, kErrorTypeValidation, "Cannot remove extmarks during on_line callbacks");
-    return;
-  }
-
   if (line_end < 0 || line_end > MAXLNUM) {
     line_end = MAXLNUM;
   }
diff --git a/src/nvim/decoration.c b/src/nvim/decoration.c
index b784d8bea3..91d5bfcc54 100644
--- a/src/nvim/decoration.c
+++ b/src/nvim/decoration.c
@@ -1,6 +1,7 @@
 #include <assert.h>
 #include <limits.h>
 
+#include "nvim/api/extmark.h"
 #include "nvim/buffer.h"
 #include "nvim/decoration.h"
 #include "nvim/drawscreen.h"
@@ -17,6 +18,17 @@
 # include "decoration.c.generated.h"
 #endif
 
+// TODO(bfredl): These should maybe be per-buffer, so that all resources
+// asssociated with a buffer can be freed when the buffer is unloaded.
+kvec_t(DecorSignHighlight) decor_items = KV_INITIAL_VALUE;
+uint32_t decor_freelist = UINT32_MAX;
+
+// Decorations might be requested to be deleted in a callback in the middle of redrawing.
+// In this case, there might still be live references to the memory allocated for the decoration.
+// Keep a "to free" list which can be safely processed when redrawing is done.
+DecorVirtText *to_free_virt = NULL;
+uint32_t to_free_sh = UINT32_MAX;
+
 /// Add highlighting to a buffer, bounded by two cursor positions,
 /// with an offset.
 ///
@@ -35,8 +47,8 @@ void bufhl_add_hl_pos_offset(buf_T *buf, int src_id, int hl_id, lpos_T pos_start
 {
   colnr_T hl_start = 0;
   colnr_T hl_end = 0;
-  Decoration decor = DECORATION_INIT;
-  decor.hl_id = hl_id;
+  DecorInline decor = DECOR_INLINE_INIT;
+  decor.data.hl.hl_id = hl_id;
 
   // TODO(bfredl): if decoration had blocky mode, we could avoid this loop
   for (linenr_T lnum = pos_start.lnum; lnum <= pos_end.lnum; lnum++) {
@@ -61,113 +73,271 @@ void bufhl_add_hl_pos_offset(buf_T *buf, int src_id, int hl_id, lpos_T pos_start
       hl_start = pos_start.col + offset;
       hl_end = pos_end.col + offset;
     }
+
     extmark_set(buf, (uint32_t)src_id, NULL,
                 (int)lnum - 1, hl_start, (int)lnum - 1 + end_off, hl_end,
-                &decor, true, false, true, false, NULL);
+                decor, MT_FLAG_DECOR_HL, true, false, true, false, NULL);
   }
 }
 
-void decor_redraw(buf_T *buf, int row1, int row2, Decoration *decor)
+void decor_redraw(buf_T *buf, int row1, int row2, DecorInline decor)
 {
   if (row2 >= row1) {
-    if (!decor
-        || decor->hl_id
-        || decor_has_sign(decor)
-        || decor->conceal
-        || decor->spell != kNone) {
-      redraw_buf_range_later(buf, row1 + 1, row2 + 1);
+    redraw_buf_range_later(buf, row1 + 1, row2 + 1);
+  }
+
+  if (decor.ext) {
+    DecorVirtText *vt = decor.data.ext.vt;
+    while (vt) {
+      if (vt->flags & kVTIsLines) {
+        redraw_buf_line_later(buf, row1 + 1 + ((vt->flags & kVTLinesAbove) ? 0 : 1), true);
+        changed_line_display_buf(buf);
+      } else {
+        if (vt->pos == kVPosInline) {
+          changed_line_display_buf(buf);
+        }
+      }
+      vt = vt->next;
     }
+
+    uint32_t idx = decor.data.ext.sh_idx;
+    while (idx != DECOR_ID_INVALID) {
+      DecorSignHighlight *sh = &kv_A(decor_items, idx);
+      decor_redraw_sh(buf, row1, row2, *sh);
+      idx = sh->next;
+    }
+  } else {
+    decor_redraw_sh(buf, row1, row2, decor_sh_from_inline(decor.data.hl, (String)STRING_INIT));
   }
+}
 
-  if (decor && decor_virt_pos(decor)) {
-    redraw_buf_line_later(buf, row1 + 1, false);
-    if (decor->virt_text_pos == kVTInline) {
-      changed_line_display_buf(buf);
+void decor_redraw_sh(buf_T *buf, int row1, int row2, DecorSignHighlight sh)
+{
+  if (sh.hl_id || (sh.flags & (kSHIsSign|kSHSpellOn|kSHSpellOff))) {
+    if (row2 >= row1) {
+      redraw_buf_range_later(buf, row1 + 1, row2 + 1);
     }
   }
+  if (sh.flags & kSHUIWatched) {
+    redraw_buf_line_later(buf, row1 + 1, false);
+  }
+}
 
-  if (decor && kv_size(decor->virt_lines)) {
-    redraw_buf_line_later(buf, row1 + 1 + (decor->virt_lines_above ? 0 : 1), true);
-    changed_line_display_buf(buf);
+uint32_t decor_put_sh(DecorSignHighlight item)
+{
+  if (decor_freelist != UINT32_MAX) {
+    uint32_t pos = decor_freelist;
+    decor_freelist = kv_A(decor_items, decor_freelist).next;
+    kv_A(decor_items, pos) = item;
+    return pos;
+  } else {
+    uint32_t pos = (uint32_t)kv_size(decor_items);
+    kv_push(decor_items, item);
+    return pos;
   }
 }
 
-static int sign_add_id = 0;
+DecorVirtText *decor_put_vt(DecorVirtText vt, DecorVirtText *next)
+{
+  DecorVirtText *decor_alloc = xmalloc(sizeof *decor_alloc);
+  *decor_alloc = vt;
+  decor_alloc->next = next;
+  return decor_alloc;
+}
 
-void decor_add(buf_T *buf, int row, int row2, Decoration *decor, bool hl_id)
+DecorSignHighlight decor_sh_from_inline(DecorHighlightInline item, String conceal_large)
 {
-  if (decor) {
-    if (kv_size(decor->virt_text) && decor->virt_text_pos == kVTInline) {
-      buf->b_virt_text_inline++;
+  // TODO(bfredl): Eventually simple signs will be inlinable as well
+  assert(!(item.flags & kSHIsSign));
+  DecorSignHighlight conv = {
+    .flags = item.flags,
+    .priority = item.priority,
+    .text.data = { 0 },
+    .hl_id = item.hl_id,
+    .number_hl_id = 0,
+    .line_hl_id = 0,
+    .cursorline_hl_id = 0,
+    .next = DECOR_ID_INVALID,
+  };
+
+  // TODO(bfredl): 'tis a little bullshit. Won't need it once conceals and signs to use schar_T
+  if (conceal_large.size) {
+    String c = conceal_large;
+    if (c.size <= 8) {
+      memcpy(conv.text.data, c.data, c.size + (c.size < 8 ? 1 : 0));
+    } else {
+      conv.flags |= kSHConcealAlloc;
+      conv.text.ptr = xstrdup(conceal_large.data);
+    }
+  } else {
+    memcpy(conv.text.data, item.conceal_char, 4);
+    conv.text.data[4] = NUL;
+  }
+  return conv;
+}
+
+void buf_put_decor(buf_T *buf, DecorInline decor, int row)
+{
+  if (decor.ext) {
+    if (decor.data.ext.vt) {
+      buf_put_decor_virt(buf, decor.data.ext.vt);
     }
-    if (kv_size(decor->virt_lines)) {
-      buf->b_virt_line_blocks++;
+    if (decor.data.ext.sh_idx != DECOR_ID_INVALID) {
+      buf_put_decor_sh(buf, &kv_A(decor_items, decor.data.ext.sh_idx), row);
     }
-    if (decor_has_sign(decor)) {
-      decor->sign_add_id = sign_add_id++;
-      buf->b_signs++;
+  }
+}
+
+void buf_put_decor_virt(buf_T *buf, DecorVirtText *vt)
+{
+  if (vt->flags &kVTIsLines) {
+    buf->b_virt_line_blocks++;
+  } else {
+    if (vt->pos == kVPosInline) {
+      buf->b_virt_text_inline++;
     }
-    if (decor->sign_text) {
+  }
+  if (vt->next) {
+    buf_put_decor_virt(buf, vt->next);
+  }
+}
+
+static int sign_add_id = 0;
+void buf_put_decor_sh(buf_T *buf, DecorSignHighlight *sh, int row)
+{
+  if (sh->flags & kSHIsSign) {
+    sh->sign_add_id = sign_add_id++;
+    buf->b_signs++;
+    if (sh->text.ptr) {
       buf->b_signs_with_text++;
       buf_signcols_add_check(buf, row + 1);
     }
   }
-  if (decor || hl_id) {
-    decor_redraw(buf, row, row2 > -1 ? row2 : row, decor);
-  }
 }
 
-void decor_remove(buf_T *buf, int row, int row2, Decoration *decor, bool invalidate)
+void buf_decor_remove(buf_T *buf, int row, int row2, DecorInline decor, bool free)
 {
   decor_redraw(buf, row, row2, decor);
-  if (decor) {
-    if (kv_size(decor->virt_text) && decor->virt_text_pos == kVTInline) {
+  if (decor.ext) {
+    DecorVirtText *vt = decor.data.ext.vt;
+    while (vt) {
+      buf_remove_decor_virt(buf, vt);
+      vt = vt->next;
+    }
+    uint32_t idx = decor.data.ext.sh_idx;
+    while (idx != DECOR_ID_INVALID) {
+      DecorSignHighlight *sh = &kv_A(decor_items, idx);
+      buf_remove_decor_sh(buf, row, row2, sh);
+      idx = sh->next;
+    }
+    if (free) {
+      decor_free(decor);
+    }
+  }
+}
+
+void buf_remove_decor_virt(buf_T *buf, DecorVirtText *vt)
+{
+  if (vt->flags &kVTIsLines) {
+    assert(buf->b_virt_line_blocks > 0);
+    buf->b_virt_line_blocks--;
+  } else {
+    if (vt->pos == kVPosInline) {
       assert(buf->b_virt_text_inline > 0);
       buf->b_virt_text_inline--;
     }
-    if (kv_size(decor->virt_lines)) {
-      assert(buf->b_virt_line_blocks > 0);
-      buf->b_virt_line_blocks--;
-    }
-    if (decor_has_sign(decor)) {
-      assert(buf->b_signs > 0);
-      buf->b_signs--;
-      if (decor->sign_text) {
-        assert(buf->b_signs_with_text > 0);
-        buf->b_signs_with_text--;
-        if (row2 >= row) {
-          buf_signcols_del_check(buf, row + 1, row2 + 1);
-        }
+  }
+}
+
+void buf_remove_decor_sh(buf_T *buf, int row, int row2, DecorSignHighlight *sh)
+{
+  if (sh->flags & kSHIsSign) {
+    assert(buf->b_signs > 0);
+    buf->b_signs--;
+    if (sh->text.ptr) {
+      assert(buf->b_signs_with_text > 0);
+      buf->b_signs_with_text--;
+      if (row2 >= row) {
+        buf_signcols_del_check(buf, row + 1, row2 + 1);
       }
     }
   }
-  if (!invalidate) {
-    decor_free(decor);
-  }
 }
 
-void decor_clear(Decoration *decor)
+void decor_free(DecorInline decor)
 {
-  clear_virttext(&decor->virt_text);
-  for (size_t i = 0; i < kv_size(decor->virt_lines); i++) {
-    clear_virttext(&kv_A(decor->virt_lines, i).line);
+  if (!decor.ext) {
+    return;
+  }
+  DecorVirtText *vt = decor.data.ext.vt;
+  uint32_t idx = decor.data.ext.sh_idx;
+
+  if (decor_state.running_decor_provider) {
+    while (vt) {
+      if (vt->next == NULL) {
+        vt->next = to_free_virt;
+        to_free_virt = decor.data.ext.vt;
+        break;
+      }
+      vt = vt->next;
+    }
+    while (idx != DECOR_ID_INVALID) {
+      DecorSignHighlight *sh = &kv_A(decor_items, idx);
+      if (sh->next == DECOR_ID_INVALID) {
+        sh->next = to_free_sh;
+        to_free_sh = decor.data.ext.sh_idx;
+        break;
+      }
+      idx = sh->next;
+    }
+  } else {
+    // safe to delete right now
+    decor_free_inner(vt, idx);
   }
-  kv_destroy(decor->virt_lines);
-  xfree(decor->sign_text);
-  xfree(decor->sign_name);
 }
 
-void decor_free(Decoration *decor)
+void decor_free_inner(DecorVirtText *vt, uint32_t first_idx)
 {
-  if (decor) {
-    decor_clear(decor);
-    xfree(decor);
+  while (vt) {
+    if (vt->flags & kVTIsLines) {
+      clear_virtlines(&vt->data.virt_lines);
+    } else {
+      clear_virttext(&vt->data.virt_text);
+    }
+    DecorVirtText *tofree = vt;
+    vt = vt->next;
+    xfree(tofree);
+  }
+
+  uint32_t idx = first_idx;
+  while (idx != DECOR_ID_INVALID) {
+    DecorSignHighlight *sh = &kv_A(decor_items, idx);
+    if (sh->flags & (kSHIsSign | kSHConcealAlloc)) {
+      xfree(sh->text.ptr);
+    }
+    if (sh->flags & kSHIsSign) {
+      xfree(sh->sign_name);
+    }
+    if (sh->next == DECOR_ID_INVALID) {
+      sh->next = decor_freelist;
+      decor_freelist = first_idx;
+      break;
+    }
+    idx = sh->next;
   }
 }
 
+void decor_check_to_be_deleted(void)
+{
+  assert(!decor_state.running_decor_provider);
+  decor_free_inner(to_free_virt, to_free_sh);
+  to_free_virt = NULL;
+  to_free_sh = DECOR_ID_INVALID;
+}
+
 void decor_state_free(DecorState *state)
 {
-  xfree(state->active.items);
+  kv_destroy(state->active);
 }
 
 void clear_virttext(VirtText *text)
@@ -179,6 +349,15 @@ void clear_virttext(VirtText *text)
   *text = (VirtText)KV_INITIAL_VALUE;
 }
 
+void clear_virtlines(VirtLines *lines)
+{
+  for (size_t i = 0; i < kv_size(*lines); i++) {
+    clear_virttext(&kv_A(*lines, i).line);
+  }
+  kv_destroy(*lines);
+  *lines = (VirtLines)KV_INITIAL_VALUE;
+}
+
 /// Get the next chunk of a virtual text item.
 ///
 /// @param[in]     vt    The virtual text item
@@ -197,7 +376,7 @@ char *next_virt_text_chunk(VirtText vt, size_t *pos, int *attr)
   return text;
 }
 
-Decoration *decor_find_virttext(buf_T *buf, int row, uint64_t ns_id)
+DecorVirtText *decor_find_virttext(buf_T *buf, int row, uint64_t ns_id)
 {
   MarkTreeIter itr[1] = { 0 };
   marktree_itr_get(buf->b_marktree, row, 0,  itr);
@@ -205,12 +384,14 @@ Decoration *decor_find_virttext(buf_T *buf, int row, uint64_t ns_id)
     MTKey mark = marktree_itr_current(itr);
     if (mark.pos.row < 0 || mark.pos.row > row) {
       break;
-    } else if (mt_invalid(mark) || marktree_decor_level(mark) < kDecorLevelVisible) {
+    } else if (mt_invalid(mark) || !(mark.flags & MT_FLAG_DECOR_EXT)) {
       goto next_mark;
     }
-    Decoration *decor = mark.decor_full;
-    if ((ns_id == 0 || ns_id == mark.ns)
-        && decor && kv_size(decor->virt_text)) {
+    DecorVirtText *decor = mark.decor_data.ext.vt;
+    while (decor && (decor->flags & kVTIsLines)) {
+      decor = decor->next;
+    }
+    if ((ns_id == 0 || ns_id == mark.ns) && decor) {
       return decor;
     }
 next_mark:
@@ -225,30 +406,30 @@ bool decor_redraw_reset(win_T *wp, DecorState *state)
   state->win = wp;
   for (size_t i = 0; i < kv_size(state->active); i++) {
     DecorRange item = kv_A(state->active, i);
-    if (item.virt_text_owned) {
-      clear_virttext(&item.decor.virt_text);
+    if (item.owned && item.kind == kDecorKindVirtText) {
+      clear_virttext(&item.data.vt->data.virt_text);
+      xfree(item.data.vt);
     }
   }
   kv_size(state->active) = 0;
   return wp->w_buffer->b_marktree->n_keys;
 }
 
-Decoration get_decor(MTKey mark)
+/// @return true if decor has a virtual position (virtual text or ui_watched)
+bool decor_virt_pos(const DecorRange *decor)
 {
-  if (mark.decor_full) {
-    return *mark.decor_full;
-  }
-  Decoration fake = DECORATION_INIT;
-  fake.hl_id = mark.hl_id;
-  fake.priority = mark.priority;
-  fake.hl_eol = (mark.flags & MT_FLAG_HL_EOL);