refactor(grid): change schar_T representation to be more compact

Previously, a screen cell would occupy 28+4=32 bytes per cell
as we always made space for up to MAX_MCO+1 codepoints in a cell.

As an example, even a pretty modest 50*80 screen would consume

50*80*2*32 = 256000, i e a quarter megabyte

With the factor of two due to the TUI side buffer, and even more when
using msg_grid and/or ext_multigrid.

This instead stores a 4-byte union of either:
- a valid UTF-8 sequence up to 4 bytes
- an escape char which is invalid UTF-8 (0xFF) plus a 24-bit index to a
  glyph cache

This avoids allocating space for huge composed glyphs _upfront_, while
still keeping rendering such glyphs reasonably fast (1 hash table lookup
+ one plain index lookup). If the same large glyphs are using repeatedly
on the screen, this is still a net reduction of memory/cache
consumption. The only case which really gets worse is if you blast
the screen full with crazy emojis and zalgo text and even this case
only leads to 4 extra bytes per char.

When only <= 4-byte glyphs are used, plus the 4-byte attribute code,
i e 8 bytes in total there is a factor of four reduction of memory use.
Memory which will be quite hot in cache as the screen buffer is scanned
over in win_line() buffer text drawing

A slight complication is that the representation depends on host byte
order. I've tested this manually by compling and running this
in qemu-s390x and it works fine. We might add a qemu based solution
to CI at some point.

1 files changed, 33 insertions, 15 deletions

diff --git a/src/nvim/drawline.c b/src/nvim/drawline.c
index 811cfc1eb2..8b4786a98e 100644
--- a/src/nvim/drawline.c
+++ b/src/nvim/drawline.c
@@ -106,6 +106,8 @@ typedef struct {
   int c_extra;               ///< extra chars, all the same
   int c_final;               ///< final char, mandatory if set
 
+  int n_closing;             ///< number of chars in fdc which will be closing
+
   bool extra_for_extmark;    ///< n_extra set for inline virtual text
 
   // saved "extra" items for when draw_state becomes WL_LINE (again)
@@ -221,11 +223,11 @@ static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, b
   if (*p == TAB) {
     cells = MIN(tabstop_padding(vcol, buf->b_p_ts, buf->b_p_vts_array), maxcells);
     for (int c = 0; c < cells; c++) {
-      schar_from_ascii(dest[c], ' ');
+      dest[c] = schar_from_ascii(' ');
     }
     goto done;
   } else if ((uint8_t)(*p) < 0x80 && u8cc[0] == 0) {
-    schar_from_ascii(dest[0], *p);
+    dest[0] = schar_from_ascii(*p);
     s->prev_c = u8c;
   } else {
     if (p_arshape && !p_tbidi && ARABIC_CHAR(u8c)) {
@@ -252,10 +254,10 @@ static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, b
     } else {
       s->prev_c = u8c;
     }
-    schar_from_cc(dest[0], u8c, u8cc);
+    dest[0] = schar_from_cc(u8c, u8cc);
   }
   if (cells > 1) {
-    dest[1][0] = 0;
+    dest[1] = 0;
   }
 done:
   s->p += c_len;
@@ -348,17 +350,17 @@ static int draw_virt_text_item(buf_T *buf, int col, VirtText vt, HlMode hl_mode,
                              max_col - col, false, vcol);
     // If we failed to emit a char, we still need to put a space and advance.
     if (cells < 1) {
-      schar_from_ascii(linebuf_char[col], ' ');
+      linebuf_char[col] = schar_from_ascii(' ');
       cells = 1;
     }
     for (int c = 0; c < cells; c++) {
       linebuf_attr[col++] = attr;
     }
-    if (col < max_col && linebuf_char[col][0] == 0) {
+    if (col < max_col && linebuf_char[col] == 0) {
       // If the left half of a double-width char is overwritten,
       // change the right half to a space so that grid redraws properly,
       // but don't advance the current column.
-      schar_from_ascii(linebuf_char[col], ' ');
+      linebuf_char[col] = schar_from_ascii(' ');
     }
     vcol += cells;
   }
@@ -384,7 +386,8 @@ static void handle_foldcolumn(win_T *wp, winlinevars_T *wlv)
   // Allocate a buffer, "wlv->extra[]" may already be in use.
   xfree(wlv->p_extra_free);
   wlv->p_extra_free = xmalloc(MAX_MCO * (size_t)fdc + 1);
-  wlv->n_extra = (int)fill_foldcolumn(wlv->p_extra_free, wp, wlv->foldinfo, wlv->lnum);
+  wlv->n_extra = (int)fill_foldcolumn(wlv->p_extra_free, wp, wlv->foldinfo, wlv->lnum,
+                                      &wlv->n_closing);
   wlv->p_extra_free[wlv->n_extra] = NUL;
   wlv->p_extra = wlv->p_extra_free;
   wlv->c_extra = NUL;
@@ -405,7 +408,7 @@ static void handle_foldcolumn(win_T *wp, winlinevars_T *wlv)
 ///
 /// Assume monocell characters
 /// @return number of chars added to \param p
-size_t fill_foldcolumn(char *p, win_T *wp, foldinfo_T foldinfo, linenr_T lnum)
+size_t fill_foldcolumn(char *p, win_T *wp, foldinfo_T foldinfo, linenr_T lnum, int *n_closing)
 {
   int i = 0;
   int level;
@@ -447,16 +450,23 @@ size_t fill_foldcolumn(char *p, win_T *wp, foldinfo_T foldinfo, linenr_T lnum)
     }
   }
 
+  int n_closing_val = i;
+
   if (closed) {
     if (symbol != 0) {
       // rollback previous write
       char_counter -= (size_t)len;
       memset(&p[char_counter], ' ', (size_t)len);
+      n_closing_val--;
     }
     len = utf_char2bytes(wp->w_p_fcs_chars.foldclosed, &p[char_counter]);
     char_counter += (size_t)len;
   }
 
+  if (n_closing) {
+    *n_closing = n_closing_val;
+  }
+
   return MAX(char_counter + (size_t)(fdc - i), (size_t)fdc);
 }
 
@@ -2737,7 +2747,7 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
           wlv.col += n;
         } else {
           // Add a blank character to highlight.
-          schar_from_ascii(linebuf_char[wlv.off], ' ');
+          linebuf_char[wlv.off] = schar_from_ascii(' ');
         }
         if (area_attr == 0 && !has_fold) {
           // Use attributes from match with highest priority among
@@ -2839,7 +2849,7 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
         int col_stride = wp->w_p_rl ? -1 : 1;
 
         while (wp->w_p_rl ? wlv.col >= 0 : wlv.col < grid->cols) {
-          schar_from_ascii(linebuf_char[wlv.off], ' ');
+          linebuf_char[wlv.off] = schar_from_ascii(' ');
           linebuf_vcol[wlv.off] = MAXCOL;
           wlv.col += col_stride;
           if (draw_color_col) {
@@ -2873,7 +2883,7 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
         // logical line
         int n = wp->w_p_rl ? -1 : 1;
         while (wlv.col >= 0 && wlv.col < grid->cols) {
-          schar_from_ascii(linebuf_char[wlv.off], ' ');
+          linebuf_char[wlv.off] = schar_from_ascii(' ');
           linebuf_attr[wlv.off] = wlv.vcol >= TERM_ATTRS_MAX ? 0 : term_attrs[wlv.vcol];
           linebuf_vcol[wlv.off] = wlv.vcol;
           wlv.off += n;
@@ -2973,9 +2983,9 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
         wlv.col--;
       }
       if (mb_utf8) {
-        schar_from_cc(linebuf_char[wlv.off], mb_c, u8cc);
+        linebuf_char[wlv.off] = schar_from_cc(mb_c, u8cc);
       } else {
-        schar_from_ascii(linebuf_char[wlv.off], (char)c);
+        linebuf_char[wlv.off] = schar_from_ascii((char)c);
       }
       if (multi_attr) {
         linebuf_attr[wlv.off] = multi_attr;
@@ -2986,12 +2996,20 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
 
       linebuf_vcol[wlv.off] = wlv.vcol;
 
+      if (wlv.draw_state == WL_FOLD) {
+        linebuf_vcol[wlv.off] = -2;
+        if (wlv.n_closing > 0) {
+          linebuf_vcol[wlv.off] = -3;
+          wlv.n_closing--;
+        }
+      }
+
       if (utf_char2cells(mb_c) > 1) {
         // Need to fill two screen columns.
         wlv.off++;
         wlv.col++;
         // UTF-8: Put a 0 in the second screen char.
-        linebuf_char[wlv.off][0] = 0;
+        linebuf_char[wlv.off] = 0;
         linebuf_attr[wlv.off] = linebuf_attr[wlv.off - 1];
         if (wlv.draw_state > WL_STC && wlv.filler_todo <= 0) {
           wlv.vcol++;