refactor(grid): change schar_T representation to be more compact

Previously, a screen cell would occupy 28+4=32 bytes per cell
as we always made space for up to MAX_MCO+1 codepoints in a cell.

As an example, even a pretty modest 50*80 screen would consume

50*80*2*32 = 256000, i e a quarter megabyte

With the factor of two due to the TUI side buffer, and even more when
using msg_grid and/or ext_multigrid.

This instead stores a 4-byte union of either:
- a valid UTF-8 sequence up to 4 bytes
- an escape char which is invalid UTF-8 (0xFF) plus a 24-bit index to a
  glyph cache

This avoids allocating space for huge composed glyphs _upfront_, while
still keeping rendering such glyphs reasonably fast (1 hash table lookup
+ one plain index lookup). If the same large glyphs are using repeatedly
on the screen, this is still a net reduction of memory/cache
consumption. The only case which really gets worse is if you blast
the screen full with crazy emojis and zalgo text and even this case
only leads to 4 extra bytes per char.

When only <= 4-byte glyphs are used, plus the 4-byte attribute code,
i e 8 bytes in total there is a factor of four reduction of memory use.
Memory which will be quite hot in cache as the screen buffer is scanned
over in win_line() buffer text drawing

A slight complication is that the representation depends on host byte
order. I've tested this manually by compling and running this
in qemu-s390x and it works fine. We might add a qemu based solution
to CI at some point.

23 files changed, 420 insertions, 171 deletions

diff --git a/src/nvim/api/ui.c b/src/nvim/api/ui.c
index 70c97be984..0ea2310042 100644
--- a/src/nvim/api/ui.c
+++ b/src/nvim/api/ui.c
@@ -833,8 +833,7 @@ void remote_ui_raw_line(UI *ui, Integer grid, Integer row, Integer startcol, Int
     bool was_space = false;
     for (size_t i = 0; i < ncells; i++) {
       repeat++;
-      if (i == ncells - 1 || attrs[i] != attrs[i + 1]
-          || strcmp(chunk[i], chunk[i + 1]) != 0) {
+      if (i == ncells - 1 || attrs[i] != attrs[i + 1] || chunk[i] != chunk[i + 1]) {
         if (UI_BUF_SIZE - BUF_POS(data) < 2 * (1 + 2 + sizeof(schar_T) + 5 + 5) + 1) {
           // close to overflowing the redraw buffer. finish this event,
           // flush, and start a new "grid_line" event at the current position.
@@ -859,7 +858,9 @@ void remote_ui_raw_line(UI *ui, Integer grid, Integer row, Integer startcol, Int
         uint32_t csize = (repeat > 1) ? 3 : ((attrs[i] != last_hl) ? 2 : 1);
         nelem++;
         mpack_array(buf, csize);
-        mpack_str(buf, chunk[i]);
+        char sc_buf[MAX_SCHAR_SIZE];
+        schar_get(sc_buf, chunk[i]);
+        mpack_str(buf, sc_buf);
         if (csize >= 2) {
           mpack_uint(buf, (uint32_t)attrs[i]);
           if (csize >= 3) {
@@ -869,7 +870,7 @@ void remote_ui_raw_line(UI *ui, Integer grid, Integer row, Integer startcol, Int
         data->ncells_pending += MIN(repeat, 2);
         last_hl = attrs[i];
         repeat = 0;
-        was_space = strequal(chunk[i], " ");
+        was_space = chunk[i] == schar_from_ascii(' ');
       }
     }
     // If the last chunk was all spaces, add a clearing chunk even if there are
@@ -893,8 +894,10 @@ void remote_ui_raw_line(UI *ui, Integer grid, Integer row, Integer startcol, Int
     for (int i = 0; i < endcol - startcol; i++) {
       remote_ui_cursor_goto(ui, row, startcol + i);
       remote_ui_highlight_set(ui, attrs[i]);
-      remote_ui_put(ui, chunk[i]);
-      if (utf_ambiguous_width(utf_ptr2char((char *)chunk[i]))) {
+      char sc_buf[MAX_SCHAR_SIZE];
+      schar_get(sc_buf, chunk[i]);
+      remote_ui_put(ui, sc_buf);
+      if (utf_ambiguous_width(utf_ptr2char(sc_buf))) {
         data->client_col = -1;  // force cursor update
       }
     }
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 916409b973..0a94b8aafc 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -1947,7 +1947,9 @@ Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Arena *arena, E
   }
   ret = arena_array(arena, 3);
   size_t off = g->line_offset[(size_t)row] + (size_t)col;
-  ADD_C(ret, CSTR_AS_OBJ((char *)g->chars[off]));
+  char *sc_buf = arena_alloc(arena, MAX_SCHAR_SIZE, false);
+  schar_get(sc_buf, g->chars[off]);
+  ADD_C(ret, CSTR_AS_OBJ(sc_buf));
   int attr = g->attrs[off];
   ADD_C(ret, DICTIONARY_OBJ(hl_get_attr_by_id(attr, true, arena, err)));
   // will not work first time
@@ -1963,6 +1965,11 @@ void nvim__screenshot(String path)
   ui_call_screenshot(path);
 }
 
+void nvim__invalidate_glyph_cache(void)
+{
+  schar_cache_clear_force();
+}
+
 Object nvim__unpack(String str, Error *err)
   FUNC_API_FAST
 {
diff --git a/src/nvim/api/win_config.c b/src/nvim/api/win_config.c
index 63cf3bb701..a579b0dde5 100644
--- a/src/nvim/api/win_config.c
+++ b/src/nvim/api/win_config.c
@@ -16,7 +16,7 @@
 #include "nvim/drawscreen.h"
 #include "nvim/extmark_defs.h"
 #include "nvim/globals.h"
-#include "nvim/grid_defs.h"
+#include "nvim/grid.h"
 #include "nvim/highlight_group.h"
 #include "nvim/macros.h"
 #include "nvim/mbyte.h"
@@ -348,7 +348,7 @@ Dictionary nvim_win_get_config(Window window, Error *err)
       for (size_t i = 0; i < 8; i++) {
         Array tuple = ARRAY_DICT_INIT;
 
-        String s = cstrn_to_string(config->border_chars[i], sizeof(schar_T));
+        String s = cstrn_to_string(config->border_chars[i], MAX_SCHAR_SIZE);
 
         int hi_id = config->border_hl_ids[i];
         char *hi_name = syn_id2name(hi_id);
@@ -520,7 +520,7 @@ static void parse_border_style(Object style,  FloatConfig *fconfig, Error *err)
 {
   struct {
     const char *name;
-    schar_T chars[8];
+    char chars[8][MAX_SCHAR_SIZE];
     bool shadow_color;
   } defaults[] = {
     { "double", { "╔", "═", "╗", "║", "╝", "═", "╚", "║" }, false },
@@ -531,7 +531,7 @@ static void parse_border_style(Object style,  FloatConfig *fconfig, Error *err)
     { NULL, { { NUL } }, false },
   };
 
-  schar_T *chars = fconfig->border_chars;
+  char (*chars)[MAX_SCHAR_SIZE] = fconfig->border_chars;
   int *hl_ids = fconfig->border_hl_ids;
 
   fconfig->border = true;
diff --git a/src/nvim/buffer_defs.h b/src/nvim/buffer_defs.h
index 3229cb2fe7..2698af98e2 100644
--- a/src/nvim/buffer_defs.h
+++ b/src/nvim/buffer_defs.h
@@ -954,7 +954,7 @@ typedef struct {
   WinStyle style;
   bool border;
   bool shadow;
-  schar_T border_chars[8];
+  char border_chars[8][MAX_SCHAR_SIZE];
   int border_hl_ids[8];
   int border_attr[8];
   bool title;
diff --git a/src/nvim/drawline.c b/src/nvim/drawline.c
index 811cfc1eb2..8b4786a98e 100644
--- a/src/nvim/drawline.c
+++ b/src/nvim/drawline.c
@@ -106,6 +106,8 @@ typedef struct {
   int c_extra;               ///< extra chars, all the same
   int c_final;               ///< final char, mandatory if set
 
+  int n_closing;             ///< number of chars in fdc which will be closing
+
   bool extra_for_extmark;    ///< n_extra set for inline virtual text
 
   // saved "extra" items for when draw_state becomes WL_LINE (again)
@@ -221,11 +223,11 @@ static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, b
   if (*p == TAB) {
     cells = MIN(tabstop_padding(vcol, buf->b_p_ts, buf->b_p_vts_array), maxcells);
     for (int c = 0; c < cells; c++) {
-      schar_from_ascii(dest[c], ' ');
+      dest[c] = schar_from_ascii(' ');
     }
     goto done;
   } else if ((uint8_t)(*p) < 0x80 && u8cc[0] == 0) {
-    schar_from_ascii(dest[0], *p);
+    dest[0] = schar_from_ascii(*p);
     s->prev_c = u8c;
   } else {
     if (p_arshape && !p_tbidi && ARABIC_CHAR(u8c)) {
@@ -252,10 +254,10 @@ static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, b
     } else {
       s->prev_c = u8c;
     }
-    schar_from_cc(dest[0], u8c, u8cc);
+    dest[0] = schar_from_cc(u8c, u8cc);
   }
   if (cells > 1) {
-    dest[1][0] = 0;
+    dest[1] = 0;
   }
 done:
   s->p += c_len;
@@ -348,17 +350,17 @@ static int draw_virt_text_item(buf_T *buf, int col, VirtText vt, HlMode hl_mode,
                              max_col - col, false, vcol);
     // If we failed to emit a char, we still need to put a space and advance.
     if (cells < 1) {
-      schar_from_ascii(linebuf_char[col], ' ');
+      linebuf_char[col] = schar_from_ascii(' ');
       cells = 1;
     }
     for (int c = 0; c < cells; c++) {
       linebuf_attr[col++] = attr;
     }
-    if (col < max_col && linebuf_char[col][0] == 0) {
+    if (col < max_col && linebuf_char[col] == 0) {
       // If the left half of a double-width char is overwritten,
       // change the right half to a space so that grid redraws properly,
       // but don't advance the current column.
-      schar_from_ascii(linebuf_char[col], ' ');
+      linebuf_char[col] = schar_from_ascii(' ');
     }
     vcol += cells;
   }
@@ -384,7 +386,8 @@ static void handle_foldcolumn(win_T *wp, winlinevars_T *wlv)
   // Allocate a buffer, "wlv->extra[]" may already be in use.
   xfree(wlv->p_extra_free);
   wlv->p_extra_free = xmalloc(MAX_MCO * (size_t)fdc + 1);
-  wlv->n_extra = (int)fill_foldcolumn(wlv->p_extra_free, wp, wlv->foldinfo, wlv->lnum);
+  wlv->n_extra = (int)fill_foldcolumn(wlv->p_extra_free, wp, wlv->foldinfo, wlv->lnum,
+                                      &wlv->n_closing);
   wlv->p_extra_free[wlv->n_extra] = NUL;
   wlv->p_extra = wlv->p_extra_free;
   wlv->c_extra = NUL;
@@ -405,7 +408,7 @@ static void handle_foldcolumn(win_T *wp, winlinevars_T *wlv)
 ///
 /// Assume monocell characters
 /// @return number of chars added to \param p
-size_t fill_foldcolumn(char *p, win_T *wp, foldinfo_T foldinfo, linenr_T lnum)
+size_t fill_foldcolumn(char *p, win_T *wp, foldinfo_T foldinfo, linenr_T lnum, int *n_closing)
 {
   int i = 0;
   int level;
@@ -447,16 +450,23 @@ size_t fill_foldcolumn(char *p, win_T *wp, foldinfo_T foldinfo, linenr_T lnum)
     }
   }
 
+  int n_closing_val = i;
+
   if (closed) {
     if (symbol != 0) {
       // rollback previous write
       char_counter -= (size_t)len;
       memset(&p[char_counter], ' ', (size_t)len);
+      n_closing_val--;
     }
     len = utf_char2bytes(wp->w_p_fcs_chars.foldclosed, &p[char_counter]);
     char_counter += (size_t)len;
   }
 
+  if (n_closing) {
+    *n_closing = n_closing_val;
+  }
+
   return MAX(char_counter + (size_t)(fdc - i), (size_t)fdc);
 }
 
@@ -2737,7 +2747,7 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
           wlv.col += n;
         } else {
           // Add a blank character to highlight.
-          schar_from_ascii(linebuf_char[wlv.off], ' ');
+          linebuf_char[wlv.off] = schar_from_ascii(' ');
         }
         if (area_attr == 0 && !has_fold) {
           // Use attributes from match with highest priority among
@@ -2839,7 +2849,7 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
         int col_stride = wp->w_p_rl ? -1 : 1;
 
         while (wp->w_p_rl ? wlv.col >= 0 : wlv.col < grid->cols) {
-          schar_from_ascii(linebuf_char[wlv.off], ' ');
+          linebuf_char[wlv.off] = schar_from_ascii(' ');
           linebuf_vcol[wlv.off] = MAXCOL;
           wlv.col += col_stride;
           if (draw_color_col) {
@@ -2873,7 +2883,7 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
         // logical line
         int n = wp->w_p_rl ? -1 : 1;
         while (wlv.col >= 0 && wlv.col < grid->cols) {
-          schar_from_ascii(linebuf_char[wlv.off], ' ');
+          linebuf_char[wlv.off] = schar_from_ascii(' ');
           linebuf_attr[wlv.off] = wlv.vcol >= TERM_ATTRS_MAX ? 0 : term_attrs[wlv.vcol];
           linebuf_vcol[wlv.off] = wlv.vcol;
           wlv.off += n;
@@ -2973,9 +2983,9 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
         wlv.col--;
       }
       if (mb_utf8) {
-        schar_from_cc(linebuf_char[wlv.off], mb_c, u8cc);
+        linebuf_char[wlv.off] = schar_from_cc(mb_c, u8cc);
       } else {
-        schar_from_ascii(linebuf_char[wlv.off], (char)c);
+        linebuf_char[wlv.off] = schar_from_ascii((char)c);
       }
       if (multi_attr) {
         linebuf_attr[wlv.off] = multi_attr;
@@ -2986,12 +2996,20 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
 
       linebuf_vcol[wlv.off] = wlv.vcol;
 
+      if (wlv.draw_state == WL_FOLD) {
+        linebuf_vcol[wlv.off] = -2;
+        if (wlv.n_closing > 0) {
+          linebuf_vcol[wlv.off] = -3;
+          wlv.n_closing--;
+        }
+      }
+
       if (utf_char2cells(mb_c) > 1) {
         // Need to fill two screen columns.
         wlv.off++;
         wlv.col++;
         // UTF-8: Put a 0 in the second screen char.
-        linebuf_char[wlv.off][0] = 0;
+        linebuf_char[wlv.off] = 0;
         linebuf_attr[wlv.off] = linebuf_attr[wlv.off - 1];
         if (wlv.draw_state > WL_STC && wlv.filler_todo <= 0) {
           wlv.vcol++;
diff --git a/src/nvim/drawscreen.c b/src/nvim/drawscreen.c
index f71a47a596..e3e9fc8170 100644
--- a/src/nvim/drawscreen.c
+++ b/src/nvim/drawscreen.c
@@ -444,6 +444,15 @@ int update_screen(void)
   display_tick++;  // let syntax code know we're in a next round of
                    // display updating
 
+  // glyph cache full, very rare
+  if (schar_cache_clear_if_full()) {
+    // must use CLEAR, as the contents of screen buffers cannot be
+    // compared to their previous state here.
+    // TODO(bfredl): if start to cache schar_T values in places (like fcs/lcs)
+    // we need to revalidate these here as well!
+    type = MAX(type, UPD_CLEAR);
+  }
+
   // Tricky: vim code can reset msg_scrolled behind our back, so need
   // separate bookkeeping for now.
   if (msg_did_scroll) {
@@ -736,7 +745,10 @@ static void win_redr_border(win_T *wp)
 
   ScreenGrid *grid = &wp->w_grid_alloc;
 
-  schar_T *chars = wp->w_float_config.border_chars;
+  schar_T chars[8];
+  for (int i = 0; i < 8; i++) {
+    chars[i] = schar_from_str(wp->w_float_config.border_chars[i]);
+  }
   int *attrs = wp->w_float_config.border_attr;
 
   int *adj = wp->w_border_adj;
@@ -770,7 +782,7 @@ static void win_redr_border(win_T *wp)
       grid_puts_line_flush(false);
     }
     if (adj[1]) {
-      int ic = (i == 0 && !adj[0] && chars[2][0]) ? 2 : 3;
+      int ic = (i == 0 && !adj[0] && chars[2]) ? 2 : 3;
       grid_puts_line_start(grid, i + adj[0]);
       grid_put_schar(grid, i + adj[0], icol + adj[3], chars[ic], attrs[ic]);
       grid_puts_line_flush(false);
@@ -784,7 +796,7 @@ static void win_redr_border(win_T *wp)
     }
 
     for (int i = 0; i < icol; i++) {
-      int ic = (i == 0 && !adj[3] && chars[6][0]) ? 6 : 5;
+      int ic = (i == 0 && !adj[3] && chars[6]) ? 6 : 5;
       grid_put_schar(grid, irow + adj[0], i + adj[3], chars[ic], attrs[ic]);
     }
 
diff --git a/src/nvim/grid.c b/src/nvim/grid.c
index fa7f270172..cf6cd2f04e 100644
--- a/src/nvim/grid.c
+++ b/src/nvim/grid.c
@@ -17,6 +17,7 @@
 
 #include "nvim/arabic.h"
 #include "nvim/buffer_defs.h"
+#include "nvim/drawscreen.h"
 #include "nvim/globals.h"
 #include "nvim/grid.h"
 #include "nvim/highlight.h"
@@ -36,6 +37,15 @@
 // Per-cell attributes
 static size_t linebuf_size = 0;
 
+// Used to cache glyphs which doesn't fit an a sizeof(schar_T) length UTF-8 string.
+// Then it instead stores an index into glyph_cache.keys[] which is a flat char array.
+// The hash part is used by schar_from_buf() to quickly lookup glyphs which already
+// has been interned. schar_get() should used to convert a schar_T value
+// back to a string buffer.
+//
+// The maximum byte size of a glyph is MAX_SCHAR_SIZE (including the final NUL).
+static Set(glyph) glyph_cache = SET_INIT;
+
 /// Determine if dedicated window grid should be used or the default_grid
 ///
 /// If UI did not request multigrid support, draw all windows on the
@@ -56,25 +66,119 @@ void grid_adjust(ScreenGrid **grid, int *row_off, int *col_off)
 }
 
 /// Put a unicode char, and up to MAX_MCO composing chars, in a screen cell.
-int schar_from_cc(char *p, int c, int u8cc[MAX_MCO])
+schar_T schar_from_cc(int c, int u8cc[MAX_MCO])
 {
-  int len = utf_char2bytes(c, p);
+  char buf[MAX_SCHAR_SIZE];
+  int len = utf_char2bytes(c, buf);
   for (int i = 0; i < MAX_MCO; i++) {
     if (u8cc[i] == 0) {
       break;
     }
-    len += utf_char2bytes(u8cc[i], p + len);
+    len += utf_char2bytes(u8cc[i], buf + len);
+  }
+  buf[len] = 0;
+  return schar_from_buf(buf, (size_t)len);
+}
+
+schar_T schar_from_str(char *str)
+{
+  if (str == NULL) {
+    return 0;
+  }
+  return schar_from_buf(str, strlen(str));
+}
+
+/// @param buf need not be NUL terminated, but may not contain embedded NULs.
+///
+/// caller must ensure len < MAX_SCHAR_SIZE (not =, as NUL needs a byte)
+schar_T schar_from_buf(const char *buf, size_t len)
+{
+  assert(len < MAX_SCHAR_SIZE);
+  if (len <= 4) {
+    schar_T sc = 0;
+    memcpy((char *)&sc, buf, len);
+    return sc;
+  } else {
+    String str = { .data = (char *)buf, .size = len };
+
+    MHPutStatus status;
+    uint32_t idx = set_put_idx(glyph, &glyph_cache, str, &status);
+    assert(idx < 0xFFFFFF);
+#ifdef ORDER_BIG_ENDIAN
+    return idx + ((uint32_t)0xFF << 24);
+#else
+    return 0xFF + (idx << 8);
+#endif
+  }
+}
+
+/// Check if cache is full, and if it is, clear it.
+///
+/// This should normally only be called in update_screen()
+///
+/// @return true if cache was clered, and all your screen buffers now are hosed
+/// and you need to use UPD_CLEAR
+bool schar_cache_clear_if_full(void)
+{
+  // note: critical max is really (1<<24)-1. This gives us some marginal
+  // until next time update_screen() is called
+  if (glyph_cache.h.n_keys > (1<<21)) {
+    set_clear(glyph, &glyph_cache);
+    return true;
+  }
+  return false;
+}
+
+/// For testing. The condition in schar_cache_clear_force is hard to
+/// reach, so this function can be used to force a cache clear in a test.
+void schar_cache_clear_force(void)
+{
+  set_clear(glyph, &glyph_cache);
+  must_redraw = UPD_CLEAR;
+}
+
+bool schar_high(schar_T sc)
+{
+#ifdef ORDER_BIG_ENDIAN
+  return ((sc & 0xFF000000) == 0xFF000000);
+#else
+  return ((sc & 0xFF) == 0xFF);
+#endif
+}
+
+void schar_get(char *buf_out, schar_T sc)
+{
+  if (schar_high(sc)) {
+#ifdef ORDER_BIG_ENDIAN
+    uint32_t idx = sc & (0x00FFFFFF);
+#else
+    uint32_t idx = sc >> 8;
+#endif
+    if (idx >= glyph_cache.h.n_keys) {
+      abort();
+    }
+    xstrlcpy(buf_out, &glyph_cache.keys[idx], 32);
+  } else {
+    memcpy(buf_out, (char *)&sc, 4);
+    buf_out[4] = NUL;
   }
-  p[len] = 0;
-  return len;
 }
 
+/// @return ascii char or NUL if not ascii
+char schar_get_ascii(schar_T sc)
+{
+#ifdef ORDER_BIG_ENDIAN
+  return (!(sc & 0x80FFFFFF)) ? *(char *)&sc : NUL;
+#else
+  return (sc < 0x80) ? (char)sc : NUL;
+#endif
+}
 /// clear a line in the grid starting at "off" until "width" characters
 /// are cleared.
 void grid_clear_line(ScreenGrid *grid, size_t off, int width, bool valid)
 {
   for (int col = 0; col < width; col++) {
-    schar_from_ascii(grid->chars[off + (size_t)col], ' ');
+    grid->chars[off + (size_t)col] = schar_from_ascii(' ');
   }
   int fill = valid ? 0 : -1;
   (void)memset(grid->attrs + off, fill, (size_t)width * sizeof(sattr_T));
@@ -93,7 +197,7 @@ bool grid_invalid_row(ScreenGrid *grid, int row)
 
 static int line_off2cells(schar_T *line, size_t off, size_t max_off)
 {
-  return (off + 1 < max_off && line[off + 1][0] == 0) ? 2 : 1;
+  return (off + 1 < max_off && line[off + 1] == 0) ? 2 : 1;
 }
 
 /// Return number of display cells for char at grid->chars[off].
@@ -124,7 +228,7 @@ int grid_fix_col(ScreenGrid *grid, int col, int row)
 
   col += coloff;
   if (grid->chars != NULL && col > 0
-      && grid->chars[grid->line_offset[row] + (size_t)col][0] == 0) {
+      && grid->chars[grid->line_offset[row] + (size_t)col] == 0) {
     return col - 1 - coloff;
   }
   return col - coloff;
@@ -155,7 +259,7 @@ void grid_getbytes(ScreenGrid *grid, int row, int col, char *bytes, int *attrp)
   if (attrp != NULL) {
     *attrp = grid->attrs[off];
   }
-  schar_copy(bytes, grid->chars[off]);
+  schar_get(bytes, grid->chars[off]);
 }
 
 /// put string '*text' on the window grid at position 'row' and 'col', with
@@ -185,12 +289,12 @@ void grid_puts_line_start(ScreenGrid *grid, int row)
   put_dirty_grid = grid;
 }
 
-void grid_put_schar(ScreenGrid *grid, int row, int col, char *schar, int attr)
+void grid_put_schar(ScreenGrid *grid, int row, int col, schar_T schar, int attr)
 {
   assert(put_dirty_row == row);
   size_t off = grid->line_offset[row] + (size_t)col;
-  if (grid->attrs[off] != attr || schar_cmp(grid->chars[off], schar) || rdb_flags & RDB_NODELTA) {
-    schar_copy(grid->chars[off], schar);
+  if (grid->attrs[off] != attr || grid->chars[off] != schar || rdb_flags & RDB_NODELTA) {
+    grid->chars[off] = schar;
     grid->attrs[off] = attr;
 
     put_dirty_first = MIN(put_dirty_first, col);
@@ -293,10 +397,12 @@ int grid_puts_len(ScreenGrid *grid, const char *text, int textlen, int row, int
     }
 
     schar_T buf;
-    schar_from_cc(buf, u8c, u8cc);
+    // TODO(bfredl): why not just keep the original byte sequence. arabshape is
+    // an edge case, treat it as such..
+    buf = schar_from_cc(u8c, u8cc);
 
-    int need_redraw = schar_cmp(grid->chars[off], buf)
-                      || (mbyte_cells == 2 && grid->chars[off + 1][0] != 0)
+    int need_redraw = grid->chars[off] != buf
+                      || (mbyte_cells == 2 && grid->chars[off + 1] != 0)
                       || grid->attrs[off] != attr
                       || exmode_active
                       || rdb_flags & RDB_NODELTA;
@@ -320,15 +426,15 @@ int grid_puts_len(ScreenGrid *grid, const char *text, int textlen, int row, int
 
       // When at the start of the text and overwriting the right half of a
       // two-cell character in the same grid, truncate that into a '>'.
-      if (ptr == text && col > 0 && grid->chars[off][0] == 0) {
-        schar_from_ascii(grid->chars[off - 1], '>');
+      if (ptr == text && col > 0 && grid->chars[off] == 0) {
+        grid->chars[off - 1] = schar_from_ascii('>');
       }
 
-      schar_copy(grid->chars[off], buf);
+      grid->chars[off] = buf;
       grid->attrs[off] = attr;
       grid->vcols[off] = -1;
       if (mbyte_cells == 2) {
-        grid->chars[off + 1][0] = 0;
+        grid->chars[off + 1] = 0;
         grid->attrs[off + 1] = attr;
         grid->vcols[off + 1] = -1;
       }
@@ -429,12 +535,12 @@ void grid_fill(ScreenGrid *grid, int start_row, int end_row, int start_col, int
     int dirty_last = 0;
 
     int col = start_col;
-    schar_from_char(sc, c1);
+    sc = schar_from_char(c1);
     size_t lineoff = grid->line_offset[row];
     for (col = start_col; col < end_col; col++) {
       size_t off = lineoff + (size_t)col;
-      if (schar_cmp(grid->chars[off], sc) || grid->attrs[off] != attr || rdb_flags & RDB_NODELTA) {
-        schar_copy(grid->chars[off], sc);
+      if (grid->chars[off] != sc || grid->attrs[off] != attr || rdb_flags & RDB_NODELTA) {
+        grid->chars[off] = sc;
         grid->attrs[off] = attr;
         if (dirty_first == INT_MAX) {
           dirty_first = col;
@@ -443,7 +549,7 @@ void grid_fill(ScreenGrid *grid, int start_row, int end_row, int start_col, int
       }
       grid->vcols[off] = -1;
       if (col == start_col) {
-        schar_from_char(sc, c2);
+        sc = schar_from_char(c2);
       }
     }
     if (dirty_last > dirty_first) {
@@ -483,11 +589,10 @@ void grid_fill(ScreenGrid *grid, int start_row, int end_row, int start_col, int
 static int grid_char_needs_redraw(ScreenGrid *grid, size_t off_from, size_t off_to, int cols)
 {
   return (cols > 0
-          && ((schar_cmp(linebuf_char[off_from], grid->chars[off_to])
+          && ((linebuf_char[off_from] != grid->chars[off_to]
                || linebuf_attr[off_from] != grid->attrs[off_to]
                || (line_off2cells(linebuf_char, off_from, off_from + (size_t)cols) > 1
-                   && schar_cmp(linebuf_char[off_from + 1],
-                                grid->chars[off_to + 1])))
+                   && linebuf_char[off_from + 1] != grid->chars[off_to + 1]))
               || rdb_flags & RDB_NODELTA));
 }
 
@@ -544,7 +649,7 @@ void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int endcol, int cle
     if (wp->w_p_nu && wp->w_p_rnu) {
       // do not overwrite the line number, change "123 text" to
       // "123<<<xt".
-      while (skip < max_off_from && ascii_isdigit(*linebuf_char[off])) {
+      while (skip < max_off_from && ascii_isdigit(schar_get_ascii(linebuf_char[off]))) {
         off++;
         skip++;
       }
@@ -554,9 +659,9 @@ void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int endcol, int cle
       if (line_off2cells(linebuf_char, off, max_off_from) > 1) {
         // When the first half of a double-width character is
         // overwritten, change the second half to a space.
-        schar_from_ascii(linebuf_char[off + 1], ' ');
+        linebuf_char[off + 1] = schar_from_ascii(' ');
       }
-      schar_from_ascii(linebuf_char[off], '<');
+      linebuf_char[off] = schar_from_ascii('<');
       linebuf_attr[off] = HL_ATTR(HLF_AT);
       off++;
     }
@@ -565,8 +670,7 @@ void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int endcol, int cle
   if (rlflag) {
     // Clear rest first, because it's left of the text.
     if (clear_width > 0) {
-      while (col <= endcol && grid->chars[off_to][0] == ' '
-             && grid->chars[off_to][1] == NUL
+      while (col <= endcol && grid->chars[off_to] == schar_from_ascii(' ')
              && grid->attrs[off_to] == bg_attr) {
         off_to++;
         col++;
@@ -619,9 +723,9 @@ void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int endcol, int cle
         clear_next = true;
       }
 
-      schar_copy(grid->chars[off_to], linebuf_char[off_from]);
+      grid->chars[off_to] = linebuf_char[off_from];
       if (char_cells == 2) {
-        schar_copy(grid->chars[off_to + 1], linebuf_char[off_from + 1]);
+        grid->chars[off_to + 1] = linebuf_char[off_from + 1];
       }
 
       grid->attrs[off_to] = linebuf_attr[off_from];
@@ -645,7 +749,7 @@ void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int endcol, int cle
   if (clear_next) {
     // Clear the second half of a double-wide character of which the left
     // half was overwritten with a single-wide character.
-    schar_from_ascii(grid->chars[off_to], ' ');
+    grid->chars[off_to] = schar_from_ascii(' ');
     end_dirty++;
   }
 
@@ -654,12 +758,10 @@ void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int endcol, int cle
     // blank out the rest of the line
     // TODO(bfredl): we could cache winline widths
     while (col < clear_width) {
-      if (grid->chars[off_to][0] != ' '
-          || grid->chars[off_to][1] != NUL
+      if (grid->chars[off_to] != schar_from_ascii(' ')
           || grid->attrs[off_to] != bg_attr
           || rdb_flags & RDB_NODELTA) {
-        grid->chars[off_to][0] = ' ';
-        grid->chars[off_to][1] = NUL;
+        grid->chars[off_to] = schar_from_ascii(' ');
         grid->attrs[off_to] = bg_attr;
         if (start_dirty == -1) {
           start_dirty = col;
diff --git a/src/nvim/grid.h b/src/nvim/grid.h
index 0db97345d1..9cdc6c6677 100644
--- a/src/nvim/grid.h
+++ b/src/nvim/grid.h
@@ -33,30 +33,25 @@ EXTERN colnr_T *linebuf_vcol INIT(= NULL);
 // screen grid.
 
 /// Put a ASCII character in a screen cell.
-static inline void schar_from_ascii(char *p, const char c)
-{
-  p[0] = c;
-  p[1] = 0;
-}
+///
+/// If `x` is a compile time constant, schar_from_ascii(x) will also be.
+/// But the specific value varies per plattform.
+#ifdef ORDER_BIG_ENDIAN
+# define schar_from_ascii(x) ((schar_T)((x) << 24))
+#else
+# define schar_from_ascii(x) ((schar_T)(x))
+#endif
 
 /// Put a unicode character in a screen cell.
-static inline int schar_from_char(char *p, int c)
-{
-  int len = utf_char2bytes(c, p);
-  p[len] = NUL;
-  return len;
-}
-
-/// compare the contents of two screen cells.
-static inline int schar_cmp(char *sc1, char *sc2)
-{
-  return strncmp(sc1, sc2, sizeof(schar_T));
-}
-
-/// copy the contents of screen cell `sc2` into cell `sc1`
-static inline void schar_copy(char *sc1, char *sc2)
+static inline schar_T schar_from_char(int c)
 {
-  xstrlcpy(sc1, sc2, sizeof(schar_T));
+  schar_T sc = 0;
+  if (c >= 0x200000) {
+    // TODO(bfredl): this must NEVER happen, even if the file contained overlong sequences
+    c = 0xFFFD;
+  }
+  utf_char2bytes(c, (char *)&sc);
+  return sc;
 }
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
diff --git a/src/nvim/grid_defs.h b/src/nvim/grid_defs.h
index 4ad7d4cdb4..b0d1dafd29 100644
--- a/src/nvim/grid_defs.h
+++ b/src/nvim/grid_defs.h
@@ -9,9 +9,13 @@
 #include "nvim/types.h"
 
 #define MAX_MCO  6  // fixed value for 'maxcombine'
+// Includes final NUL. at least 4*(MAX_MCO+1)+1
+#define MAX_SCHAR_SIZE 32
 
-// The characters and attributes drawn on grids.
-typedef char schar_T[(MAX_MCO + 1) * 4 + 1];
+// if data[0] is 0xFF, then data[1..4] is a 24-bit index (in machine endianess)
+// otherwise it must be a UTF-8 string of length maximum 4 (no NUL when n=4)
+
+typedef uint32_t schar_T;
 typedef int sattr_T;
 
 enum {
@@ -41,6 +45,9 @@ enum {
 ///
 /// vcols[] contains the virtual columns in the line. -1 means not available
 /// (below last line), MAXCOL means after the end of the line.
+/// -2 or -3 means in fold column and a mouse click should:
+///  -2: open a fold
+///  -3: close a fold
 ///
 /// line_offset[n] is the offset from chars[], attrs[] and vcols[] for the start
 /// of line 'n'. These offsets are in general not linear, as full screen scrolling
diff --git a/src/nvim/highlight.c b/src/nvim/highlight.c
index df4bffdac3..3728db31d8 100644
--- a/src/nvim/highlight.c
+++ b/