ui/tui: highlighting refactor

Make HlAttr contain highlighting state for both color modes (cterm and rgb).
This allows us to implement termguicolors completely in the TUI.

Simplify some logic duplicated between ui.c and screen.c. Also avoid
some superfluous highlighting reset events.

1 files changed, 60 insertions, 53 deletions

diff --git a/src/nvim/tui/tui.c b/src/nvim/tui/tui.c
index 70e19e1d93..62bdc61a68 100644
--- a/src/nvim/tui/tui.c
+++ b/src/nvim/tui/tui.c
@@ -125,7 +125,6 @@ UI *tui_start(void)
 {
   UI *ui = xcalloc(1, sizeof(UI));  // Freed by ui_bridge_stop().
   ui->stop = tui_stop;
-  ui->rgb = p_tgc;
   ui->resize = tui_resize;
   ui->clear = tui_clear;
   ui->eol_clear = tui_eol_clear;
@@ -143,15 +142,12 @@ UI *tui_start(void)
   ui->put = tui_put;
   ui->bell = tui_bell;
   ui->visual_bell = tui_visual_bell;
-  ui->update_fg = tui_update_fg;
-  ui->update_bg = tui_update_bg;
-  ui->update_sp = tui_update_sp;
+  ui->default_colors_set = tui_default_colors_set;
   ui->flush = tui_flush;
   ui->suspend = tui_suspend;
   ui->set_title = tui_set_title;
   ui->set_icon = tui_set_icon;
   ui->option_set= tui_option_set;
-  ui->event = tui_event;
 
   memset(ui->ui_ext, 0, sizeof(ui->ui_ext));
 
@@ -410,36 +406,58 @@ static void sigwinch_cb(SignalWatcher *watcher, int signum, void *data)
   ui_schedule_refresh();
 }
 
-static bool attrs_differ(HlAttrs a1, HlAttrs a2)
+static bool attrs_differ(HlAttrs a1, HlAttrs a2, bool rgb)
 {
-  return a1.foreground != a2.foreground || a1.background != a2.background
-    || a1.bold != a2.bold || a1.italic != a2.italic
-    || a1.undercurl != a2.undercurl || a1.underline != a2.underline
-    || a1.reverse != a2.reverse;
+  if (rgb) {
+    // TODO(bfredl): when we start to support special color,
+    // rgb_sp_color must be added here
+    return a1.rgb_fg_color != a2.rgb_fg_color
+      || a1.rgb_bg_color != a2.rgb_bg_color
+      || a1.rgb_ae_attr != a2.rgb_ae_attr;
+  } else {
+    return a1.cterm_fg_color != a2.cterm_fg_color
+      || a1.cterm_bg_color != a2.cterm_bg_color
+      || a1.cterm_ae_attr != a2.cterm_ae_attr;
+  }
 }
 
 static void update_attrs(UI *ui, HlAttrs attrs)
 {
   TUIData *data = ui->data;
 
-  if (!attrs_differ(attrs, data->print_attrs)) {
+  if (!attrs_differ(attrs, data->print_attrs, ui->rgb)) {
     return;
   }
 
   data->print_attrs = attrs;
   UGrid *grid = &data->grid;
 
-  int fg = attrs.foreground != -1 ? attrs.foreground : grid->fg;
-  int bg = attrs.background != -1 ? attrs.background : grid->bg;
+  int fg = ui->rgb ? attrs.rgb_fg_color : (attrs.cterm_fg_color - 1);
+  if (fg == -1) {
+    fg = ui->rgb ? grid->clear_attrs.rgb_fg_color
+                 : (grid->clear_attrs.cterm_fg_color - 1);
+  }
+
+  int bg = ui->rgb ? attrs.rgb_bg_color : (attrs.cterm_bg_color - 1);
+  if (bg == -1) {
+    bg = ui->rgb ? grid->clear_attrs.rgb_bg_color
+                 : (grid->clear_attrs.cterm_bg_color - 1);
+  }
+
+  int attr = ui->rgb ? attrs.rgb_ae_attr : attrs.cterm_ae_attr;
+  bool bold = attr & HL_BOLD;
+  bool italic = attr & HL_ITALIC;
+  bool reverse = attr & (HL_INVERSE | HL_STANDOUT);
+  bool underline = attr & (HL_UNDERLINE), undercurl = attr & (HL_UNDERCURL);
 
   if (unibi_get_str(data->ut, unibi_set_attributes)) {
-    if (attrs.bold || attrs.reverse || attrs.underline || attrs.undercurl) {
+    if (bold || reverse || underline || undercurl) {
       UNIBI_SET_NUM_VAR(data->params[0], 0);   // standout
-      UNIBI_SET_NUM_VAR(data->params[1], attrs.underline || attrs.undercurl);
-      UNIBI_SET_NUM_VAR(data->params[2], attrs.reverse);
+      UNIBI_SET_NUM_VAR(data->params[1], underline || undercurl);
+      UNIBI_SET_NUM_VAR(data->params[2], reverse);
       UNIBI_SET_NUM_VAR(data->params[3], 0);   // blink
       UNIBI_SET_NUM_VAR(data->params[4], 0);   // dim
-      UNIBI_SET_NUM_VAR(data->params[5], attrs.bold);
+      UNIBI_SET_NUM_VAR(data->params[5], bold);
       UNIBI_SET_NUM_VAR(data->params[6], 0);   // blank
       UNIBI_SET_NUM_VAR(data->params[7], 0);   // protect
       UNIBI_SET_NUM_VAR(data->params[8], 0);   // alternate character set
@@ -451,17 +469,17 @@ static void update_attrs(UI *ui, HlAttrs attrs)
     if (!data->default_attr) {
       unibi_out(ui, unibi_exit_attribute_mode);
     }
-    if (attrs.bold) {
+    if (bold) {
       unibi_out(ui, unibi_enter_bold_mode);
     }
-    if (attrs.underline || attrs.undercurl) {
+    if (underline || undercurl) {
       unibi_out(ui, unibi_enter_underline_mode);
     }
-    if (attrs.reverse) {
+    if (reverse) {
       unibi_out(ui, unibi_enter_reverse_mode);
     }
   }
-  if (attrs.italic) {
+  if (italic) {
     unibi_out(ui, unibi_enter_italics_mode);
   }
   if (ui->rgb) {
@@ -491,8 +509,7 @@ static void update_attrs(UI *ui, HlAttrs attrs)
   }
 
   data->default_attr = fg == -1 && bg == -1
-    && !attrs.bold && !attrs.italic && !attrs.underline && !attrs.undercurl
-    && !attrs.reverse;
+    && !bold && !italic && !underline && !undercurl && !reverse;
 }
 
 static void final_column_wrap(UI *ui)
@@ -534,7 +551,7 @@ static bool cheap_to_print(UI *ui, int row, int col, int next)
   UCell *cell = grid->cells[row] + col;
   while (next) {
     next--;
-    if (attrs_differ(cell->attrs, data->print_attrs)) {
+    if (attrs_differ(cell->attrs, data->print_attrs, ui->rgb)) {
       if (data->default_attr) {
         return false;
       }
@@ -659,13 +676,12 @@ static void clear_region(UI *ui, int top, int bot, int left, int right)
   int saved_col = grid->col;
 
   bool cleared = false;
-  if (grid->bg == -1 && right == ui->width -1) {
+  bool nobg = ui->rgb ? grid->clear_attrs.rgb_bg_color == -1
+                      : grid->clear_attrs.cterm_bg_color == 0;
+  if (nobg && right == ui->width -1) {
     // Background is set to the default color and the right edge matches the
     // screen end, try to use terminal codes for clearing the requested area.
-    HlAttrs clear_attrs = HLATTRS_INIT;
-    clear_attrs.foreground = grid->fg;
-    clear_attrs.background = grid->bg;
-    update_attrs(ui, clear_attrs);
+    update_attrs(ui, grid->clear_attrs);
     if (left == 0) {
       if (bot == ui->height - 1) {
         if (top == 0) {
@@ -788,6 +804,7 @@ static void tui_clear(UI *ui)
   TUIData *data = ui->data;
   UGrid *grid = &data->grid;
   ugrid_clear(grid);
+  kv_size(data->invalid_regions) = 0;
   clear_region(ui, grid->top, grid->bot, grid->left, grid->right);
 }
 
@@ -905,7 +922,7 @@ static void tui_set_mode(UI *ui, ModeShape mode)
   if (c.id != 0 && ui->rgb) {
     int attr = syn_id2attr(c.id);
     if (attr > 0) {
-      attrentry_T *aep = syn_cterm_attr2entry(attr);
+      HlAttrs *aep = syn_cterm_attr2entry(attr);
       UNIBI_SET_NUM_VAR(data->params[0], aep->rgb_bg_color);
       unibi_out_ext(ui, data->unibi_ext.set_cursor_color);
     }
@@ -960,10 +977,7 @@ static void tui_scroll(UI *ui, Integer count)
     cursor_goto(ui, grid->top, grid->left);
     // also set default color attributes or some terminals can become funny
     if (scroll_clears_to_current_colour) {
-      HlAttrs clear_attrs = HLATTRS_INIT;
-      clear_attrs.foreground = grid->fg;
-      clear_attrs.background = grid->bg;
-      update_attrs(ui, clear_attrs);
+      update_attrs(ui, grid->clear_attrs);
     }
 
     if (count > 0) {
@@ -1028,19 +1042,16 @@ static void tui_visual_bell(UI *ui)
   unibi_out(ui, unibi_flash_screen);
 }
 
-static void tui_update_fg(UI *ui, Integer fg)
-{
-  ((TUIData *)ui->data)->grid.fg = (int)fg;
-}
-
-static void tui_update_bg(UI *ui, Integer bg)
-{
-  ((TUIData *)ui->data)->grid.bg = (int)bg;
-}
-
-static void tui_update_sp(UI *ui, Integer sp)
+static void tui_default_colors_set(UI *ui, Integer rgb_fg, Integer rgb_bg,
+                                   Integer rgb_sp,
+                                   Integer cterm_fg, Integer cterm_bg)
 {
-  // Do nothing; 'special' color is for GUI only
+  UGrid *grid = &((TUIData *)ui->data)->grid;
+  grid->clear_attrs.rgb_fg_color = (int)rgb_fg;
+  grid->clear_attrs.rgb_bg_color = (int)rgb_bg;
+  grid->clear_attrs.rgb_sp_color = (int)rgb_sp;
+  grid->clear_attrs.cterm_fg_color = (int)cterm_fg;
+  grid->clear_attrs.cterm_bg_color = (int)cterm_bg;
 }
 
 static void tui_flush(UI *ui)
@@ -1065,6 +1076,7 @@ static void tui_flush(UI *ui)
 
   while (kv_size(data->invalid_regions)) {
     Rect r = kv_pop(data->invalid_regions);
+    assert(r.bot < grid->height && r.right < grid->width);
     UGRID_FOREACH_CELL(grid, r.top, r.bot, r.left, r.right, {
       cursor_goto(ui, row, col);
       print_cell(ui, cell);
@@ -1149,18 +1161,13 @@ static void tui_set_icon(UI *ui, String icon)
 
 static void tui_option_set(UI *ui, String name, Object value)
 {
+  TUIData *data = ui->data;
   if (strequal(name.data, "termguicolors")) {
-    // NB: value for bridge is set in ui_bridge.c
     ui->rgb = value.data.boolean;
+    invalidate(ui, 0, data->grid.height-1, 0, data->grid.width-1);
   }
 }
 
-// NB: if we start to use this, the ui_bridge must be updated
-// to make a copy for the tui thread
-static void tui_event(UI *ui, char *name, Array args, bool *args_consumed)
-{
-}
-
 static void invalidate(UI *ui, int top, int bot, int left, int right)
 {
   TUIData *data = ui->data;