refactor(highlight): reshape the HL_UNDER* bits into a 3-bit integer mask

Saves two bits for reuse for new features

4 files changed, 61 insertions, 59 deletions

diff --git a/src/nvim/highlight.c b/src/nvim/highlight.c
index c3a259c87d..e3fb409f34 100644
--- a/src/nvim/highlight.c
+++ b/src/nvim/highlight.c
@@ -649,7 +649,7 @@ int hl_blend_attrs(int back_attr, int front_attr, bool *through)
     cattrs = battrs;
     cattrs.rgb_fg_color = rgb_blend(ratio, battrs.rgb_fg_color,
                                     fattrs.rgb_bg_color);
-    if (cattrs.rgb_ae_attr & (HL_ANY_UNDERLINE)) {
+    if (cattrs.rgb_ae_attr & (HL_UNDERLINE_MASK)) {
       cattrs.rgb_sp_color = rgb_blend(ratio, battrs.rgb_sp_color,
                                       fattrs.rgb_bg_color);
     } else {
@@ -667,7 +667,7 @@ int hl_blend_attrs(int back_attr, int front_attr, bool *through)
     }
     cattrs.rgb_fg_color = rgb_blend(ratio/2, battrs.rgb_fg_color,
                                     fattrs.rgb_fg_color);
-    if (cattrs.rgb_ae_attr & (HL_ANY_UNDERLINE)) {
+    if (cattrs.rgb_ae_attr & (HL_UNDERLINE_MASK)) {
       cattrs.rgb_sp_color = rgb_blend(ratio/2, battrs.rgb_bg_color,
                                       fattrs.rgb_sp_color);
     } else {
@@ -825,40 +825,42 @@ void hlattrs2dict(Dictionary *dict, HlAttrs ae, bool use_rgb)
   Dictionary hl = *dict;
   int mask  = use_rgb ? ae.rgb_ae_attr : ae.cterm_ae_attr;
 
+  if (mask & HL_INVERSE) {
+    PUT_C(hl, "reverse", BOOLEAN_OBJ(true));
+  }
+
   if (mask & HL_BOLD) {
     PUT_C(hl, "bold", BOOLEAN_OBJ(true));
   }
 
-  if (mask & HL_STANDOUT) {
-    PUT_C(hl, "standout", BOOLEAN_OBJ(true));
+  if (mask & HL_ITALIC) {
+    PUT_C(hl, "italic", BOOLEAN_OBJ(true));
   }
 
-  if (mask & HL_UNDERLINE) {
+  switch (mask & HL_UNDERLINE_MASK) {
+  case HL_UNDERLINE:
     PUT_C(hl, "underline", BOOLEAN_OBJ(true));
-  }
-
-  if (mask & HL_UNDERCURL) {
-    PUT_C(hl, "undercurl", BOOLEAN_OBJ(true));
-  }
+    break;
 
-  if (mask & HL_UNDERDOUBLE) {
+  case HL_UNDERDOUBLE:
     PUT_C(hl, "underdouble", BOOLEAN_OBJ(true));
-  }
+    break;
 
-  if (mask & HL_UNDERDOTTED) {
+  case HL_UNDERCURL:
+    PUT_C(hl, "undercurl", BOOLEAN_OBJ(true));
+    break;
+
+  case HL_UNDERDOTTED:
     PUT_C(hl, "underdotted", BOOLEAN_OBJ(true));
-  }
+    break;
 
-  if (mask & HL_UNDERDASHED) {
+  case HL_UNDERDASHED:
     PUT_C(hl, "underdashed", BOOLEAN_OBJ(true));
+    break;
   }
 
-  if (mask & HL_ITALIC) {
-    PUT_C(hl, "italic", BOOLEAN_OBJ(true));
-  }
-
-  if (mask & HL_INVERSE) {
-    PUT_C(hl, "reverse", BOOLEAN_OBJ(true));
+  if (mask & HL_STANDOUT) {
+    PUT_C(hl, "standout", BOOLEAN_OBJ(true));
   }
 
   if (mask & HL_STRIKETHROUGH) {
@@ -920,15 +922,15 @@ HlAttrs dict2hlattrs(Dict(highlight) *dict, bool use_rgb, int *link_id, Error *e
     m = m | flag; \
   }
 
+  CHECK_FLAG(dict, mask, reverse, , HL_INVERSE);
   CHECK_FLAG(dict, mask, bold, , HL_BOLD);
-  CHECK_FLAG(dict, mask, standout, , HL_STANDOUT);
+  CHECK_FLAG(dict, mask, italic, , HL_ITALIC);
   CHECK_FLAG(dict, mask, underline, , HL_UNDERLINE);
-  CHECK_FLAG(dict, mask, undercurl, , HL_UNDERCURL);
   CHECK_FLAG(dict, mask, underdouble, , HL_UNDERDOUBLE);
+  CHECK_FLAG(dict, mask, undercurl, , HL_UNDERCURL);
   CHECK_FLAG(dict, mask, underdotted, , HL_UNDERDOTTED);
   CHECK_FLAG(dict, mask, underdashed, , HL_UNDERDASHED);
-  CHECK_FLAG(dict, mask, italic, , HL_ITALIC);
-  CHECK_FLAG(dict, mask, reverse, , HL_INVERSE);
+  CHECK_FLAG(dict, mask, standout, , HL_STANDOUT);
   CHECK_FLAG(dict, mask, strikethrough, , HL_STRIKETHROUGH);
   if (use_rgb) {
     CHECK_FLAG(dict, mask, fg_indexed, , HL_FG_INDEXED);
@@ -1005,12 +1007,12 @@ HlAttrs dict2hlattrs(Dict(highlight) *dict, bool use_rgb, int *link_id, Error *e
     }
 
     cterm_mask_provided = true;
+    CHECK_FLAG(cterm, cterm_mask, reverse, , HL_INVERSE);
     CHECK_FLAG(cterm, cterm_mask, bold, , HL_BOLD);
-    CHECK_FLAG(cterm, cterm_mask, standout, , HL_STANDOUT);
+    CHECK_FLAG(cterm, cterm_mask, italic, , HL_ITALIC);
     CHECK_FLAG(cterm, cterm_mask, underline, , HL_UNDERLINE);
     CHECK_FLAG(cterm, cterm_mask, undercurl, , HL_UNDERCURL);
-    CHECK_FLAG(cterm, cterm_mask, italic, , HL_ITALIC);
-    CHECK_FLAG(cterm, cterm_mask, reverse, , HL_INVERSE);
+    CHECK_FLAG(cterm, cterm_mask, standout, , HL_STANDOUT);
     CHECK_FLAG(cterm, cterm_mask, strikethrough, , HL_STRIKETHROUGH);
     CHECK_FLAG(cterm, cterm_mask, nocombine, , HL_NOCOMBINE);
   } else if (dict->cterm.type == kObjectTypeArray && dict->cterm.data.array.size == 0) {
diff --git a/src/nvim/highlight_defs.h b/src/nvim/highlight_defs.h
index 2557a248c3..1e109a0316 100644
--- a/src/nvim/highlight_defs.h
+++ b/src/nvim/highlight_defs.h
@@ -15,19 +15,22 @@ typedef enum {
   HL_INVERSE         = 0x01,
   HL_BOLD            = 0x02,
   HL_ITALIC          = 0x04,
+  // The next three bits are all underline styles
+  HL_UNDERLINE_MASK  = 0x38,
   HL_UNDERLINE       = 0x08,
-  HL_UNDERCURL       = 0x10,
-  HL_UNDERDOUBLE     = 0x20,
-  HL_UNDERDOTTED     = 0x40,
-  HL_UNDERDASHED     = 0x80,
-  HL_STANDOUT      = 0x0100,
-  HL_NOCOMBINE     = 0x0200,
-  HL_STRIKETHROUGH = 0x0400,
+  HL_UNDERDOUBLE     = 0x10,
+  HL_UNDERCURL       = 0x18,
+  HL_UNDERDOTTED     = 0x20,
+  HL_UNDERDASHED     = 0x28,
+                       // 0x30 and 0x38 spare for underline styles
+  HL_STANDOUT      = 0x0040,
+  HL_STRIKETHROUGH = 0x0080,
+  // 0x0100-0x0200 spare
+  HL_NOCOMBINE     = 0x0400,
   HL_BG_INDEXED    = 0x0800,
   HL_FG_INDEXED    = 0x1000,
   HL_DEFAULT       = 0x2000,
   HL_GLOBAL        = 0x4000,
-  HL_ANY_UNDERLINE = HL_UNDERLINE | HL_UNDERDOUBLE | HL_UNDERCURL | HL_UNDERDOTTED | HL_UNDERDASHED,
 } HlAttrFlags;
 
 /// Stores a complete highlighting entry, including colors and attributes
diff --git a/src/nvim/tui/tui.c b/src/nvim/tui/tui.c
index 44b99f6c84..3b1cc1e420 100644
--- a/src/nvim/tui/tui.c
+++ b/src/nvim/tui/tui.c
@@ -511,7 +511,7 @@ static bool attrs_differ(TUIData *tui, int id1, int id2, bool rgb)
     return a1.cterm_fg_color != a2.cterm_fg_color
            || a1.cterm_bg_color != a2.cterm_bg_color
            || a1.cterm_ae_attr != a2.cterm_ae_attr
-           || (a1.cterm_ae_attr & HL_ANY_UNDERLINE
+           || (a1.cterm_ae_attr & HL_UNDERLINE_MASK
                && a1.rgb_sp_color != a2.rgb_sp_color);
   }
 }
@@ -538,13 +538,14 @@ static void update_attrs(TUIData *tui, int attr_id)
   bool underdotted;
   bool underdashed;
   if (tui->unibi_ext.set_underline_style != -1) {
-    underline = attr & HL_UNDERLINE;
-    undercurl = attr & HL_UNDERCURL;
-    underdouble = attr & HL_UNDERDOUBLE;
-    underdashed = attr & HL_UNDERDASHED;
-    underdotted = attr & HL_UNDERDOTTED;
+    int ul = attr & HL_UNDERLINE_MASK;
+    underline = ul == HL_UNDERLINE;
+    undercurl = ul == HL_UNDERCURL;
+    underdouble = ul == HL_UNDERDOUBLE;
+    underdashed = ul == HL_UNDERDASHED;
+    underdotted = ul == HL_UNDERDOTTED;
   } else {
-    underline = attr & HL_ANY_UNDERLINE;
+    underline = attr & HL_UNDERLINE_MASK;
     undercurl = false;
     underdouble = false;
     underdotted = false;
diff --git a/test/functional/api/highlight_spec.lua b/test/functional/api/highlight_spec.lua
index 541ac0ab63..8510eb6094 100644
--- a/test/functional/api/highlight_spec.lua
+++ b/test/functional/api/highlight_spec.lua
@@ -32,13 +32,15 @@ describe('API: highlight',function()
     italic = true,
     reverse = true,
     underline = true,
-    undercurl = true,
-    underdouble = true,
-    underdotted = true,
-    underdashed = true,
     strikethrough = true,
     nocombine = true,
   }
+  local expected_undercurl = {
+    background = Screen.colors.Yellow,
+    foreground = Screen.colors.Red,
+    special = Screen.colors.Blue,
+    undercurl = true,
+  }
 
   before_each(function()
     clear()
@@ -59,9 +61,13 @@ describe('API: highlight',function()
     eq('Invalid highlight id: 30000', string.match(emsg, 'Invalid.*'))
 
     -- Test all highlight properties.
-    command('hi NewHighlight gui=underline,bold,undercurl,underdouble,underdotted,underdashed,italic,reverse,strikethrough,nocombine')
+    command('hi NewHighlight gui=underline,bold,italic,reverse,strikethrough,nocombine')
     eq(expected_rgb2, nvim("get_hl_by_id", hl_id, true))
 
+    -- Test undercurl
+    command('hi NewHighlight gui=undercurl')
+    eq(expected_undercurl, nvim("get_hl_by_id", hl_id, true))
+
     -- Test nil argument.
     err, emsg = pcall(meths.get_hl_by_id, { nil }, false)
     eq(false, err)
@@ -207,16 +213,11 @@ describe("API: set highlight", function()
     bold = true,
     italic = true,
     reverse = true,
-    undercurl = true,
     underline = true,
-    underdashed = true,
-    underdotted = true,
-    underdouble = true,
     strikethrough = true,
     cterm = {
       italic = true,
       reverse = true,
-      undercurl = true,
       strikethrough = true,
       nocombine = true,
     }
@@ -227,11 +228,7 @@ describe("API: set highlight", function()
     bold = true,
     italic = true,
     reverse = true,
-    undercurl = true,
     underline = true,
-    underdashed = true,
-    underdotted = true,
-    underdouble = true,
     strikethrough = true,
   }
   local highlight3_result_cterm = {
@@ -239,7 +236,6 @@ describe("API: set highlight", function()
     foreground = highlight_color.ctermfg,
     italic = true,
     reverse = true,
-    undercurl = true,
     strikethrough = true,
     nocombine = true,
   }
@@ -296,7 +292,7 @@ describe("API: set highlight", function()
       exec_capture('highlight Test_hl'))
 
     meths.set_hl(0, 'Test_hl2', highlight3_config)
-    eq('Test_hl2       xxx cterm=undercurl,italic,reverse,strikethrough,nocombine ctermfg=8 ctermbg=15 gui=bold,underline,undercurl,underdouble,underdotted,underdashed,italic,reverse,strikethrough guifg=#ff0000 guibg=#0032aa',
+    eq('Test_hl2       xxx cterm=italic,reverse,strikethrough,nocombine ctermfg=8 ctermbg=15 gui=bold,underline,italic,reverse,strikethrough guifg=#ff0000 guibg=#0032aa',
       exec_capture('highlight Test_hl2'))
 
     -- Colors are stored with the name they are defined, but