tui: support rgba background detection (#10205)

Fixes https://github.com/neovim/neovim/issues/10159.

2 files changed, 107 insertions, 81 deletions

diff --git a/src/nvim/tui/input.c b/src/nvim/tui/input.c
index b7072768ec..fe8ffee8e0 100644
--- a/src/nvim/tui/input.c
+++ b/src/nvim/tui/input.c
@@ -388,17 +388,20 @@ static void set_bg_deferred(void **argv)
 // During startup, tui.c requests the background color (see `ext.get_bg`).
 //
 // Here in input.c, we watch for the terminal response `\e]11;COLOR\a`.  If
-// COLOR matches `rgb:RRRR/GGGG/BBBB` where R, G, and B are hex digits, then
-// compute the luminance[1] of the RGB color and classify it as light/dark
+// COLOR matches `rgb:RRRR/GGGG/BBBB/AAAA` where R, G, B, and A are hex digits,
+// then compute the luminance[1] of the RGB color and classify it as light/dark
 // accordingly. Note that the color components may have anywhere from one to
 // four hex digits, and require scaling accordingly as values out of 4, 8, 12,
-// or 16 bits.
+// or 16 bits. Also note the A(lpha) component is optional, and is parsed but
+// ignored in the calculations.
 //
 // [1] https://en.wikipedia.org/wiki/Luma_%28video%29
 static bool handle_background_color(TermInput *input)
 {
   size_t count = 0;
   size_t component = 0;
+  size_t header_size = 0;
+  size_t num_components = 0;
   uint16_t rgb[] = { 0, 0, 0 };
   uint16_t rgb_max[] = { 0, 0, 0 };
   bool eat_backslash = false;
@@ -406,48 +409,54 @@ static bool handle_background_color(TermInput *input)
   bool bad = false;
   if (rbuffer_size(input->read_stream.buffer) >= 9
       && !rbuffer_cmp(input->read_stream.buffer, "\x1b]11;rgb:", 9)) {
-    rbuffer_consumed(input->read_stream.buffer, 9);
-    RBUFFER_EACH(input->read_stream.buffer, c, i) {
-      count = i + 1;
-      if (eat_backslash) {
-        done = true;
-        break;
-      } else if (c == '\x07') {
-        done = true;
-        break;
-      } else if (c == '\x1b') {
-        eat_backslash = true;
-      } else if (bad) {
-        // ignore
-      } else if (c == '/') {
-        if (component < 3) {
-          component++;
-        }
-      } else if (ascii_isxdigit(c)) {
-        if (component < 3 && rgb_max[component] != 0xffff) {
-          rgb_max[component] = (uint16_t)((rgb_max[component] << 4) | 0xf);
-          rgb[component] = (uint16_t)((rgb[component] << 4) | hex2nr(c));
-        }
-      } else {
-        bad = true;
+    header_size = 9;
+    num_components = 3;
+  } else if (rbuffer_size(input->read_stream.buffer) >= 10
+             && !rbuffer_cmp(input->read_stream.buffer, "\x1b]11;rgba:", 10)) {
+    header_size = 10;
+    num_components = 4;
+  } else {
+    return false;
+  }
+  rbuffer_consumed(input->read_stream.buffer, header_size);
+  RBUFFER_EACH(input->read_stream.buffer, c, i) {
+    count = i + 1;
+    if (eat_backslash) {
+      done = true;
+      break;
+    } else if (c == '\x07') {
+      done = true;
+      break;
+    } else if (c == '\x1b') {
+      eat_backslash = true;
+    } else if (bad) {
+      // ignore
+    } else if ((c == '/') && (++component < num_components)) {
+      // work done in condition
+    } else if (ascii_isxdigit(c)) {
+      if (component < 3 && rgb_max[component] != 0xffff) {
+        rgb_max[component] = (uint16_t)((rgb_max[component] << 4) | 0xf);
+        rgb[component] = (uint16_t)((rgb[component] << 4) | hex2nr(c));
       }
-    }
-    rbuffer_consumed(input->read_stream.buffer, count);
-    if (done && !bad && rgb_max[0] && rgb_max[1] && rgb_max[2]) {
-      double r = (double)rgb[0] / (double)rgb_max[0];
-      double g = (double)rgb[1] / (double)rgb_max[1];
-      double b = (double)rgb[2] / (double)rgb_max[2];
-      double luminance = (0.299 * r) + (0.587 * g) + (0.114 * b);  // CCIR 601
-      char *bgvalue = luminance < 0.5 ? "dark" : "light";
-      DLOG("bg response: %s", bgvalue);
-      loop_schedule_deferred(&main_loop,
-                             event_create(set_bg_deferred, 1, bgvalue));
     } else {
-      DLOG("failed to parse bg response");
+      bad = true;
     }
-    return true;
   }
-  return false;
+  rbuffer_consumed(input->read_stream.buffer, count);
+  if (done && !bad && rgb_max[0] && rgb_max[1] && rgb_max[2]) {
+    double r = (double)rgb[0] / (double)rgb_max[0];
+    double g = (double)rgb[1] / (double)rgb_max[1];
+    double b = (double)rgb[2] / (double)rgb_max[2];
+    double luminance = (0.299 * r) + (0.587 * g) + (0.114 * b);  // CCIR 601
+    char *bgvalue = luminance < 0.5 ? "dark" : "light";
+    DLOG("bg response: %s", bgvalue);
+    loop_schedule_deferred(&main_loop,
+                           event_create(set_bg_deferred, 1, bgvalue));
+  } else {
+    DLOG("failed to parse bg response");
+    return false;
+  }
+  return true;
 }
 
 static void tinput_read_cb(Stream *stream, RBuffer *buf, size_t count_,
diff --git a/test/functional/terminal/tui_spec.lua b/test/functional/terminal/tui_spec.lua
index defa700f3b..a8a49cd378 100644
--- a/test/functional/terminal/tui_spec.lua
+++ b/test/functional/terminal/tui_spec.lua
@@ -871,9 +871,9 @@ describe('TUI background color', function()
     screen:expect{any='did OptionSet, yay!'}
   end)
 
-  local function assert_bg(color, bg)
+  local function assert_bg(colorspace, color, bg)
     it('handles '..color..' as '..bg, function()
-      feed_data('\027]11;rgb:'..color..'\007')
+      feed_data('\027]11;'..colorspace..':'..color..'\007')
       -- Retry until the terminal response is handled.
       retry(100, nil, function()
         feed_data(':echo &background\n')
@@ -893,42 +893,59 @@ describe('TUI background color', function()
     end)
   end
 
-  assert_bg('0000/0000/0000', 'dark')
-  assert_bg('ffff/ffff/ffff', 'light')
-  assert_bg('000/000/000', 'dark')
-  assert_bg('fff/fff/fff', 'light')
-  assert_bg('00/00/00', 'dark')
-  assert_bg('ff/ff/ff', 'light')
-  assert_bg('0/0/0', 'dark')
-  assert_bg('f/f/f', 'light')
-
-  assert_bg('f/0/0', 'dark')
-  assert_bg('0/f/0', 'light')
-  assert_bg('0/0/f', 'dark')
-
-  assert_bg('1/1/1', 'dark')
-  assert_bg('2/2/2', 'dark')
-  assert_bg('3/3/3', 'dark')
-  assert_bg('4/4/4', 'dark')
-  assert_bg('5/5/5', 'dark')
-  assert_bg('6/6/6', 'dark')
-  assert_bg('7/7/7', 'dark')
-  assert_bg('8/8/8', 'light')
-  assert_bg('9/9/9', 'light')
-  assert_bg('a/a/a', 'light')
-  assert_bg('b/b/b', 'light')
-  assert_bg('c/c/c', 'light')
-  assert_bg('d/d/d', 'light')
-  assert_bg('e/e/e', 'light')
-
-  assert_bg('0/e/0', 'light')
-  assert_bg('0/d/0', 'light')
-  assert_bg('0/c/0', 'dark')
-  assert_bg('0/b/0', 'dark')
-
-  assert_bg('f/0/f', 'dark')
-  assert_bg('f/1/f', 'dark')
-  assert_bg('f/2/f', 'dark')
-  assert_bg('f/3/f', 'light')
-  assert_bg('f/4/f', 'light')
+  assert_bg('rgb', '0000/0000/0000', 'dark')
+  assert_bg('rgb', 'ffff/ffff/ffff', 'light')
+  assert_bg('rgb', '000/000/000', 'dark')
+  assert_bg('rgb', 'fff/fff/fff', 'light')
+  assert_bg('rgb', '00/00/00', 'dark')
+  assert_bg('rgb', 'ff/ff/ff', 'light')
+  assert_bg('rgb', '0/0/0', 'dark')
+  assert_bg('rgb', 'f/f/f', 'light')
+
+  assert_bg('rgb', 'f/0/0', 'dark')
+  assert_bg('rgb', '0/f/0', 'light')
+  assert_bg('rgb', '0/0/f', 'dark')
+
+  assert_bg('rgb', '1/1/1', 'dark')
+  assert_bg('rgb', '2/2/2', 'dark')
+  assert_bg('rgb', '3/3/3', 'dark')
+  assert_bg('rgb', '4/4/4', 'dark')
+  assert_bg('rgb', '5/5/5', 'dark')
+  assert_bg('rgb', '6/6/6', 'dark')
+  assert_bg('rgb', '7/7/7', 'dark')
+  assert_bg('rgb', '8/8/8', 'light')
+  assert_bg('rgb', '9/9/9', 'light')
+  assert_bg('rgb', 'a/a/a', 'light')
+  assert_bg('rgb', 'b/b/b', 'light')
+  assert_bg('rgb', 'c/c/c', 'light')
+  assert_bg('rgb', 'd/d/d', 'light')
+  assert_bg('rgb', 'e/e/e', 'light')
+
+  assert_bg('rgb', '0/e/0', 'light')
+  assert_bg('rgb', '0/d/0', 'light')
+  assert_bg('rgb', '0/c/0', 'dark')
+  assert_bg('rgb', '0/b/0', 'dark')
+
+  assert_bg('rgb', 'f/0/f', 'dark')
+  assert_bg('rgb', 'f/1/f', 'dark')
+  assert_bg('rgb', 'f/2/f', 'dark')
+  assert_bg('rgb', 'f/3/f', 'light')
+  assert_bg('rgb', 'f/4/f', 'light')
+
+  assert_bg('rgba', '0000/0000/0000/0000', 'dark')
+  assert_bg('rgba', '0000/0000/0000/ffff', 'dark')
+  assert_bg('rgba', 'ffff/ffff/ffff/0000', 'light')
+  assert_bg('rgba', 'ffff/ffff/ffff/ffff', 'light')
+  assert_bg('rgba', '000/000/000/000', 'dark')
+  assert_bg('rgba', '000/000/000/fff', 'dark')
+  assert_bg('rgba', 'fff/fff/fff/000', 'light')
+  assert_bg('rgba', 'fff/fff/fff/fff', 'light')
+  assert_bg('rgba', '00/00/00/00', 'dark')
+  assert_bg('rgba', '00/00/00/ff', 'dark')
+  assert_bg('rgba', 'ff/ff/ff/00', 'light')
+  assert_bg('rgba', 'ff/ff/ff/ff', 'light')
+  assert_bg('rgba', '0/0/0/0', 'dark')
+  assert_bg('rgba', '0/0/0/f', 'dark')
+  assert_bg('rgba', 'f/f/f/0', 'light')
+  assert_bg('rgba', 'f/f/f/f', 'light')
 end)