feat(defaults): enable 'termguicolors' by default when supported by terminal

Enable 'termguicolors' automatically when Nvim can detect that truecolor
is supported by the host terminal.

If $COLORTERM is set to "truecolor" or "24bit", or the terminal's
terminfo entry contains capabilities for Tc, RGB, or setrgbf and
setrgbb, then we assume that the terminal supports truecolor. Otherwise,
the terminal is queried (using both XTGETTCAP and SGR + DECRQSS). If the
terminal's response to these queries (if any) indicates that it supports
truecolor, then 'termguicolors' is enabled.

1 files changed, 180 insertions, 75 deletions

diff --git a/runtime/lua/vim/_defaults.lua b/runtime/lua/vim/_defaults.lua
index c3bb36fc36..b73681be04 100644
--- a/runtime/lua/vim/_defaults.lua
+++ b/runtime/lua/vim/_defaults.lua
@@ -165,91 +165,92 @@ do
   })
 end
 
---- Guess value of 'background' based on terminal color.
----
---- We write Operating System Command (OSC) 11 to the terminal to request the
---- terminal's background color. We then wait for a response. If the response
---- matches `rgba:RRRR/GGGG/BBBB/AAAA` where R, G, B, and A are hex digits, then
---- compute the luminance[1] of the RGB color and classify it as light/dark
---- accordingly. Note that the color components may have anywhere from one to
---- four hex digits, and require scaling accordingly as values out of 4, 8, 12,
---- or 16 bits. Also note the A(lpha) component is optional, and is parsed but
---- ignored in the calculations.
----
---- [1] https://en.wikipedia.org/wiki/Luma_%28video%29
-do
-  --- Parse a string of hex characters as a color.
-  ---
-  --- The string can contain 1 to 4 hex characters. The returned value is
-  --- between 0.0 and 1.0 (inclusive) representing the intensity of the color.
-  ---
-  --- For instance, if only a single hex char "a" is used, then this function
-  --- returns 0.625 (10 / 16), while a value of "aa" would return 0.664 (170 /
-  --- 256).
-  ---
-  --- @param c string Color as a string of hex chars
-  --- @return number? Intensity of the color
-  local function parsecolor(c)
-    if #c == 0 or #c > 4 then
-      return nil
-    end
-
-    local val = tonumber(c, 16)
-    if not val then
-      return nil
-    end
-
-    local max = tonumber(string.rep('f', #c), 16)
-    return val / max
+-- Only do the following when the TUI is attached
+local tty = nil
+for _, ui in ipairs(vim.api.nvim_list_uis()) do
+  if ui.chan == 1 and ui.stdout_tty then
+    tty = ui
+    break
   end
+end
 
-  --- Parse an OSC 11 response
-  ---
-  --- Either of the two formats below are accepted:
-  ---
-  ---   OSC 11 ; rgb:<red>/<green>/<blue>
-  ---
-  --- or
-  ---
-  ---   OSC 11 ; rgba:<red>/<green>/<blue>/<alpha>
+if tty then
+  --- Guess value of 'background' based on terminal color.
   ---
-  --- where
+  --- We write Operating System Command (OSC) 11 to the terminal to request the
+  --- terminal's background color. We then wait for a response. If the response
+  --- matches `rgba:RRRR/GGGG/BBBB/AAAA` where R, G, B, and A are hex digits, then
+  --- compute the luminance[1] of the RGB color and classify it as light/dark
+  --- accordingly. Note that the color components may have anywhere from one to
+  --- four hex digits, and require scaling accordingly as values out of 4, 8, 12,
+  --- or 16 bits. Also note the A(lpha) component is optional, and is parsed but
+  --- ignored in the calculations.
   ---
-  ---   <red>, <green>, <blue>, <alpha> := h | hh | hhh | hhhh
-  ---
-  --- The alpha component is ignored, if present.
-  ---
-  --- @param resp string OSC 11 response
-  --- @return string? Red component
-  --- @return string? Green component
-  --- @return string? Blue component
-  local function parseosc11(resp)
-    local r, g, b
-    r, g, b = resp:match('^\027%]11;rgb:(%x+)/(%x+)/(%x+)$')
-    if not r and not g and not b then
-      local a
-      r, g, b, a = resp:match('^\027%]11;rgba:(%x+)/(%x+)/(%x+)/(%x+)$')
-      if not a or #a > 4 then
-        return nil, nil, nil
+  --- [1] https://en.wikipedia.org/wiki/Luma_%28video%29
+  do
+    --- Parse a string of hex characters as a color.
+    ---
+    --- The string can contain 1 to 4 hex characters. The returned value is
+    --- between 0.0 and 1.0 (inclusive) representing the intensity of the color.
+    ---
+    --- For instance, if only a single hex char "a" is used, then this function
+    --- returns 0.625 (10 / 16), while a value of "aa" would return 0.664 (170 /
+    --- 256).
+    ---
+    --- @param c string Color as a string of hex chars
+    --- @return number? Intensity of the color
+    local function parsecolor(c)
+      if #c == 0 or #c > 4 then
+        return nil
       end
-    end
 
-    if r and g and b and #r <= 4 and #g <= 4 and #b <= 4 then
-      return r, g, b
+      local val = tonumber(c, 16)
+      if not val then
+        return nil
+      end
+
+      local max = tonumber(string.rep('f', #c), 16)
+      return val / max
     end
 
-    return nil, nil, nil
-  end
+    --- Parse an OSC 11 response
+    ---
+    --- Either of the two formats below are accepted:
+    ---
+    ---   OSC 11 ; rgb:<red>/<green>/<blue>
+    ---
+    --- or
+    ---
+    ---   OSC 11 ; rgba:<red>/<green>/<blue>/<alpha>
+    ---
+    --- where
+    ---
+    ---   <red>, <green>, <blue>, <alpha> := h | hh | hhh | hhhh
+    ---
+    --- The alpha component is ignored, if present.
+    ---
+    --- @param resp string OSC 11 response
+    --- @return string? Red component
+    --- @return string? Green component
+    --- @return string? Blue component
+    local function parseosc11(resp)
+      local r, g, b
+      r, g, b = resp:match('^\027%]11;rgb:(%x+)/(%x+)/(%x+)$')
+      if not r and not g and not b then
+        local a
+        r, g, b, a = resp:match('^\027%]11;rgba:(%x+)/(%x+)/(%x+)/(%x+)$')
+        if not a or #a > 4 then
+          return nil, nil, nil
+        end
+      end
+
+      if r and g and b and #r <= 4 and #g <= 4 and #b <= 4 then
+        return r, g, b
+      end
 
-  local tty = false
-  for _, ui in ipairs(vim.api.nvim_list_uis()) do
-    if ui.chan == 1 and ui.stdout_tty then
-      tty = true
-      break
+      return nil, nil, nil
     end
-  end
 
-  if tty then
     local timer = assert(vim.uv.new_timer())
 
     ---@param bg string New value of the 'background' option
@@ -300,7 +301,7 @@ do
     io.stdout:write('\027]11;?\007')
 
     timer:start(1000, 0, function()
-      -- No response received. Delete the autocommand
+      -- Delete the autocommand if no response was received
       vim.schedule(function()
         -- Suppress error if autocommand has already been deleted
         pcall(vim.api.nvim_del_autocmd, id)
@@ -311,4 +312,108 @@ do
       end
     end)
   end
+
+  --- If the TUI (term_has_truecolor) was able to determine that the host
+  --- terminal supports truecolor, enable 'termguicolors'. Otherwise, query the
+  --- terminal (using both XTGETTCAP and SGR + DECRQSS). If the terminal's
+  --- response indicates that it does support truecolor enable 'termguicolors',
+  --- but only if the user has not already disabled it.
+  do
+    if tty.rgb then
+      -- The TUI was able to determine truecolor support
+      vim.o.termguicolors = true
+    else
+      --- Enable 'termguicolors', but only if it was not already set by the user.
+      local function settgc()
+        if not vim.api.nvim_get_option_info2('termguicolors', {}).was_set then
+          vim.o.termguicolors = true
+        end
+      end
+
+      local caps = {} ---@type table<string, boolean>
+      require('vim.termcap').query({ 'Tc', 'RGB', 'setrgbf', 'setrgbb' }, function(cap, found)
+        if not found then
+          return
+        end
+
+        caps[cap] = true
+        if caps.Tc or caps.RGB or (caps.setrgbf and caps.setrgbb) then
+          settgc()
+        end
+      end)
+
+      local timer = assert(vim.uv.new_timer())
+
+      -- Arbitrary colors to set in the SGR sequence
+      local r = 1
+      local g = 2
+      local b = 3
+
+      local id = vim.api.nvim_create_autocmd('TermResponse', {
+        nested = true,
+        callback = function(args)
+          local resp = args.data ---@type string
+          local decrqss = resp:match('^\027P1%$r([%d;:]+)m$')
+
+          if decrqss then
+            -- The DECRQSS SGR response first contains attributes separated by
+            -- semicolons, followed by the SGR itself with parameters separated
+            -- by colons. Some terminals include "0" in the attribute list
+            -- unconditionally; others do not. Our SGR sequence did not set any
+            -- attributes, so there should be no attributes in the list.
+            local attrs = vim.split(decrqss, ';')
+            if #attrs ~= 1 and (#attrs ~= 2 or attrs[1] ~= '0') then
+              return true
+            end
+
+            -- The returned SGR sequence should begin with 48:2
+            local sgr = attrs[#attrs]:match('^48:2:([%d:]+)$')
+            if not sgr then
+              return true
+            end
+
+            -- The remaining elements of the SGR sequence should be the 3 colors
+            -- we set. Some terminals also include an additional parameter
+            -- (which can even be empty!), so handle those cases as well
+            local params = vim.split(sgr, ':')
+            if #params ~= 3 and (#params ~= 4 or (params[1] ~= '' and params[1] ~= '1')) then
+              return true
+            end
+
+            if
+              tonumber(params[#params - 2]) == r
+              and tonumber(params[#params - 1]) == g
+              and tonumber(params[#params]) == b
+            then
+              settgc()
+            end
+
+            return true
+          end
+        end,
+      })
+
+      -- Write SGR followed by DECRQSS. This sets the background color then
+      -- immediately asks the terminal what the background color is. If the
+      -- terminal responds to the DECRQSS with the same SGR sequence that we
+      -- sent then the terminal supports truecolor.
+      local decrqss = '\027P$qm\027\\'
+      if os.getenv('TMUX') then
+        decrqss = string.format('\027Ptmux;%s\027\\', decrqss:gsub('\027', '\027\027'))
+      end
+      io.stdout:write(string.format('\027[48;2;%d;%d;%dm%s', r, g, b, decrqss))
+
+      timer:start(1000, 0, function()
+        -- Delete the autocommand if no response was received
+        vim.schedule(function()
+          -- Suppress error if autocommand has already been deleted
+          pcall(vim.api.nvim_del_autocmd, id)
+        end)
+
+        if not timer:is_closing() then
+          timer:close()
+        end
+      end)
+    end
+  end
 end