fix(tui): move $COLORTERM check to _defaults.lua (#29197)

We currently check $COLORTERM in the TUI process to determine if the
terminal supports 24 bit color (truecolor). If $COLORTERM is "truecolor"
or "24bit" then we automatically assume that the terminal supports
truecolor, but if $COLORTERM is set to any other value we still query
the terminal.

The `rgb` flag of the UI struct is a boolean which only indicates
whether the UI supports truecolor, but does not have a 3rd state that we
can use to represent "we don't know if the UI supports truecolor". We
currently use `rgb=false` to represent this "we don't know" state, and
we use XTGETTCAP and DECRQSS queries to determine at runtime if the
terminal supports truecolor. However, if $COLORTERM is set to a value
besides "truecolor" or "24bit" (e.g. "256" or "16) that is a clear
indication that the terminal _does not_ support truecolor, so it is
incorrect to treat `rgb=false` as "we don't know" in that case.

Instead, in the TUI process we only check for the terminfo capabilities.
This must be done in the TUI process because we do not have access to
this information in the core Neovim process when `_defaults.lua` runs.
If the TUI cannot determine truecolor support from terminfo alone, we
set `rgb=false` to indicate "we don't know if the terminal supports
truecolor yet, keep checking". When we get to `_defaults.lua`, we can
then check $COLORTERM and only query the terminal if it is unset.

This means that users can explicitly opt out of truecolor determination
by setting `COLORTERM=256` (or similar) in their environment.

1 files changed, 55 insertions, 0 deletions

diff --git a/test/functional/terminal/tui_spec.lua b/test/functional/terminal/tui_spec.lua
index efa65b7441..80df336cc4 100644
--- a/test/functional/terminal/tui_spec.lua
+++ b/test/functional/terminal/tui_spec.lua
@@ -2939,6 +2939,61 @@ describe('TUI', function()
     end)
   end)
 
+  it('does not query the terminal for truecolor support if $COLORTERM is set', function()
+    clear()
+    exec_lua([[
+      vim.api.nvim_create_autocmd('TermRequest', {
+        callback = function(args)
+          local req = args.data
+          vim.g.termrequest = req
+          local xtgettcap = req:match('^\027P%+q([%x;]+)$')
+          if xtgettcap then
+            local t = {}
+            for cap in vim.gsplit(xtgettcap, ';') do
+              local resp = string.format('\027P1+r%s\027\\', xtgettcap)
+              vim.api.nvim_chan_send(vim.bo[args.buf].channel, resp)
+              t[vim.text.hexdecode(cap)] = true
+            end
+            vim.g.xtgettcap = t
+            return true
+          elseif req:match('^\027P$qm\027\\$') then
+            vim.g.decrqss = true
+          end
+        end,
+      })
+    ]])
+
+    local child_server = new_pipename()
+    screen = tt.setup_child_nvim({
+      '--listen',
+      child_server,
+      '-u',
+      'NONE',
+      '-i',
+      'NONE',
+    }, {
+      env = {
+        VIMRUNTIME = os.getenv('VIMRUNTIME'),
+        -- With COLORTERM=256, Nvim should not query the terminal and should not set 'tgc'
+        COLORTERM = '256',
+        TERM = 'xterm-256colors',
+      },
+    })
+
+    screen:expect({ any = '%[No Name%]' })
+
+    local child_session = n.connect(child_server)
+    retry(nil, 1000, function()
+      local xtgettcap = eval("get(g:, 'xtgettcap', {})")
+      eq(nil, xtgettcap['Tc'])
+      eq(nil, xtgettcap['RGB'])
+      eq(nil, xtgettcap['setrgbf'])
+      eq(nil, xtgettcap['setrgbb'])
+      eq(0, eval([[get(g:, 'decrqss')]]))
+      eq({ true, 0 }, { child_session:request('nvim_eval', '&termguicolors') })
+    end)
+  end)
+
   it('queries the terminal for OSC 52 support', function()
     clear()
     exec_lua([[