feat(treesitter)!: use @foo.bar style highlight groups

This removes the support for defining links via
vim.treesitter.highlighter.hl_map (never documented, but plugins did
anyway), or the uppercase-only `@FooGroup.Bar` to `FooGroup` rule.

The fallback is now strictly `@foo.bar.lang` to `@foo.bar` to `@foo`,
and casing is irrelevant (as it already was outside of treesitter)

For compatibility, define default links to builting syntax groups
as defined by pre-existing color schemes

5 files changed, 140 insertions, 153 deletions

diff --git a/runtime/doc/treesitter.txt b/runtime/doc/treesitter.txt
index c80d8f4479..99780b0f29 100644
--- a/runtime/doc/treesitter.txt
+++ b/runtime/doc/treesitter.txt
@@ -323,16 +323,34 @@ for a buffer with this code: >
     local query2 = [[ ... ]]
     highlighter:set_query(query2)
 
-As mentioned above the supported predicate is currently only `eq?`. `match?`
-predicates behave like matching always fails. As an addition a capture which
-begin with an upper-case letter like `@WarningMsg` will map directly to this
-highlight group, if defined. Also if the predicate begins with upper-case and
-contains a dot only the part before the first will be interpreted as the
-highlight group. As an example, this warns of a binary expression with two
+
+                                             *lua-treesitter-highlight-groups*
+The capture names, with `@` included, are directly usable as highlight groups.
+A fallback system is implemented, so that more specific groups fallback to
+more generic ones. For instance, in a language that has separate doc
+comments, `@comment.doc` could be used. If this group is not defined, the
+highlighting for an ordinary `@comment` is used. This way, existing color
+schemes already work out of the box, but it is possible to add
+more specific variants for queries that make them available.
+
+As an additional rule, captures highlights can always be specialized by
+language, by appending the language name after an additional dot. For
+instance, to highlight comments differently per language: >
+
+    hi @comment.c guifg=Blue
+    hi @comment.lua @guifg=DarkBlue
+    hi link @comment.doc.java String
+<
+It is possible to use custom highlight groups. As an example, if we
+define the `@warning` group: >
+
+    hi link @warning WarningMsg
+<
+the following query warns of a binary expression with two
 identical identifiers, highlighting both as |hl-WarningMsg|: >
 
-    ((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right)
-     (eq? @WarningMsg.left @WarningMsg.right))
+    ((binary_expression left: (identifier) @warning.left right: (identifier) @warning.right)
+     (eq? @warning.left @warning.right))
 <
 Treesitter Highlighting Priority            *lua-treesitter-highlight-priority*
 
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index e27a5fa9c3..1f242b0fdd 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -12,105 +12,18 @@ TSHighlighterQuery.__index = TSHighlighterQuery
 
 local ns = a.nvim_create_namespace('treesitter/highlighter')
 
-local _default_highlights = {}
-local _link_default_highlight_once = function(from, to)
-  if not _default_highlights[from] then
-    _default_highlights[from] = true
-    a.nvim_set_hl(0, from, { link = to, default = true })
-  end
-
-  return from
-end
-
--- If @definition.special does not exist use @definition instead
-local subcapture_fallback = {
-  __index = function(self, capture)
-    local rtn
-    local shortened = capture
-    while not rtn and shortened do
-      shortened = shortened:match('(.*)%.')
-      rtn = shortened and rawget(self, shortened)
-    end
-    rawset(self, capture, rtn or '__notfound')
-    return rtn
-  end,
-}
-
-TSHighlighter.hl_map = setmetatable({
-  ['error'] = 'Error',
-  ['text.underline'] = 'Underlined',
-  ['todo'] = 'Todo',
-  ['debug'] = 'Debug',
-
-  -- Miscs
-  ['comment'] = 'Comment',
-  ['punctuation.delimiter'] = 'Delimiter',
-  ['punctuation.bracket'] = 'Delimiter',
-  ['punctuation.special'] = 'Delimiter',
-
-  -- Constants
-  ['constant'] = 'Constant',
-  ['constant.builtin'] = 'Special',
-  ['constant.macro'] = 'Define',
-  ['define'] = 'Define',
-  ['macro'] = 'Macro',
-  ['string'] = 'String',
-  ['string.regex'] = 'String',
-  ['string.escape'] = 'SpecialChar',
-  ['character'] = 'Character',
-  ['character.special'] = 'SpecialChar',
-  ['number'] = 'Number',
-  ['boolean'] = 'Boolean',
-  ['float'] = 'Float',
-
-  -- Functions
-  ['function'] = 'Function',
-  ['function.special'] = 'Function',
-  ['function.builtin'] = 'Special',
-  ['function.macro'] = 'Macro',
-  ['parameter'] = 'Identifier',
-  ['method'] = 'Function',
-  ['field'] = 'Identifier',
-  ['property'] = 'Identifier',
-  ['constructor'] = 'Special',
-
-  -- Keywords
-  ['conditional'] = 'Conditional',
-  ['repeat'] = 'Repeat',
-  ['label'] = 'Label',
-  ['operator'] = 'Operator',
-  ['keyword'] = 'Keyword',
-  ['exception'] = 'Exception',
-
-  ['type'] = 'Type',
-  ['type.builtin'] = 'Type',
-  ['type.qualifier'] = 'Type',
-  ['type.definition'] = 'Typedef',
-  ['storageclass'] = 'StorageClass',
-  ['structure'] = 'Structure',
-  ['include'] = 'Include',
-  ['preproc'] = 'PreProc',
-}, subcapture_fallback)
-
----@private
-local function is_highlight_name(capture_name)
-  local firstc = string.sub(capture_name, 1, 1)
-  return firstc ~= string.lower(firstc)
-end
-
 ---@private
 function TSHighlighterQuery.new(lang, query_string)
   local self = setmetatable({}, { __index = TSHighlighterQuery })
 
   self.hl_cache = setmetatable({}, {
     __index = function(table, capture)
-      local hl, is_vim_highlight = self:_get_hl_from_capture(capture)
-      if not is_vim_highlight then
-        hl = _link_default_highlight_once(lang .. hl, hl)
+      local name = self._query.captures[capture]
+      local id = 0
+      if not vim.startswith(name, '_') then
+        id = a.nvim_get_hl_id_by_name('@' .. name .. '.' .. lang)
       end
 
-      local id = a.nvim_get_hl_id_by_name(hl)
-
       rawset(table, capture, id)
       return id
     end,
@@ -130,20 +43,6 @@ function TSHighlighterQuery:query()
   return self._query
 end
 
----@private
---- Get the hl from capture.
---- Returns a tuple { highlight_name: string, is_builtin: bool }
-function TSHighlighterQuery:_get_hl_from_capture(capture)
-  local name = self._query.captures[capture]
-
-  if is_highlight_name(name) then
-    -- From "Normal.left" only keep "Normal"
-    return vim.split(name, '.', true)[1], true
-  else
-    return TSHighlighter.hl_map[name] or 0, false
-  end
-end
-
 --- Creates a new highlighter using @param tree
 ---
 ---@param tree The language tree to use for highlighting
diff --git a/src/nvim/highlight_group.c b/src/nvim/highlight_group.c
index 4c207fdb53..7a0c7ba395 100644
--- a/src/nvim/highlight_group.c
+++ b/src/nvim/highlight_group.c
@@ -185,6 +185,54 @@ static const char *highlight_init_both[] = {
   "default link DiagnosticSignWarn DiagnosticWarn",
   "default link DiagnosticSignInfo DiagnosticInfo",
   "default link DiagnosticSignHint DiagnosticHint",
+
+  "default link @error Error",
+  "default link @text.underline Underlined",
+  "default link @todo Todo",
+  "default link @debug Debug",
+
+  // Miscs
+  "default link @comment Comment",
+  "default link @punctuation Delimiter",
+
+  // Constants
+  "default link @constant Constant",
+  "default link @constant.builtin Special",
+  "default link @constant.macro Define",
+  "default link @define Define",
+  "default link @macro Macro",
+  "default link @string String",
+  "default link @string.escape SpecialChar",
+  "default link @character Character",
+  "default link @character.special SpecialChar",
+  "default link @number Number",
+  "default link @boolean Boolean",
+  "default link @float Float",
+
+  // Functions
+  "default link @function Function",
+  "default link @function.builtin Special",
+  "default link @function.macro Macro",
+  "default link @parameter Identifier",
+  "default link @method Function",
+  "default link @field Identifier",
+  "default link @property Identifier",
+  "default link @constructor Special",
+
+  // Keywords
+  "default link @conditional Conditional",
+  "default link @repeat Repeat",
+  "default link @label Label",
+  "default link @operator Operator",
+  "default link @keyword Keyword",
+  "default link @exception Exception",
+
+  "default link @type Type",
+  "default link @type.definition Typedef",
+  "default link @storageclass StorageClass",
+  "default link @structure Structure",
+  "default link @include Include",
+  "default link @preproc PreProc",
   NULL
 };
 
diff --git a/src/nvim/testdir/test_cmdline.vim b/src/nvim/testdir/test_cmdline.vim
index 84dc361529..517509004a 100644
--- a/src/nvim/testdir/test_cmdline.vim
+++ b/src/nvim/testdir/test_cmdline.vim
@@ -244,7 +244,7 @@ func Test_match_completion()
     return
   endif
   hi Aardig ctermfg=green
-  call feedkeys(":match \<Tab>\<Home>\"\<CR>", 'xt')
+  call feedkeys(":match A\<Tab>\<Home>\"\<CR>", 'xt')
   call assert_equal('"match Aardig', getreg(':'))
   call feedkeys(":match \<S-Tab>\<Home>\"\<CR>", 'xt')
   call assert_equal('"match none', getreg(':'))
@@ -255,9 +255,7 @@ func Test_highlight_completion()
     return
   endif
   hi Aardig ctermfg=green
-  call feedkeys(":hi \<Tab>\<Home>\"\<CR>", 'xt')
-  call assert_equal('"hi Aardig', getreg(':'))
-  call feedkeys(":hi default \<Tab>\<Home>\"\<CR>", 'xt')
+  call feedkeys(":hi default A\<Tab>\<Home>\"\<CR>", 'xt')
   call assert_equal('"hi default Aardig', getreg(':'))
   call feedkeys(":hi clear Aa\<Tab>\<Home>\"\<CR>", 'xt')
   call assert_equal('"hi clear Aardig', getreg(':'))
diff --git a/test/functional/treesitter/highlight_spec.lua b/test/functional/treesitter/highlight_spec.lua
index 5ec0a8a060..e4ef975d34 100644
--- a/test/functional/treesitter/highlight_spec.lua
+++ b/test/functional/treesitter/highlight_spec.lua
@@ -6,11 +6,14 @@ local insert = helpers.insert
 local exec_lua = helpers.exec_lua
 local feed = helpers.feed
 local pending_c_parser = helpers.pending_c_parser
+local command = helpers.command
+local meths = helpers.meths
+local eq = helpers.eq
 
 before_each(clear)
 
 local hl_query = [[
-  (ERROR) @ErrorMsg
+  (ERROR) @error
 
   "if" @keyword
   "else" @keyword
@@ -23,23 +26,24 @@ local hl_query = [[
   "enum" @type
   "extern" @type
 
-  (string_literal) @string.nonexistent-specializer-for-string.should-fallback-to-string
+  ; nonexistent specializer for string should fallback to string
+  (string_literal) @string.nonexistent_specializer
 
   (number_literal) @number
   (char_literal) @string
 
   (type_identifier) @type
-  ((type_identifier) @Special (#eq? @Special "LuaRef"))
+  ((type_identifier) @constant.builtin (#eq? @constant.builtin "LuaRef"))
 
   (primitive_type) @type
   (sized_type_specifier) @type
 
   ; Use lua regexes
-  ((identifier) @Identifier (#contains? @Identifier "lua_"))
+  ((identifier) @function (#contains? @function "lua_"))
   ((identifier) @Constant (#lua-match? @Constant "^[A-Z_]+$"))
-  ((identifier) @Normal (#vim-match? @Constant "^lstate$"))
+  ((identifier) @Normal (#vim-match? @Normal "^lstate$"))
 
-  ((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right) (#eq? @WarningMsg.left @WarningMsg.right))
+  ((binary_expression left: (identifier) @warning.left right: (identifier) @warning.right) (#eq? @warning.left @warning.right))
 
   (comment) @comment
 ]]
@@ -103,6 +107,7 @@ describe('treesitter highlighting', function()
     }
 
     exec_lua([[ hl_query = ... ]], hl_query)
+    command [[ hi link @warning WarningMsg ]]
   end)
 
   it('is updated with edits', function()
@@ -547,7 +552,7 @@ describe('treesitter highlighting', function()
 
     -- This will change ONLY the literal strings to look like comments
     -- The only literal string is the "vim.schedule: expected function" in this test.
-    exec_lua [[vim.cmd("highlight link cString comment")]]
+    exec_lua [[vim.cmd("highlight link @string.nonexistent_specializer comment")]]
     screen:expect{grid=[[
       {2:/// Schedule Lua callback on main loop's event queue}             |
       {3:static} {3:int} {11:nlua_schedule}({3:lua_State} *{3:const} lstate)                |
@@ -642,11 +647,13 @@ describe('treesitter highlighting', function()
                                                                        |
     ]]}
 
+    command [[
+      hi link @foo.bar Type
+      hi link @foo String
+    ]]
     exec_lua [[
       local parser = vim.treesitter.get_parser(0, "c", {})
       local highlighter = vim.treesitter.highlighter
-      highlighter.hl_map['foo.bar'] = 'Type'
-      highlighter.hl_map['foo'] = 'String'
       test_hl = highlighter.new(parser, {queries = {c = "(primitive_type) @foo.bar (string_literal) @foo"}})
     ]]
 
@@ -670,6 +677,29 @@ describe('treesitter highlighting', function()
       {1:~                                                                }|
                                                                        |
     ]]}
+
+    -- clearing specialization reactivates fallback
+    command [[ hi clear @foo.bar ]]
+    screen:expect{grid=[[
+      {5:char}* x = {5:"Will somebody ever read this?"};                       |
+      ^                                                                 |
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+      {1:~                                                                }|
+                                                                       |
+    ]]}
   end)
 
   it("supports conceal attribute", function()
@@ -712,32 +742,26 @@ describe('treesitter highlighting', function()
     ]]}
   end)
 
-  it("hl_map has the correct fallback behavior", function()
-    exec_lua [[
-      local hl_map = vim.treesitter.highlighter.hl_map
-      hl_map["foo"] = 1
-      hl_map["foo.bar"] = 2
-      hl_map["foo.bar.baz"] = 3
-
-      assert(hl_map["foo"] == 1)
-      assert(hl_map["foo.a.b.c.d"] == 1)
-      assert(hl_map["foo.bar"] == 2)
-      assert(hl_map["foo.bar.a.b.c.d"] == 2)
-      assert(hl_map["foo.bar.baz"] == 3)
-      assert(hl_map["foo.bar.baz.d"] == 3)
-
-      hl_map["FOO"] = 1
-      hl_map["FOO.BAR"] = 2
-      assert(hl_map["FOO.BAR.BAZ"] == 2)
-
-      hl_map["foo.missing.exists"] = 3
-      assert(hl_map["foo.missing"] == 1)
-      assert(hl_map["foo.missing.exists"] == 3)
-      assert(hl_map["foo.missing.exists.bar"] == 3)
-      assert(hl_map["total.nonsense.but.a.lot.of.dots"] == nil)
-      -- It will not perform a second look up of this variable but return a sentinel value
-      assert(hl_map["total.nonsense.but.a.lot.of.dots"] == "__notfound")
-    ]]
-
+  it("@foo.bar groups has the correct fallback behavior", function()
+    local get_hl = function(name) return meths.get_hl_by_name(name,1).foreground end
+    meths.set_hl(0, "@foo", {fg = 1})
+    meths.set_hl(0, "@foo.bar", {fg = 2})
+    meths.set_hl(0, "@foo.bar.baz", {fg = 3})
+
+    eq(1, get_hl"@foo")
+    eq(1, get_hl"@foo.a.b.c.d")
+    eq(2, get_hl"@foo.bar")
+    eq(2, get_hl"@foo.bar.a.b.c.d")
+    eq(3, get_hl"@foo.bar.baz")
+    eq(3, get_hl"@foo.bar.baz.d")
+
+    -- lookup is case insensitive
+    eq(2, get_hl"@FOO.BAR.SPAM")
+
+    meths.set_hl(0, "@foo.missing.exists", {fg = 3})
+    eq(1, get_hl"@foo.missing")
+    eq(3, get_hl"@foo.missing.exists")
+    eq(3, get_hl"@foo.missing.exists.bar")
+    eq(nil, get_hl"@total.nonsense.but.a.lot.of.dots")
   end)
 end)