feat(treesitter): support URLs (#27132)

Tree-sitter queries can add URLs to a capture using the `#set!`
directive, e.g.

  (inline_link
    (link_text) @text.reference
    (link_destination) @text.uri
    (#set! @text.reference "url" @text.uri))

The pattern above is included by default in the `markdown_inline`
highlight query so that users with supporting terminals will see
hyperlinks. For now, this creates a hyperlink for *all* Markdown URLs of
the pattern [link text](link url), even if `link url` does not contain
a valid protocol (e.g. if `link url` is a path to a file). We may wish to
change this in the future to only linkify when the URL has a valid
protocol scheme, but for now we delegate handling this to the terminal
emulator.

In order to support directives which reference other nodes, the
highlighter must be updated to use `iter_matches` rather than
`iter_captures`. The former provides the `match` table which maps
capture IDs to nodes. However, this has its own challenges:

- `iter_matches` does not guarantee the order in which patterns are
  iterated matches the order in the query file. So we must enforce
  ordering manually using "subpriorities" (#27131). The pattern index of
  each match dictates the extmark's subpriority.
- When injections are used, the highlighter contains multiple trees. The
  pattern indices of each tree must be offset relative to the maximum
  pattern index from all previous trees to ensure that extmarks appear
  in the correct order.
- The `iter_captures` implementation currently has a bug where the
  "match" table is only returned for the first capture within a pattern
  (see #27274). This bug means that `#set!` directives in a query
  apply only to the first capture within a pattern. Unfortunately, many
  queries in the wild have come to depend on this behavior.
  `iter_matches` does not share this flaw, so switching to
  `iter_matches` exposed bugs in existing highlight queries. These
  queries have been updated in this repo, but may still need to be
  updated by users. The `#set!` directive applies to the _entire_ query
  pattern when used without a capture argument. To make `#set!`
  apply only to a single capture, the capture must be given as an
  argument.

1 files changed, 88 insertions, 1 deletions

diff --git a/test/functional/treesitter/highlight_spec.lua b/test/functional/treesitter/highlight_spec.lua
index 2bf230fe69..7f2b5751ae 100644
--- a/test/functional/treesitter/highlight_spec.lua
+++ b/test/functional/treesitter/highlight_spec.lua
@@ -681,6 +681,12 @@ describe('treesitter highlighting (C)', function()
         ((identifier) @Identifier
          (#set! conceal "")
          (#eq? @Identifier "lstate"))
+
+        ((call_expression
+            function: (identifier) @function
+            arguments: (argument_list) @arguments)
+         (#eq? @function "multiqueue_put")
+         (#set! @function conceal "V"))
       ]]}})
     ]=]
 
@@ -697,7 +703,7 @@ describe('treesitter highlighting (C)', function()
                                                                        |
         LuaRef cb = nlua_ref(, 1);                                     |
                                                                        |
-        multiqueue_put(main_loop.events, nlua_schedule_event,          |
+        {11:V}(main_loop.events, nlua_schedule_event,                       |
                        1, (void *)(ptrdiff_t)cb);                      |
         return 0;                                                      |
       ^}                                                                |
@@ -758,6 +764,44 @@ describe('treesitter highlighting (C)', function()
   end)
 end)
 
+describe('treesitter highlighting (lua)', function()
+  local screen
+
+  before_each(function()
+    screen = Screen.new(65, 18)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = { bold = true, foreground = Screen.colors.Blue },
+      [2] = { foreground = Screen.colors.DarkCyan },
+      [3] = { foreground = Screen.colors.Magenta },
+      [4] = { foreground = Screen.colors.SlateBlue },
+      [5] = { bold = true, foreground = Screen.colors.Brown },
+    }
+  end)
+
+  it('supports language injections', function()
+    insert [[
+      local ffi = require('ffi')
+      ffi.cdef("int (*fun)(int, char *);")
+    ]]
+
+    exec_lua [[
+      vim.bo.filetype = 'lua'
+      vim.treesitter.start()
+    ]]
+
+    screen:expect {
+      grid = [[
+        {5:local} {2:ffi} {5:=} {4:require(}{3:'ffi'}{4:)}                                     |
+        {2:ffi}{4:.}{2:cdef}{4:(}{3:"}{4:int}{3: }{4:(}{5:*}{3:fun}{4:)(int,}{3: }{4:char}{3: }{5:*}{4:);}{3:"}{4:)}                           |
+      ^                                                                 |
+      {1:~                                                                }|*14
+                                                                       |
+    ]],
+    }
+  end)
+end)
+
 describe('treesitter highlighting (help)', function()
   local screen
 
@@ -891,3 +935,46 @@ vim.cmd([[
     }
   end)
 end)
+
+describe('treesitter highlighting (markdown)', function()
+  local screen
+
+  before_each(function()
+    screen = Screen.new(40, 6)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = { foreground = Screen.colors.Blue1 },
+      [2] = { bold = true, foreground = Screen.colors.Blue1 },
+      [3] = { bold = true, foreground = Screen.colors.Brown },
+      [4] = { foreground = Screen.colors.Cyan4 },
+      [5] = { foreground = Screen.colors.Magenta1 },
+    }
+  end)
+
+  it('supports hyperlinks', function()
+    local url = 'https://example.com'
+    insert(string.format('[This link text](%s) is a hyperlink.', url))
+    exec_lua([[
+      vim.bo.filetype = 'markdown'
+      vim.treesitter.start()
+    ]])
+
+    screen:expect {
+      grid = [[
+      {4:[}{6:This link text}{4:](}{7:https://example.com}{4:)} is|
+       a hyperlink^.                           |
+      {2:~                                       }|*3
+                                              |
+    ]],
+      attr_ids = {
+        [1] = { foreground = Screen.colors.Blue1 },
+        [2] = { bold = true, foreground = Screen.colors.Blue1 },
+        [3] = { bold = true, foreground = Screen.colors.Brown },
+        [4] = { foreground = Screen.colors.Cyan4 },
+        [5] = { foreground = Screen.colors.Magenta },
+        [6] = { foreground = Screen.colors.Cyan4, url = url },
+        [7] = { underline = true, foreground = Screen.colors.SlateBlue },
+      },
+    }
+  end)
+end)