From cb46f6e467268edf917cc3617b4c024a66b256de Mon Sep 17 00:00:00 2001
From: Gregory Anders <8965202+gpanders@users.noreply.github.com>
Date: Tue, 12 Mar 2024 09:32:17 -0500
Subject: feat(treesitter): support URLs (#27132)

Tree-sitter queries can add URLs to a capture using the `#set!`
directive, e.g.

  (inline_link
    (link_text) @text.reference
    (link_destination) @text.uri
    (#set! @text.reference "url" @text.uri))

The pattern above is included by default in the `markdown_inline`
highlight query so that users with supporting terminals will see
hyperlinks. For now, this creates a hyperlink for *all* Markdown URLs of
the pattern [link text](link url), even if `link url` does not contain
a valid protocol (e.g. if `link url` is a path to a file). We may wish to
change this in the future to only linkify when the URL has a valid
protocol scheme, but for now we delegate handling this to the terminal
emulator.

In order to support directives which reference other nodes, the
highlighter must be updated to use `iter_matches` rather than
`iter_captures`. The former provides the `match` table which maps
capture IDs to nodes. However, this has its own challenges:

- `iter_matches` does not guarantee the order in which patterns are
  iterated matches the order in the query file. So we must enforce
  ordering manually using "subpriorities" (#27131). The pattern index of
  each match dictates the extmark's subpriority.
- When injections are used, the highlighter contains multiple trees. The
  pattern indices of each tree must be offset relative to the maximum
  pattern index from all previous trees to ensure that extmarks appear
  in the correct order.
- The `iter_captures` implementation currently has a bug where the
  "match" table is only returned for the first capture within a pattern
  (see #27274). This bug means that `#set!` directives in a query
  apply only to the first capture within a pattern. Unfortunately, many
  queries in the wild have come to depend on this behavior.
  `iter_matches` does not share this flaw, so switching to
  `iter_matches` exposed bugs in existing highlight queries. These
  queries have been updated in this repo, but may still need to be
  updated by users. The `#set!` directive applies to the _entire_ query
  pattern when used without a capture argument. To make `#set!`
  apply only to a single capture, the capture must be given as an
  argument.
---
 runtime/lua/vim/treesitter/highlighter.lua | 91 ++++++++++++++++++++++--------
 1 file changed, 66 insertions(+), 25 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index 388680259a..cc5e11d632 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -4,7 +4,7 @@ local Range = require('vim.treesitter._range')
 
 local ns = api.nvim_create_namespace('treesitter/highlighter')
 
----@alias vim.treesitter.highlighter.Iter fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata
+---@alias vim.treesitter.highlighter.Iter fun(): integer, table<integer, TSNode[]>, vim.treesitter.query.TSMetadata
 
 ---@class (private) vim.treesitter.highlighter.Query
 ---@field private _query vim.treesitter.Query?
@@ -248,6 +248,13 @@ end
 ---@param line integer
 ---@param is_spell_nav boolean
 local function on_line_impl(self, buf, line, is_spell_nav)
+  -- Track the maximum pattern index encountered in each tree. For subsequent
+  -- trees, the subpriority passed to nvim_buf_set_extmark is offset by the
+  -- largest pattern index from the prior tree. This ensures that extmarks
+  -- from subsequent trees always appear "on top of" extmarks from previous
+  -- trees (e.g. injections should always appear over base highlights).
+  local pattern_offset = 0
+
   self:for_each_highlight_state(function(state)
     local root_node = state.tstree:root()
     local root_start_row, _, root_end_row, _ = root_node:range()
@@ -258,22 +265,24 @@ local function on_line_impl(self, buf, line, is_spell_nav)
     end
 
     if state.iter == nil or state.next_row < line then
-      state.iter =
-        state.highlighter_query:query():iter_captures(root_node, self.bufnr, line, root_end_row + 1)
+      state.iter = state.highlighter_query
+        :query()
+        :iter_matches(root_node, self.bufnr, line, root_end_row + 1, { all = true })
     end
 
+    local max_pattern_index = -1
     while line >= state.next_row do
-      local capture, node, metadata = state.iter(line)
+      local pattern, match, metadata = state.iter()
 
-      local range = { root_end_row + 1, 0, root_end_row + 1, 0 }
-      if node then
-        range = vim.treesitter.get_range(node, buf, metadata and metadata[capture])
+      if pattern and pattern > max_pattern_index then
+        max_pattern_index = pattern
       end
-      local start_row, start_col, end_row, end_col = Range.unpack4(range)
 
-      if capture then
-        local hl = state.highlighter_query:get_hl_from_capture(capture)
+      if not match then
+        state.next_row = root_end_row + 1
+      end
 
+      for capture, nodes in pairs(match or {}) do
         local capture_name = state.highlighter_query:query().captures[capture]
         local spell = nil ---@type boolean?
         if capture_name == 'spell' then
@@ -282,28 +291,60 @@ local function on_line_impl(self, buf, line, is_spell_nav)
           spell = false
         end
 
+        local hl = state.highlighter_query:get_hl_from_capture(capture)
+
         -- Give nospell a higher priority so it always overrides spell captures.
         local spell_pri_offset = capture_name == 'nospell' and 1 or 0
 
-        if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
-          local priority = (tonumber(metadata.priority) or vim.highlight.priorities.treesitter)
-            + spell_pri_offset
-          api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
-            end_line = end_row,
-            end_col = end_col,
-            hl_group = hl,
-            ephemeral = true,
-            priority = priority,
-            conceal = metadata.conceal,
-            spell = spell,
-          })
+        -- The "priority" attribute can be set at the pattern level or on a particular capture
+        local priority = (
+          tonumber(metadata.priority or metadata[capture] and metadata[capture].priority)
+          or vim.highlight.priorities.treesitter
+        ) + spell_pri_offset
+
+        local url = metadata[capture] and metadata[capture].url ---@type string|number|nil
+        if type(url) == 'number' then
+          if match and match[url] then
+            -- Assume there is only one matching node. If there is more than one, take the URL
+            -- from the first.
+            local other_node = match[url][1]
+            url = vim.treesitter.get_node_text(other_node, buf, {
+              metadata = metadata[url],
+            })
+          else
+            url = nil
+          end
         end
-      end
 
-      if start_row > line then
-        state.next_row = start_row
+        -- The "conceal" attribute can be set at the pattern level or on a particular capture
+        local conceal = metadata.conceal or metadata[capture] and metadata[capture].conceal
+
+        for _, node in ipairs(nodes) do
+          local range = vim.treesitter.get_range(node, buf, metadata[capture])
+          local start_row, start_col, end_row, end_col = Range.unpack4(range)
+
+          if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
+            api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
+              end_line = end_row,
+              end_col = end_col,
+              hl_group = hl,
+              ephemeral = true,
+              priority = priority,
+              _subpriority = pattern_offset + pattern,
+              conceal = conceal,
+              spell = spell,
+              url = url,
+            })
+          end
+
+          if start_row > line then
+            state.next_row = start_row
+          end
+        end
       end
     end
+
+    pattern_offset = pattern_offset + max_pattern_index
   end)
 end
 
-- 
cgit 


From dc7ccd6bca81dfa6ade6462a6e30770c63d48266 Mon Sep 17 00:00:00 2001
From: Gregory Anders <8965202+gpanders@users.noreply.github.com>
Date: Tue, 12 Mar 2024 16:13:40 -0500
Subject: fix(treesitter): use 0 as initial value for computing maximum
 (#27837)

Using -1 as the initial value can cause the pattern offset to become
negative, which in turn results in a negative subpriority, which fails
validation in nvim_buf_set_extmark.
---
 runtime/lua/vim/treesitter/highlighter.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index cc5e11d632..cbab5e990e 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -270,7 +270,7 @@ local function on_line_impl(self, buf, line, is_spell_nav)
         :iter_matches(root_node, self.bufnr, line, root_end_row + 1, { all = true })
     end
 
-    local max_pattern_index = -1
+    local max_pattern_index = 0
     while line >= state.next_row do
       local pattern, match, metadata = state.iter()
 
-- 
cgit 


From 12faaf40f487132b9397d9f3e59e44840985612c Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Wed, 13 Mar 2024 14:40:41 +0000
Subject: fix(treesitter): highlight injections properly

`on_line_impl` doesn't highlight single lines, so using pattern indexes
to offset priority doesn't work.
---
 runtime/lua/vim/treesitter/highlighter.lua  | 26 ++++++++++++--------------
 runtime/lua/vim/treesitter/languagetree.lua | 16 ++++++++++------
 2 files changed, 22 insertions(+), 20 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index cbab5e990e..6175977b49 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -57,6 +57,7 @@ end
 ---@field next_row integer
 ---@field iter vim.treesitter.highlighter.Iter?
 ---@field highlighter_query vim.treesitter.highlighter.Query
+---@field level integer Injection level
 
 ---@nodoc
 ---@class vim.treesitter.highlighter
@@ -192,12 +193,20 @@ function TSHighlighter:prepare_highlight_states(srow, erow)
       return
     end
 
+    local level = 0
+    local t = tree
+    while t do
+      t = t:parent()
+      level = level + 1
+    end
+
     -- _highlight_states should be a list so that the highlights are added in the same order as
     -- for_each_tree traversal. This ensures that parents' highlight don't override children's.
     table.insert(self._highlight_states, {
       tstree = tstree,
       next_row = 0,
       iter = nil,
+      level = level,
       highlighter_query = highlighter_query,
     })
   end)
@@ -248,14 +257,10 @@ end
 ---@param line integer
 ---@param is_spell_nav boolean
 local function on_line_impl(self, buf, line, is_spell_nav)
-  -- Track the maximum pattern index encountered in each tree. For subsequent
-  -- trees, the subpriority passed to nvim_buf_set_extmark is offset by the
-  -- largest pattern index from the prior tree. This ensures that extmarks
-  -- from subsequent trees always appear "on top of" extmarks from previous
-  -- trees (e.g. injections should always appear over base highlights).
-  local pattern_offset = 0
-
   self:for_each_highlight_state(function(state)
+    -- Use the injection level to offset the subpriority passed to nvim_buf_set_extmark
+    -- so injections always appear over base highlights.
+    local pattern_offset = state.level * 1000
     local root_node = state.tstree:root()
     local root_start_row, _, root_end_row, _ = root_node:range()
 
@@ -270,14 +275,9 @@ local function on_line_impl(self, buf, line, is_spell_nav)
         :iter_matches(root_node, self.bufnr, line, root_end_row + 1, { all = true })
     end
 
-    local max_pattern_index = 0
     while line >= state.next_row do
       local pattern, match, metadata = state.iter()
 
-      if pattern and pattern > max_pattern_index then
-        max_pattern_index = pattern
-      end
-
       if not match then
         state.next_row = root_end_row + 1
       end
@@ -343,8 +343,6 @@ local function on_line_impl(self, buf, line, is_spell_nav)
         end
       end
     end
-
-    pattern_offset = pattern_offset + max_pattern_index
   end)
 end
 
diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index 62714d3f1b..ec933f5194 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -81,7 +81,7 @@ local TSCallbackNames = {
 ---List of regions this tree should manage and parse. If nil then regions are
 ---taken from _trees. This is mostly a short-lived cache for included_regions()
 ---@field private _lang string Language name
----@field private _parent_lang? string Parent language name
+---@field private _parent? vim.treesitter.LanguageTree Parent LanguageTree
 ---@field private _source (integer|string) Buffer or string to parse
 ---@field private _trees table<integer, TSTree> Reference to parsed tree (one for each language).
 ---Each key is the index of region, which is synced with _regions and _valid.
@@ -106,9 +106,8 @@ LanguageTree.__index = LanguageTree
 ---@param source (integer|string) Buffer or text string to parse
 ---@param lang string Root language of this tree
 ---@param opts vim.treesitter.LanguageTree.new.Opts?
----@param parent_lang? string Parent language name of this tree
 ---@return vim.treesitter.LanguageTree parser object
-function LanguageTree.new(source, lang, opts, parent_lang)
+function LanguageTree.new(source, lang, opts)
   language.add(lang)
   opts = opts or {}
 
@@ -122,7 +121,6 @@ function LanguageTree.new(source, lang, opts, parent_lang)
   local self = {
     _source = source,
     _lang = lang,
-    _parent_lang = parent_lang,
     _children = {},
     _trees = {},
     _opts = opts,
@@ -505,19 +503,25 @@ function LanguageTree:add_child(lang)
     self:remove_child(lang)
   end
 
-  local child = LanguageTree.new(self._source, lang, self._opts, self:lang())
+  local child = LanguageTree.new(self._source, lang, self._opts)
 
   -- Inherit recursive callbacks
   for nm, cb in pairs(self._callbacks_rec) do
     vim.list_extend(child._callbacks_rec[nm], cb)
   end
 
+  child._parent = self
   self._children[lang] = child
   self:_do_callback('child_added', self._children[lang])
 
   return self._children[lang]
 end
 
+--- @package
+function LanguageTree:parent()
+  return self._parent
+end
+
 --- Removes a child language from this |LanguageTree|.
 ---
 ---@private
@@ -792,7 +796,7 @@ function LanguageTree:_get_injection(match, metadata)
   local combined = metadata['injection.combined'] ~= nil
   local injection_lang = metadata['injection.language'] --[[@as string?]]
   local lang = metadata['injection.self'] ~= nil and self:lang()
-    or metadata['injection.parent'] ~= nil and self._parent_lang
+    or metadata['injection.parent'] ~= nil and self._parent
     or (injection_lang and resolve_lang(injection_lang))
   local include_children = metadata['injection.include-children'] ~= nil
 
-- 
cgit 


From 00c4962cd241044c9f02de39b34ca24b2711de43 Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Wed, 13 Mar 2024 14:56:11 +0000
Subject: refactor(treesitter): move some logic into functions

---
 runtime/lua/vim/treesitter/highlighter.lua | 62 +++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 22 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index 6175977b49..7bc6e5c019 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -252,6 +252,44 @@ function TSHighlighter:get_query(lang)
   return self._queries[lang]
 end
 
+--- @param match table<integer,TSNode[]>
+--- @param bufnr integer
+--- @param capture integer
+--- @param metadata vim.treesitter.query.TSMetadata
+--- @return string?
+local function get_url(match, bufnr, capture, metadata)
+  ---@type string|number|nil
+  local url = metadata[capture] and metadata[capture].url
+
+  if not url or type(url) == 'string' then
+    return url
+  end
+
+  if not match or not match[url] then
+    return
+  end
+
+  -- Assume there is only one matching node. If there is more than one, take the URL
+  -- from the first.
+  local other_node = match[url][1]
+
+  return vim.treesitter.get_node_text(other_node, bufnr, {
+    metadata = metadata[url],
+  })
+end
+
+--- @param capture_name string
+--- @return boolean?, integer
+local function get_spell(capture_name)
+  if capture_name == 'spell' then
+    return true, 0
+  elseif capture_name == 'nospell' then
+    -- Give nospell a higher priority so it always overrides spell captures.
+    return false, 1
+  end
+  return nil, 0
+end
+
 ---@param self vim.treesitter.highlighter
 ---@param buf integer
 ---@param line integer
@@ -284,37 +322,17 @@ local function on_line_impl(self, buf, line, is_spell_nav)
 
       for capture, nodes in pairs(match or {}) do
         local capture_name = state.highlighter_query:query().captures[capture]
-        local spell = nil ---@type boolean?
-        if capture_name == 'spell' then
-          spell = true
-        elseif capture_name == 'nospell' then
-          spell = false
-        end
+        local spell, spell_pri_offset = get_spell(capture_name)
 
         local hl = state.highlighter_query:get_hl_from_capture(capture)
 
-        -- Give nospell a higher priority so it always overrides spell captures.
-        local spell_pri_offset = capture_name == 'nospell' and 1 or 0
-
         -- The "priority" attribute can be set at the pattern level or on a particular capture
         local priority = (
           tonumber(metadata.priority or metadata[capture] and metadata[capture].priority)
           or vim.highlight.priorities.treesitter
         ) + spell_pri_offset
 
-        local url = metadata[capture] and metadata[capture].url ---@type string|number|nil
-        if type(url) == 'number' then
-          if match and match[url] then
-            -- Assume there is only one matching node. If there is more than one, take the URL
-            -- from the first.
-            local other_node = match[url][1]
-            url = vim.treesitter.get_node_text(other_node, buf, {
-              metadata = metadata[url],
-            })
-          else
-            url = nil
-          end
-        end
+        local url = get_url(match, buf, capture, metadata)
 
         -- The "conceal" attribute can be set at the pattern level or on a particular capture
         local conceal = metadata.conceal or metadata[capture] and metadata[capture].conceal
-- 
cgit 


From 14e4b6bbd8640675d7393bdeb3e93d74ab875ff1 Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Sat, 16 Mar 2024 17:11:42 +0000
Subject: refactor(lua): type annotations

---
 runtime/lua/vim/treesitter/languagetree.lua | 38 ++++++++++++---------
 runtime/lua/vim/treesitter/query.lua        | 51 +++++++++++++----------------
 2 files changed, 45 insertions(+), 44 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index ec933f5194..3b5d8953c9 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -156,8 +156,10 @@ function LanguageTree:_set_logger()
 
   local lang = self:lang()
 
-  vim.fn.mkdir(vim.fn.stdpath('log'), 'p')
-  local logfilename = vim.fs.joinpath(vim.fn.stdpath('log'), 'treesitter.log')
+  local logdir = vim.fn.stdpath('log') --[[@as string]]
+
+  vim.fn.mkdir(logdir, 'p')
+  local logfilename = vim.fs.joinpath(logdir, 'treesitter.log')
 
   local logfile, openerr = io.open(logfilename, 'a+')
 
@@ -463,7 +465,7 @@ end
 --- Invokes the callback for each |LanguageTree| and its children recursively
 ---
 ---@param fn fun(tree: vim.treesitter.LanguageTree, lang: string)
----@param include_self boolean|nil Whether to include the invoking tree in the results
+---@param include_self? boolean Whether to include the invoking tree in the results
 function LanguageTree:for_each_child(fn, include_self)
   vim.deprecate('LanguageTree:for_each_child()', 'LanguageTree:children()', '0.11')
   if include_self then
@@ -796,7 +798,7 @@ function LanguageTree:_get_injection(match, metadata)
   local combined = metadata['injection.combined'] ~= nil
   local injection_lang = metadata['injection.language'] --[[@as string?]]
   local lang = metadata['injection.self'] ~= nil and self:lang()
-    or metadata['injection.parent'] ~= nil and self._parent
+    or metadata['injection.parent'] ~= nil and self._parent:lang()
     or (injection_lang and resolve_lang(injection_lang))
   local include_children = metadata['injection.include-children'] ~= nil
 
@@ -1058,20 +1060,19 @@ function LanguageTree:_on_detach(...)
   end
 end
 
---- Registers callbacks for the |LanguageTree|.
----@param cbs table An |nvim_buf_attach()|-like table argument with the following handlers:
----           - `on_bytes` : see |nvim_buf_attach()|, but this will be called _after_ the parsers callback.
+--- Registers callbacks for the [LanguageTree].
+---@param cbs table<TSCallbackNameOn,function> An [nvim_buf_attach()]-like table argument with the following handlers:
+---           - `on_bytes` : see [nvim_buf_attach()], but this will be called _after_ the parsers callback.
 ---           - `on_changedtree` : a callback that will be called every time the tree has syntactical changes.
 ---              It will be passed two arguments: a table of the ranges (as node ranges) that
 ---              changed and the changed tree.
 ---           - `on_child_added` : emitted when a child is added to the tree.
 ---           - `on_child_removed` : emitted when a child is removed from the tree.
----           - `on_detach` : emitted when the buffer is detached, see |nvim_buf_detach_event|.
+---           - `on_detach` : emitted when the buffer is detached, see [nvim_buf_detach_event].
 ---              Takes one argument, the number of the buffer.
 --- @param recursive? boolean Apply callbacks recursively for all children. Any new children will
 ---                           also inherit the callbacks.
 function LanguageTree:register_cbs(cbs, recursive)
-  ---@cast cbs table<TSCallbackNameOn,function>
   if not cbs then
     return
   end
@@ -1112,12 +1113,18 @@ function LanguageTree:contains(range)
   return false
 end
 
+--- @class vim.treesitter.LanguageTree.tree_for_range.Opts
+--- @inlinedoc
+---
+--- Ignore injected languages
+--- (default: `true`)
+--- @field ignore_injections? boolean
+
 --- Gets the tree that contains {range}.
 ---
 ---@param range Range4 `{ start_line, start_col, end_line, end_col }`
----@param opts table|nil Optional keyword arguments:
----             - ignore_injections boolean Ignore injected languages (default true)
----@return TSTree|nil
+---@param opts? vim.treesitter.LanguageTree.tree_for_range.Opts
+---@return TSTree?
 function LanguageTree:tree_for_range(range, opts)
   opts = opts or {}
   local ignore = vim.F.if_nil(opts.ignore_injections, true)
@@ -1143,9 +1150,8 @@ end
 --- Gets the smallest named node that contains {range}.
 ---
 ---@param range Range4 `{ start_line, start_col, end_line, end_col }`
----@param opts table|nil Optional keyword arguments:
----             - ignore_injections boolean Ignore injected languages (default true)
----@return TSNode | nil Found node
+---@param opts? vim.treesitter.LanguageTree.tree_for_range.Opts
+---@return TSNode?
 function LanguageTree:named_node_for_range(range, opts)
   local tree = self:tree_for_range(range, opts)
   if tree then
@@ -1156,7 +1162,7 @@ end
 --- Gets the appropriate language that contains {range}.
 ---
 ---@param range Range4 `{ start_line, start_col, end_line, end_col }`
----@return vim.treesitter.LanguageTree Managing {range}
+---@return vim.treesitter.LanguageTree tree Managing {range}
 function LanguageTree:language_for_range(range)
   for _, child in pairs(self._children) do
     if child:contains(range) then
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index a086f5e876..67b8c596b8 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -88,7 +88,7 @@ end
 ---
 ---@param lang string Language to get query for
 ---@param query_name string Name of the query to load (e.g., "highlights")
----@param is_included (boolean|nil) Internal parameter, most of the time left as `nil`
+---@param is_included? boolean Internal parameter, most of the time left as `nil`
 ---@return string[] query_files List of files to load for given query and language
 function M.get_files(lang, query_name, is_included)
   local query_path = string.format('queries/%s/%s.scm', lang, query_name)
@@ -211,7 +211,7 @@ end
 ---@param lang string Language to use for the query
 ---@param query_name string Name of the query (e.g. "highlights")
 ---
----@return vim.treesitter.Query|nil : Parsed query. `nil` if no query files are found.
+---@return vim.treesitter.Query? : Parsed query. `nil` if no query files are found.
 M.get = vim.func._memoize('concat-2', function(lang, query_name)
   if explicit_queries[lang][query_name] then
     return explicit_queries[lang][query_name]
@@ -242,9 +242,9 @@ end)
 ---@param lang string Language to use for the query
 ---@param query string Query in s-expr syntax
 ---
----@return vim.treesitter.Query Parsed query
+---@return vim.treesitter.Query : Parsed query
 ---
----@see |vim.treesitter.query.get()|
+---@see [vim.treesitter.query.get()]
 M.parse = vim.func._memoize('concat-2', function(lang, query)
   language.add(lang)
 
@@ -618,20 +618,23 @@ local directive_handlers = {
   end,
 }
 
+--- @class vim.treesitter.query.add_predicate.Opts
+--- @inlinedoc
+---
+--- Override an existing predicate of the same name
+--- @field force? boolean
+---
+--- Use the correct implementation of the match table where capture IDs map to
+--- a list of nodes instead of a single node. Defaults to false (for backward
+--- compatibility). This option will eventually become the default and removed.
+--- @field all? boolean
+
 --- Adds a new predicate to be used in queries
 ---
 ---@param name string Name of the predicate, without leading #
----@param handler function(match: table<integer,TSNode[]>, pattern: integer, source: integer|string, predicate: any[], metadata: table)
+---@param handler fun(match: table<integer,TSNode[]>, pattern: integer, source: integer|string, predicate: any[], metadata: table)
 ---   - see |vim.treesitter.query.add_directive()| for argument meanings
----@param opts table<string, any> Optional options:
----                                 - force (boolean): Override an existing
----                                   predicate of the same name
----                                 - all (boolean): Use the correct
----                                   implementation of the match table where
----                                   capture IDs map to a list of nodes instead
----                                   of a single node. Defaults to false (for
----                                   backward compatibility). This option will
----                                   eventually become the default and removed.
+---@param opts vim.treesitter.query.add_predicate.Opts
 function M.add_predicate(name, handler, opts)
   -- Backward compatibility: old signature had "force" as boolean argument
   if type(opts) == 'boolean' then
@@ -669,20 +672,12 @@ end
 --- metadata table `metadata[capture_id].key = value`
 ---
 ---@param name string Name of the directive, without leading #
----@param handler function(match: table<integer,TSNode[]>, pattern: integer, source: integer|string, predicate: any[], metadata: table)
+---@param handler fun(match: table<integer,TSNode[]>, pattern: integer, source: integer|string, predicate: any[], metadata: table)
 ---   - match: A table mapping capture IDs to a list of captured nodes
 ---   - pattern: the index of the matching pattern in the query file
 ---   - predicate: list of strings containing the full directive being called, e.g.
 ---     `(node (#set! conceal "-"))` would get the predicate `{ "#set!", "conceal", "-" }`
----@param opts table<string, any> Optional options:
----                                 - force (boolean): Override an existing
----                                   predicate of the same name
----                                 - all (boolean): Use the correct
----                                   implementation of the match table where
----                                   capture IDs map to a list of nodes instead
----                                   of a single node. Defaults to false (for
----                                   backward compatibility). This option will
----                                   eventually become the default and removed.
+---@param opts vim.treesitter.query.add_predicate.Opts
 function M.add_directive(name, handler, opts)
   -- Backward compatibility: old signature had "force" as boolean argument
   if type(opts) == 'boolean' then
@@ -711,13 +706,13 @@ function M.add_directive(name, handler, opts)
 end
 
 --- Lists the currently available directives to use in queries.
----@return string[] List of supported directives.
+---@return string[] : Supported directives.
 function M.list_directives()
   return vim.tbl_keys(directive_handlers)
 end
 
 --- Lists the currently available predicates to use in queries.
----@return string[] List of supported predicates.
+---@return string[] : Supported predicates.
 function M.list_predicates()
   return vim.tbl_keys(predicate_handlers)
 end
@@ -792,8 +787,8 @@ end
 --- Returns the start and stop value if set else the node's range.
 -- When the node's range is used, the stop is incremented by 1
 -- to make the search inclusive.
----@param start integer|nil
----@param stop integer|nil
+---@param start integer?
+---@param stop integer?
 ---@param node TSNode
 ---@return integer, integer
 local function value_or_node_range(start, stop, node)
-- 
cgit 


From 3b29b39e6deb212456eba691bc79b17edaa8717b Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Sun, 17 Mar 2024 18:02:40 +0000
Subject: fix(treesitter): revert to using iter_captures in highlighter

Fixes #27895
---
 runtime/lua/vim/treesitter/highlighter.lua | 74 ++++++++++++------------------
 runtime/lua/vim/treesitter/query.lua       | 13 +++---
 2 files changed, 36 insertions(+), 51 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index 7bc6e5c019..1e6f128461 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -4,7 +4,7 @@ local Range = require('vim.treesitter._range')
 
 local ns = api.nvim_create_namespace('treesitter/highlighter')
 
----@alias vim.treesitter.highlighter.Iter fun(): integer, table<integer, TSNode[]>, vim.treesitter.query.TSMetadata
+---@alias vim.treesitter.highlighter.Iter fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, table<integer, TSNode[]>
 
 ---@class (private) vim.treesitter.highlighter.Query
 ---@field private _query vim.treesitter.Query?
@@ -57,7 +57,6 @@ end
 ---@field next_row integer
 ---@field iter vim.treesitter.highlighter.Iter?
 ---@field highlighter_query vim.treesitter.highlighter.Query
----@field level integer Injection level
 
 ---@nodoc
 ---@class vim.treesitter.highlighter
@@ -193,20 +192,12 @@ function TSHighlighter:prepare_highlight_states(srow, erow)
       return
     end
 
-    local level = 0
-    local t = tree
-    while t do
-      t = t:parent()
-      level = level + 1
-    end
-
     -- _highlight_states should be a list so that the highlights are added in the same order as
     -- for_each_tree traversal. This ensures that parents' highlight don't override children's.
     table.insert(self._highlight_states, {
       tstree = tstree,
       next_row = 0,
       iter = nil,
-      level = level,
       highlighter_query = highlighter_query,
     })
   end)
@@ -296,9 +287,6 @@ end
 ---@param is_spell_nav boolean
 local function on_line_impl(self, buf, line, is_spell_nav)
   self:for_each_highlight_state(function(state)
-    -- Use the injection level to offset the subpriority passed to nvim_buf_set_extmark
-    -- so injections always appear over base highlights.
-    local pattern_offset = state.level * 1000
     local root_node = state.tstree:root()
     local root_start_row, _, root_end_row, _ = root_node:range()
 
@@ -308,23 +296,25 @@ local function on_line_impl(self, buf, line, is_spell_nav)
     end
 
     if state.iter == nil or state.next_row < line then
-      state.iter = state.highlighter_query
-        :query()
-        :iter_matches(root_node, self.bufnr, line, root_end_row + 1, { all = true })
+      state.iter =
+        state.highlighter_query:query():iter_captures(root_node, self.bufnr, line, root_end_row + 1)
     end
 
     while line >= state.next_row do
-      local pattern, match, metadata = state.iter()
+      local capture, node, metadata, match = state.iter(line)
 
-      if not match then
-        state.next_row = root_end_row + 1
+      local range = { root_end_row + 1, 0, root_end_row + 1, 0 }
+      if node then
+        range = vim.treesitter.get_range(node, buf, metadata and metadata[capture])
       end
+      local start_row, start_col, end_row, end_col = Range.unpack4(range)
+
+      if capture then
+        local hl = state.highlighter_query:get_hl_from_capture(capture)
 
-      for capture, nodes in pairs(match or {}) do
         local capture_name = state.highlighter_query:query().captures[capture]
-        local spell, spell_pri_offset = get_spell(capture_name)
 
-        local hl = state.highlighter_query:get_hl_from_capture(capture)
+        local spell, spell_pri_offset = get_spell(capture_name)
 
         -- The "priority" attribute can be set at the pattern level or on a particular capture
         local priority = (
@@ -332,34 +322,28 @@ local function on_line_impl(self, buf, line, is_spell_nav)
           or vim.highlight.priorities.treesitter
         ) + spell_pri_offset
 
-        local url = get_url(match, buf, capture, metadata)
-
         -- The "conceal" attribute can be set at the pattern level or on a particular capture
         local conceal = metadata.conceal or metadata[capture] and metadata[capture].conceal
 
-        for _, node in ipairs(nodes) do
-          local range = vim.treesitter.get_range(node, buf, metadata[capture])
-          local start_row, start_col, end_row, end_col = Range.unpack4(range)
-
-          if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
-            api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
-              end_line = end_row,
-              end_col = end_col,
-              hl_group = hl,
-              ephemeral = true,
-              priority = priority,
-              _subpriority = pattern_offset + pattern,
-              conceal = conceal,
-              spell = spell,
-              url = url,
-            })
-          end
-
-          if start_row > line then
-            state.next_row = start_row
-          end
+        local url = get_url(match, buf, capture, metadata)
+
+        if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
+          api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
+            end_line = end_row,
+            end_col = end_col,
+            hl_group = hl,
+            ephemeral = true,
+            priority = priority,
+            conceal = conceal,
+            spell = spell,
+            url = url,
+          })
         end
       end
+
+      if start_row > line then
+        state.next_row = start_row
+      end
     end
   end)
 end
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 67b8c596b8..30cd00c617 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -811,12 +811,13 @@ end
 --- as the {node}, i.e., to get syntax highlight matches in the current
 --- viewport). When omitted, the {start} and {stop} row values are used from the given node.
 ---
---- The iterator returns three values: a numeric id identifying the capture,
---- the captured node, and metadata from any directives processing the match.
+--- The iterator returns four values: a numeric id identifying the capture,
+--- the captured node, metadata from any directives processing the match,
+--- and the match itself.
 --- The following example shows how to get captures by name:
 ---
 --- ```lua
---- for id, node, metadata in query:iter_captures(tree:root(), bufnr, first, last) do
+--- for id, node, metadata, match in query:iter_captures(tree:root(), bufnr, first, last) do
 ---   local name = query.captures[id] -- name of the capture in the query
 ---   -- typically useful info about the node:
 ---   local type = node:type() -- type of the captured node
@@ -830,8 +831,8 @@ end
 ---@param start? integer Starting line for the search. Defaults to `node:start()`.
 ---@param stop? integer Stopping line for the search (end-exclusive). Defaults to `node:end_()`.
 ---
----@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata):
----        capture id, capture node, metadata
+---@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, table<integer, TSNode>):
+---        capture id, capture node, metadata, match
 function Query:iter_captures(node, source, start, stop)
   if type(source) == 'number' and source == 0 then
     source = api.nvim_get_current_buf()
@@ -856,7 +857,7 @@ function Query:iter_captures(node, source, start, stop)
 
       self:apply_directives(match, match.pattern, source, metadata)
     end
-    return capture, captured_node, metadata
+    return capture, captured_node, metadata, match
   end
   return iter
 end
-- 
cgit 


From aca2048bcd57937ea1c7b7f0325f25d5b82588db Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Mon, 18 Mar 2024 23:19:01 +0000
Subject: refactor(treesitter): redesign query iterating

Problem:

  `TSNode:_rawquery()` is complicated, has known issues and the Lua and
  C code is awkwardly coupled (see logic with `active`).

Solution:

  - Add `TSQueryCursor` and `TSQueryMatch` bindings.
  - Replace `TSNode:_rawquery()` with `TSQueryCursor:next_capture()` and `TSQueryCursor:next_match()`
  - Do more stuff in Lua
  - API for `Query:iter_captures()` and `Query:iter_matches()` remains the same.
  - `treesitter.c` no longer contains any logic related to predicates.
  - Add `match_limit` option to `iter_matches()`. Default is still 256.
---
 runtime/lua/vim/treesitter/_meta.lua         |  44 ++++++----
 runtime/lua/vim/treesitter/_query_linter.lua |   2 +-
 runtime/lua/vim/treesitter/query.lua         | 125 +++++++++++++++++----------
 3 files changed, 109 insertions(+), 62 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/_meta.lua b/runtime/lua/vim/treesitter/_meta.lua
index 19d97d2820..e2768d4b06 100644
--- a/runtime/lua/vim/treesitter/_meta.lua
+++ b/runtime/lua/vim/treesitter/_meta.lua
@@ -34,22 +34,6 @@ error('Cannot require a meta file')
 ---@field byte_length fun(self: TSNode): integer
 local TSNode = {}
 
----@param query TSQuery
----@param captures true
----@param start? integer
----@param end_? integer
----@param opts? table
----@return fun(): integer, TSNode, vim.treesitter.query.TSMatch
-function TSNode:_rawquery(query, captures, start, end_, opts) end
-
----@param query TSQuery
----@param captures false
----@param start? integer
----@param end_? integer
----@param opts? table
----@return fun(): integer, vim.treesitter.query.TSMatch
-function TSNode:_rawquery(query, captures, start, end_, opts) end
-
 ---@alias TSLoggerCallback fun(logtype: 'parse'|'lex', msg: string)
 
 ---@class TSParser: userdata
@@ -90,3 +74,31 @@ vim._ts_parse_query = function(lang, query) end
 ---@param lang string
 ---@return TSParser
 vim._create_ts_parser = function(lang) end
+
+--- @class TSQueryMatch: userdata
+--- @field captures fun(self: TSQueryMatch): table<integer,TSNode[]>
+local TSQueryMatch = {}
+
+--- @return integer match_id
+--- @return integer pattern_index
+function TSQueryMatch:info() end
+
+--- @class TSQueryCursor: userdata
+--- @field remove_match fun(self: TSQueryCursor, id: integer)
+local TSQueryCursor = {}
+
+--- @return integer capture
+--- @return TSNode captured_node
+--- @return TSQueryMatch match
+function TSQueryCursor:next_capture() end
+
+--- @return TSQueryMatch match
+function TSQueryCursor:next_match() end
+
+--- @param node TSNode
+--- @param query TSQuery
+--- @param start integer?
+--- @param stop integer?
+--- @param opts? { max_start_depth?: integer, match_limit?: integer}
+--- @return TSQueryCursor
+function vim._create_ts_querycursor(node, query, start, stop, opts) end
diff --git a/runtime/lua/vim/treesitter/_query_linter.lua b/runtime/lua/vim/treesitter/_query_linter.lua
index 6216d4e891..12b4cbc7b9 100644
--- a/runtime/lua/vim/treesitter/_query_linter.lua
+++ b/runtime/lua/vim/treesitter/_query_linter.lua
@@ -122,7 +122,7 @@ local parse = vim.func._memoize(hash_parse, function(node, buf, lang)
 end)
 
 --- @param buf integer
---- @param match vim.treesitter.query.TSMatch
+--- @param match table<integer,TSNode[]>
 --- @param query vim.treesitter.Query
 --- @param lang_context QueryLinterLanguageContext
 --- @param diagnostics vim.Diagnostic[]
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 30cd00c617..075fd0e99b 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -258,7 +258,7 @@ end)
 --- handling the "any" vs "all" semantics. They are called from the
 --- predicate_handlers table with the appropriate arguments for each predicate.
 local impl = {
-  --- @param match vim.treesitter.query.TSMatch
+  --- @param match table<integer,TSNode[]>
   --- @param source integer|string
   --- @param predicate any[]
   --- @param any boolean
@@ -293,7 +293,7 @@ local impl = {
     return not any
   end,
 
-  --- @param match vim.treesitter.query.TSMatch
+  --- @param match table<integer,TSNode[]>
   --- @param source integer|string
   --- @param predicate any[]
   --- @param any boolean
@@ -333,7 +333,7 @@ local impl = {
       end,
     })
 
-    --- @param match vim.treesitter.query.TSMatch
+    --- @param match table<integer,TSNode[]>
     --- @param source integer|string
     --- @param predicate any[]
     --- @param any boolean
@@ -356,7 +356,7 @@ local impl = {
     end
   end)(),
 
-  --- @param match vim.treesitter.query.TSMatch
+  --- @param match table<integer,TSNode[]>
   --- @param source integer|string
   --- @param predicate any[]
   --- @param any boolean
@@ -383,13 +383,7 @@ local impl = {
   end,
 }
 
----@nodoc
----@class vim.treesitter.query.TSMatch
----@field pattern? integer
----@field active? boolean
----@field [integer] TSNode[]
-
----@alias TSPredicate fun(match: vim.treesitter.query.TSMatch, pattern: integer, source: integer|string, predicate: any[]): boolean
+---@alias TSPredicate fun(match: table<integer,TSNode[]>, pattern: integer, source: integer|string, predicate: any[]): boolean
 
 -- Predicate handler receive the following arguments
 -- (match, pattern, bufnr, predicate)
@@ -504,7 +498,7 @@ predicate_handlers['any-vim-match?'] = predicate_handlers['any-match?']
 ---@field [integer] vim.treesitter.query.TSMetadata
 ---@field [string] integer|string
 
----@alias TSDirective fun(match: vim.treesitter.query.TSMatch, _, _, predicate: (string|integer)[], metadata: vim.treesitter.query.TSMetadata)
+---@alias TSDirective fun(match: table<integer,TSNode[]>, _, _, predicate: (string|integer)[], metadata: vim.treesitter.query.TSMetadata)
 
 -- Predicate handler receive the following arguments
 -- (match, pattern, bufnr, predicate)
@@ -726,13 +720,19 @@ local function is_directive(name)
 end
 
 ---@private
----@param match vim.treesitter.query.TSMatch
----@param pattern integer
+---@param match TSQueryMatch
 ---@param source integer|string
-function Query:match_preds(match, pattern, source)
+function Query:match_preds(match, source)
+  local _, pattern = match:info()
   local preds = self.info.patterns[pattern]
 
-  for _, pred in pairs(preds or {}) do
+  if not preds then
+    return true
+  end
+
+  local captures = match:captures()
+
+  for _, pred in pairs(preds) do
     -- Here we only want to return if a predicate DOES NOT match, and
     -- continue on the other case. This way unknown predicates will not be considered,
     -- which allows some testing and easier user extensibility (#12173).
@@ -754,7 +754,7 @@ function Query:match_preds(match, pattern, source)
         return false
       end
 
-      local pred_matches = handler(match, pattern, source, pred)
+      local pred_matches = handler(captures, pattern, source, pred)
 
       if not xor(is_not, pred_matches) then
         return false
@@ -765,23 +765,33 @@ function Query:match_preds(match, pattern, source)
 end
 
 ---@private
----@param match vim.treesitter.query.TSMatch
----@param metadata vim.treesitter.query.TSMetadata
-function Query:apply_directives(match, pattern, source, metadata)
+---@param match TSQueryMatch
+---@return vim.treesitter.query.TSMetadata metadata
+function Query:apply_directives(match, source)
+  ---@type vim.treesitter.query.TSMetadata
+  local metadata = {}
+  local _, pattern = match:info()
   local preds = self.info.patterns[pattern]
 
-  for _, pred in pairs(preds or {}) do
+  if not preds then
+    return metadata
+  end
+
+  local captures = match:captures()
+
+  for _, pred in pairs(preds) do
     if is_directive(pred[1]) then
       local handler = directive_handlers[pred[1]]
 
       if not handler then
         error(string.format('No handler for %s', pred[1]))
-        return
       end
 
-      handler(match, pattern, source, pred, metadata)
+      handler(captures, pattern, source, pred, metadata)
     end
   end
+
+  return metadata
 end
 
 --- Returns the start and stop value if set else the node's range.
@@ -831,8 +841,10 @@ end
 ---@param start? integer Starting line for the search. Defaults to `node:start()`.
 ---@param stop? integer Stopping line for the search (end-exclusive). Defaults to `node:end_()`.
 ---
----@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, table<integer, TSNode>):
+---@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, table<integer,TSNode[]>?):
 ---        capture id, capture node, metadata, match
+---
+---@note Captures are only returned if the query pattern of a specific capture contained predicates.
 function Query:iter_captures(node, source, start, stop)
   if type(source) == 'number' and source == 0 then
     source = api.nvim_get_current_buf()
@@ -840,24 +852,38 @@ function Query:iter_captures(node, source, start, stop)
 
   start, stop = value_or_node_range(start, stop, node)
 
-  local raw_iter = node:_rawquery(self.query, true, start, stop) ---@type fun(): integer, TSNode, vim.treesitter.query.TSMatch
+  local cursor = vim._create_ts_querycursor(node, self.query, start, stop, { match_limit = 256 })
+
+  local max_match_id = -1
+
   local function iter(end_line)
-    local capture, captured_node, match = raw_iter()
+    local capture, captured_node, match = cursor:next_capture()
+
+    if not capture then
+      return
+    end
+
+    local captures --- @type table<integer,TSNode[]>?
+    local match_id, pattern_index = match:info()
+
     local metadata = {}
 
-    if match ~= nil then
-      local active = self:match_preds(match, match.pattern, source)
-      match.active = active
-      if not active then
+    local preds = self.info.patterns[pattern_index] or {}
+
+    if #preds > 0 and match_id > max_match_id then
+      captures = match:captures()
+      max_match_id = match_id
+      if not self:match_preds(match, source) then
+        cursor:remove_match(match_id)
         if end_line and captured_node:range() > end_line then
           return nil, captured_node, nil
         end
         return iter(end_line) -- tail call: try next match
       end
 
-      self:apply_directives(match, match.pattern, source, metadata)
+      metadata = self:apply_directives(match, source)
     end
-    return capture, captured_node, metadata, match
+    return capture, captured_node, metadata, captures
   end
   return iter
 end
@@ -899,45 +925,54 @@ end
 ---@param opts? table Optional keyword arguments:
 ---   - max_start_depth (integer) if non-zero, sets the maximum start depth
 ---     for each match. This is used to prevent traversing too deep into a tree.
+---   - match_limit (integer) Set the maximum number of in-progress matches (Default: 256).
 ---   - all (boolean) When set, the returned match table maps capture IDs to a list of nodes.
 ---     Older versions of iter_matches incorrectly mapped capture IDs to a single node, which is
 ---     incorrect behavior. This option will eventually become the default and removed.
 ---
 ---@return (fun(): integer, table<integer, TSNode[]>, table): pattern id, match, metadata
 function Query:iter_matches(node, source, start, stop, opts)
-  local all = opts and opts.all
+  opts = opts or {}
+  opts.match_limit = opts.match_limit or 256
+
   if type(source) == 'number' and source == 0 then
     source = api.nvim_get_current_buf()
   end
 
   start, stop = value_or_node_range(start, stop, node)
 
-  local raw_iter = node:_rawquery(self.query, false, start, stop, opts) ---@type fun(): integer, vim.treesitter.query.TSMatch
+  local cursor = vim._create_ts_querycursor(node, self.query, start, stop, opts)
+
   local function iter()
-    local pattern, match = raw_iter()
-    local metadata = {}
+    local match = cursor:next_match()
 
-    if match ~= nil then
-      local active = self:match_preds(match, pattern, source)
-      if not active then
-        return iter() -- tail call: try next match
-      end
+    if not match then
+      return
+    end
 
-      self:apply_directives(match, pattern, source, metadata)
+    local match_id, pattern = match:info()
+
+    if not self:match_preds(match, source) then
+      cursor:remove_match(match_id)
+      return iter() -- tail call: try next match
     end
 
-    if not all then
+    local metadata = self:apply_directives(match, source)
+
+    local captures = match:captures()
+
+    if not opts.all then
       -- Convert the match table into the old buggy version for backward
       -- compatibility. This is slow. Plugin authors, if you're reading this, set the "all"
       -- option!
       local old_match = {} ---@type table<integer, TSNode>
-      for k, v in pairs(match or {}) do
+      for k, v in pairs(captures or {}) do
         old_match[k] = v[#v]
       end
       return pattern, old_match, metadata
     end
 
-    return pattern, match, metadata
+    return pattern, captures, metadata
   end
   return iter
 end
-- 
cgit 


From 7d971500847089ec8ade926a7f84d6bb3a51c8b0 Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Mon, 25 Mar 2024 22:06:31 +0000
Subject: fix(treesitter): return correct match table in iter_captures()

---
 runtime/lua/vim/treesitter/highlighter.lua | 14 ++++++---
 runtime/lua/vim/treesitter/query.lua       | 46 +++++++++++++++---------------
 2 files changed, 33 insertions(+), 27 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index 1e6f128461..3f7e31212c 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -4,7 +4,7 @@ local Range = require('vim.treesitter._range')
 
 local ns = api.nvim_create_namespace('treesitter/highlighter')
 
----@alias vim.treesitter.highlighter.Iter fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, table<integer, TSNode[]>
+---@alias vim.treesitter.highlighter.Iter fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, TSQueryMatch
 
 ---@class (private) vim.treesitter.highlighter.Query
 ---@field private _query vim.treesitter.Query?
@@ -243,7 +243,7 @@ function TSHighlighter:get_query(lang)
   return self._queries[lang]
 end
 
---- @param match table<integer,TSNode[]>
+--- @param match TSQueryMatch
 --- @param bufnr integer
 --- @param capture integer
 --- @param metadata vim.treesitter.query.TSMetadata
@@ -256,13 +256,15 @@ local function get_url(match, bufnr, capture, metadata)
     return url
   end
 
-  if not match or not match[url] then
+  local captures = match:captures()
+
+  if not captures[url] then
     return
   end
 
   -- Assume there is only one matching node. If there is more than one, take the URL
   -- from the first.
-  local other_node = match[url][1]
+  local other_node = captures[url][1]
 
   return vim.treesitter.get_node_text(other_node, bufnr, {
     metadata = metadata[url],
@@ -296,6 +298,10 @@ local function on_line_impl(self, buf, line, is_spell_nav)
     end
 
     if state.iter == nil or state.next_row < line then
+      -- Mainly used to skip over folds
+
+      -- TODO(lewis6991): Creating a new iterator loses the cached predicate results for query
+      -- matches. Move this logic inside iter_captures() so we can maintain the cache.
       state.iter =
         state.highlighter_query:query():iter_captures(root_node, self.bufnr, line, root_end_row + 1)
     end
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 075fd0e99b..e68acac929 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -1,5 +1,6 @@
 local api = vim.api
 local language = require('vim.treesitter.language')
+local memoize = vim.func._memoize
 
 local M = {}
 
@@ -212,7 +213,7 @@ end
 ---@param query_name string Name of the query (e.g. "highlights")
 ---
 ---@return vim.treesitter.Query? : Parsed query. `nil` if no query files are found.
-M.get = vim.func._memoize('concat-2', function(lang, query_name)
+M.get = memoize('concat-2', function(lang, query_name)
   if explicit_queries[lang][query_name] then
     return explicit_queries[lang][query_name]
   end
@@ -245,7 +246,7 @@ end)
 ---@return vim.treesitter.Query : Parsed query
 ---
 ---@see [vim.treesitter.query.get()]
-M.parse = vim.func._memoize('concat-2', function(lang, query)
+M.parse = memoize('concat-2', function(lang, query)
   language.add(lang)
 
   local ts_query = vim._ts_parse_query(lang, query)
@@ -812,6 +813,12 @@ local function value_or_node_range(start, stop, node)
   return start, stop
 end
 
+--- @param match TSQueryMatch
+--- @return integer
+local function match_id_hash(_, match)
+  return (match:info())
+end
+
 --- Iterate over all captures from all matches inside {node}
 ---
 --- {source} is needed if the query contains predicates; then the caller
@@ -841,7 +848,7 @@ end
 ---@param start? integer Starting line for the search. Defaults to `node:start()`.
 ---@param stop? integer Stopping line for the search (end-exclusive). Defaults to `node:end_()`.
 ---
----@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, table<integer,TSNode[]>?):
+---@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, TSQueryMatch):
 ---        capture id, capture node, metadata, match
 ---
 ---@note Captures are only returned if the query pattern of a specific capture contained predicates.
@@ -854,7 +861,8 @@ function Query:iter_captures(node, source, start, stop)
 
   local cursor = vim._create_ts_querycursor(node, self.query, start, stop, { match_limit = 256 })
 
-  local max_match_id = -1
+  local apply_directives = memoize(match_id_hash, self.apply_directives, true)
+  local match_preds = memoize(match_id_hash, self.match_preds, true)
 
   local function iter(end_line)
     local capture, captured_node, match = cursor:next_capture()
@@ -863,27 +871,18 @@ function Query:iter_captures(node, source, start, stop)
       return
     end
 
-    local captures --- @type table<integer,TSNode[]>?
-    local match_id, pattern_index = match:info()
-
-    local metadata = {}
-
-    local preds = self.info.patterns[pattern_index] or {}
-
-    if #preds > 0 and match_id > max_match_id then
-      captures = match:captures()
-      max_match_id = match_id
-      if not self:match_preds(match, source) then
-        cursor:remove_match(match_id)
-        if end_line and captured_node:range() > end_line then
-          return nil, captured_node, nil
-        end
-        return iter(end_line) -- tail call: try next match
+    if not match_preds(self, match, source) then
+      local match_id = match:info()
+      cursor:remove_match(match_id)
+      if end_line and captured_node:range() > end_line then
+        return nil, captured_node, nil, nil
       end
-
-      metadata = self:apply_directives(match, source)
+      return iter(end_line) -- tail call: try next match
     end
-    return capture, captured_node, metadata, captures
+
+    local metadata = apply_directives(self, match, source)
+
+    return capture, captured_node, metadata, match
   end
   return iter
 end
@@ -972,6 +971,7 @@ function Query:iter_matches(node, source, start, stop, opts)
       return pattern, old_match, metadata
     end
 
+    -- TODO(lewis6991): create a new function that returns {match, metadata}
     return pattern, captures, metadata
   end
   return iter
-- 
cgit 


From 6cfca21bac6bb39b50cba1c23ffb2b69e2d94df8 Mon Sep 17 00:00:00 2001
From: Christian Clason <c.clason@uni-graz.at>
Date: Tue, 2 Apr 2024 10:54:40 +0200
Subject: feat(treesitter): add `@injection.filename`

Problem: Injecting languages for file redirects (e.g., in bash) is not
possible.

Solution: Add `@injection.filename` capture that is piped through
`vim.filetype.match({ filename = node_text })`; the resulting filetype
(if not `nil`) is then resolved as a language (either directly or
through the list maintained via `vim.treesitter.language.register()`).

Note: `@injection.filename` is a non-standard capture introduced by
Helix; having two editors implement it makes it likely to be upstreamed.
---
 runtime/lua/vim/treesitter/languagetree.lua | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index 3b5d8953c9..8f65cb57c3 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -809,6 +809,10 @@ function LanguageTree:_get_injection(match, metadata)
       if name == 'injection.language' then
         local text = vim.treesitter.get_node_text(node, self._source, { metadata = metadata[id] })
         lang = resolve_lang(text)
+      elseif name == 'injection.filename' then
+        local text = vim.treesitter.get_node_text(node, self._source, { metadata = metadata[id] })
+        local ft = vim.filetype.match({ filename = text })
+        lang = ft and resolve_lang(ft)
       elseif name == 'injection.content' then
         ranges = get_node_ranges(node, self._source, metadata[id], include_children)
       end
-- 
cgit 


From 00e6651880c32a9878797eeeaef7018c3d5d99b7 Mon Sep 17 00:00:00 2001
From: altermo <107814000+altermo@users.noreply.github.com>
Date: Wed, 10 Apr 2024 10:52:51 +0200
Subject: fix(treesitter): use tree range instead of tree root node range

---
 runtime/lua/vim/treesitter/languagetree.lua | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index 8f65cb57c3..990debc77b 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -1100,7 +1100,14 @@ end
 ---@param range Range
 ---@return boolean
 local function tree_contains(tree, range)
-  return Range.contains({ tree:root():range() }, range)
+  local tree_ranges = tree:included_ranges(false)
+
+  return Range.contains({
+    tree_ranges[1][1],
+    tree_ranges[1][2],
+    tree_ranges[#tree_ranges][3],
+    tree_ranges[#tree_ranges][4],
+  }, range)
 end
 
 --- Determines whether {range} is contained in the |LanguageTree|.
-- 
cgit 


From 5e6240ffc24e55ecf7721fd9dc3f33c6f178be8c Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Sat, 20 Apr 2024 10:36:17 -0700
Subject: feat(treesitter): handle quantified fold captures

---
 runtime/lua/vim/treesitter/_fold.lua | 58 +++++++++++++++++++++++-------------
 1 file changed, 37 insertions(+), 21 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/_fold.lua b/runtime/lua/vim/treesitter/_fold.lua
index d96cc966de..d8b9f4a261 100644
--- a/runtime/lua/vim/treesitter/_fold.lua
+++ b/runtime/lua/vim/treesitter/_fold.lua
@@ -149,27 +149,43 @@ local function get_folds_levels(bufnr, info, srow, erow, parse_injections)
 
     -- Collect folds starting from srow - 1, because we should first subtract the folds that end at
     -- srow - 1 from the level of srow - 1 to get accurate level of srow.
-    for id, node, metadata in query:iter_captures(tree:root(), bufnr, math.max(srow - 1, 0), erow) do
-      if query.captures[id] == 'fold' then
-        local range = ts.get_range(node, bufnr, metadata[id])
-        local start, _, stop, stop_col = Range.unpack4(range)
-
-        if stop_col == 0 then
-          stop = stop - 1
-        end
-
-        local fold_length = stop - start + 1
-
-        -- Fold only multiline nodes that are not exactly the same as previously met folds
-        -- Checking against just the previously found fold is sufficient if nodes
-        -- are returned in preorder or postorder when traversing tree
-        if
-          fold_length > vim.wo.foldminlines and not (start == prev_start and stop == prev_stop)
-        then
-          enter_counts[start + 1] = (enter_counts[start + 1] or 0) + 1
-          leave_counts[stop + 1] = (leave_counts[stop + 1] or 0) + 1
-          prev_start = start
-          prev_stop = stop
+    for _, match, metadata in
+      query:iter_matches(tree:root(), bufnr, math.max(srow - 1, 0), erow, { all = true })
+    do
+      for id, nodes in pairs(match) do
+        if query.captures[id] == 'fold' then
+          local range = ts.get_range(nodes[1], bufnr, metadata[id])
+          local start, _, stop, stop_col = Range.unpack4(range)
+
+          for i = 2, #nodes, 1 do
+            local node_range = ts.get_range(nodes[i], bufnr, metadata[id])
+            local node_start, _, node_stop, node_stop_col = Range.unpack4(node_range)
+            if node_start < start then
+              start = node_start
+            end
+            if node_stop > stop then
+              stop = node_stop
+              stop_col = node_stop_col
+            end
+          end
+
+          if stop_col == 0 then
+            stop = stop - 1
+          end
+
+          local fold_length = stop - start + 1
+
+          -- Fold only multiline nodes that are not exactly the same as previously met folds
+          -- Checking against just the previously found fold is sufficient if nodes
+          -- are returned in preorder or postorder when traversing tree
+          if
+            fold_length > vim.wo.foldminlines and not (start == prev_start and stop == prev_stop)
+          then
+            enter_counts[start + 1] = (enter_counts[start + 1] or 0) + 1
+            leave_counts[stop + 1] = (leave_counts[stop + 1] or 0) + 1
+            prev_start = start
+            prev_stop = stop
+          end
         end
       end
     end
-- 
cgit 


From 2b6c9bbe7f7ac950683e81129b76e35e35839ede Mon Sep 17 00:00:00 2001
From: Jaehwang Jung <tomtomjhj@gmail.com>
Date: Sat, 20 Apr 2024 02:33:44 +0900
Subject: perf(treesitter): incremental foldupdate

Problem:
While the fold level computation is incremental, the evaluation of the
foldexpr is done on the full buffer. Despite that the foldexpr reads
from the cache, it can take tens of milliseconds for moderately big (10K
lines) buffers.

Solution:
Track the range of lines on which the foldexpr should be evaluated.
---
 runtime/lua/vim/treesitter/_fold.lua | 122 +++++++++++++++++++++--------------
 1 file changed, 72 insertions(+), 50 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/_fold.lua b/runtime/lua/vim/treesitter/_fold.lua
index d8b9f4a261..d511bef7a5 100644
--- a/runtime/lua/vim/treesitter/_fold.lua
+++ b/runtime/lua/vim/treesitter/_fold.lua
@@ -4,10 +4,21 @@ local Range = require('vim.treesitter._range')
 
 local api = vim.api
 
+---Treesitter folding is done in two steps:
+---(1) compute the fold levels with the syntax tree and cache the result (`compute_folds_levels`)
+---(2) evaluate foldexpr for each window, which reads from the cache (`foldupdate`)
 ---@class TS.FoldInfo
----@field levels string[] the foldexpr result for each line
----@field levels0 integer[] the raw fold levels
----@field edits? {[1]: integer, [2]: integer} line range edited since the last invocation of the callback scheduled in on_bytes. 0-indexed, end-exclusive.
+---
+---@field levels string[] the cached foldexpr result for each line
+---@field levels0 integer[] the cached raw fold levels
+---
+---The range edited since the last invocation of the callback scheduled in on_bytes.
+---Should compute fold levels in this range.
+---@field on_bytes_range? Range2
+---
+---The range on which to evaluate foldexpr.
+---When in insert mode, the evaluation is deferred to InsertLeave.
+---@field foldupdate_range? Range2
 local FoldInfo = {}
 FoldInfo.__index = FoldInfo
 
@@ -80,31 +91,16 @@ function FoldInfo:add_range(srow, erow)
   list_insert(self.levels0, srow + 1, erow, -1)
 end
 
----@package
+---@param range Range2
 ---@param srow integer
 ---@param erow_old integer
 ---@param erow_new integer 0-indexed, exclusive
-function FoldInfo:edit_range(srow, erow_old, erow_new)
-  if self.edits then
-    self.edits[1] = math.min(srow, self.edits[1])
-    if erow_old <= self.edits[2] then
-      self.edits[2] = self.edits[2] + (erow_new - erow_old)
-    end
-    self.edits[2] = math.max(self.edits[2], erow_new)
-  else
-    self.edits = { srow, erow_new }
-  end
-end
-
----@package
----@return integer? srow
----@return integer? erow 0-indexed, exclusive
-function FoldInfo:flush_edit()
-  if self.edits then
-    local srow, erow = self.edits[1], self.edits[2]
-    self.edits = nil
-    return srow, erow
+local function edit_range(range, srow, erow_old, erow_new)
+  range[1] = math.min(srow, range[1])
+  if erow_old <= range[2] then
+    range[2] = range[2] + (erow_new - erow_old)
   end
+  range[2] = math.max(range[2], erow_new)
 end
 
 --- If a parser doesn't have any ranges explicitly set, treesitter will
@@ -128,7 +124,7 @@ end
 ---@param srow integer?
 ---@param erow integer? 0-indexed, exclusive
 ---@param parse_injections? boolean
-local function get_folds_levels(bufnr, info, srow, erow, parse_injections)
+local function compute_folds_levels(bufnr, info, srow, erow, parse_injections)
   srow = srow or 0
   erow = normalise_erow(bufnr, erow)
 
@@ -231,7 +227,7 @@ local function get_folds_levels(bufnr, info, srow, erow, parse_injections)
       clamped = nestmax
     end
 
-    -- Record the "real" level, so that it can be used as "base" of later get_folds_levels().
+    -- Record the "real" level, so that it can be used as "base" of later compute_folds_levels().
     info.levels0[lnum] = adjusted
     info.levels[lnum] = prefix .. tostring(clamped)
 
@@ -252,15 +248,14 @@ local group = api.nvim_create_augroup('treesitter/fold', {})
 ---
 --- Nvim usually automatically updates folds when text changes, but it doesn't work here because
 --- FoldInfo update is scheduled. So we do it manually.
-local function foldupdate(bufnr)
-  local function do_update()
-    for _, win in ipairs(vim.fn.win_findbuf(bufnr)) do
-      api.nvim_win_call(win, function()
-        if vim.wo.foldmethod == 'expr' then
-          vim._foldupdate()
-        end
-      end)
-    end
+---@package
+---@param srow integer
+---@param erow integer 0-indexed, exclusive
+function FoldInfo:foldupdate(bufnr, srow, erow)
+  if self.foldupdate_range then
+    edit_range(self.foldupdate_range, srow, erow, erow)
+  else
+    self.foldupdate_range = { srow, erow }
   end
 
   if api.nvim_get_mode().mode == 'i' then
@@ -275,12 +270,25 @@ local function foldupdate(bufnr)
       group = group,
       buffer = bufnr,
       once = true,
-      callback = do_update,
+      callback = function()
+        self:do_foldupdate(bufnr)
+      end,
     })
     return
   end
 
-  do_update()
+  self:do_foldupdate(bufnr)
+end
+
+---@package
+function FoldInfo:do_foldupdate(bufnr)
+  local srow, erow = self.foldupdate_range[1], self.foldupdate_range[2]
+  self.foldupdate_range = nil
+  for _, win in ipairs(vim.fn.win_findbuf(bufnr)) do
+    if vim.wo[win].foldmethod == 'expr' then
+      vim._foldupdate(win, srow, erow)
+    end
+  end
 end
 
 --- Schedule a function only if bufnr is loaded.
@@ -288,7 +296,7 @@ end
 --- * queries seem to use the old buffer state in on_bytes for some unknown reason;
 --- * to avoid textlock;
 --- * to avoid infinite recursion:
----   get_folds_levels → parse → _do_callback → on_changedtree → get_folds_levels.
+---   compute_folds_levels → parse → _do_callback → on_changedtree → compute_folds_levels.
 ---@param bufnr integer
 ---@param fn function
 local function schedule_if_loaded(bufnr, fn)
@@ -305,16 +313,20 @@ end
 ---@param tree_changes Range4[]
 local function on_changedtree(bufnr, foldinfo, tree_changes)
   schedule_if_loaded(bufnr, function()
+    local srow_upd, erow_upd ---@type integer?, integer?
     for _, change in ipairs(tree_changes) do
       local srow, _, erow, ecol = Range.unpack4(change)
       if ecol > 0 then
         erow = erow + 1
       end
       -- Start from `srow - foldminlines`, because this edit may have shrunken the fold below limit.
-      get_folds_levels(bufnr, foldinfo, math.max(srow - vim.wo.foldminlines, 0), erow)
+      srow = math.max(srow - vim.wo.foldminlines, 0)
+      compute_folds_levels(bufnr, foldinfo, srow, erow)
+      srow_upd = srow_upd and math.min(srow_upd, srow) or srow
+      erow_upd = erow_upd and math.max(erow_upd, erow) or erow
     end
     if #tree_changes > 0 then
-      foldupdate(bufnr)
+      foldinfo:foldupdate(bufnr, srow_upd, erow_upd)
     end
   end)
 end
@@ -351,19 +363,29 @@ local function on_bytes(bufnr, foldinfo, start_row, start_col, old_row, old_col,
         foldinfo:add_range(end_row_old, end_row_new)
       end
     end
-    foldinfo:edit_range(start_row, end_row_old, end_row_new)
+
+    if foldinfo.on_bytes_range then
+      edit_range(foldinfo.on_bytes_range, start_row, end_row_old, end_row_new)
+    else
+      foldinfo.on_bytes_range = { start_row, end_row_new }
+    end
+    if foldinfo.foldupdate_range then
+      edit_range(foldinfo.foldupdate_range, start_row, end_row_old, end_row_new)
+    end
 
     -- This callback must not use on_bytes arguments, because they can be outdated when the callback
     -- is invoked. For example, `J` with non-zero count triggers multiple on_bytes before executing
-    -- the scheduled callback. So we should collect the edits.
+    -- the scheduled callback. So we accumulate the edited ranges in `on_bytes_range`.
     schedule_if_loaded(bufnr, function()
-      local srow, erow = foldinfo:flush_edit()
-      if not srow then
+      if not foldinfo.on_bytes_range then
         return
       end
+      local srow, erow = foldinfo.on_bytes_range[1], foldinfo.on_bytes_range[2]
+      foldinfo.on_bytes_range = nil
       -- Start from `srow - foldminlines`, because this edit may have shrunken the fold below limit.
-      get_folds_levels(bufnr, foldinfo, math.max(srow - vim.wo.foldminlines, 0), erow)
-      foldupdate(bufnr)
+      srow = math.max(srow - vim.wo.foldminlines, 0)
+      compute_folds_levels(bufnr, foldinfo, srow, erow)
+      foldinfo:foldupdate(bufnr, srow, erow)
     end)
   end
 end
@@ -382,7 +404,7 @@ function M.foldexpr(lnum)
 
   if not foldinfos[bufnr] then
     foldinfos[bufnr] = FoldInfo.new()
-    get_folds_levels(bufnr, foldinfos[bufnr])
+    compute_folds_levels(bufnr, foldinfos[bufnr])
 
     parser:register_cbs({
       on_changedtree = function(tree_changes)
@@ -406,10 +428,10 @@ api.nvim_create_autocmd('OptionSet', {
   pattern = { 'foldminlines', 'foldnestmax' },
   desc = 'Refresh treesitter folds',
   callback = function()
-    for _, bufnr in ipairs(vim.tbl_keys(foldinfos)) do
+    for bufnr, _ in pairs(foldinfos) do
       foldinfos[bufnr] = FoldInfo.new()
-      get_folds_levels(bufnr, foldinfos[bufnr])
-      foldupdate(bufnr)
+      compute_folds_levels(bufnr, foldinfos[bufnr])
+      foldinfos[bufnr]:foldupdate(bufnr, 0, api.nvim_buf_line_count(bufnr))
     end
   end,
 })
-- 
cgit 


From 032df963bb3fb0b5652e1817e9f4da986996fa6d Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Sun, 21 Apr 2024 13:39:08 +0100
Subject: refactor(treesitter): language loading

---
 runtime/lua/vim/treesitter/_meta.lua | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/_meta.lua b/runtime/lua/vim/treesitter/_meta.lua
index e2768d4b06..34a51e42f6 100644
--- a/runtime/lua/vim/treesitter/_meta.lua
+++ b/runtime/lua/vim/treesitter/_meta.lua
@@ -60,9 +60,17 @@ local TSNode = {}
 ---@field captures string[]
 ---@field patterns table<integer, (integer|string)[][]>
 
+--- @param lang string
+vim._ts_inspect_language = function(lang) end
+
 ---@return integer
 vim._ts_get_language_version = function() end
 
+--- @param path string
+--- @param lang string
+--- @param symbol_name? string
+vim._ts_add_language = function(path, lang, symbol_name) end
+
 ---@return integer
 vim._ts_get_minimum_language_version = function() end
 
-- 
cgit 


From c5b9fb2f256516398592c81f496dae75a036b18e Mon Sep 17 00:00:00 2001
From: TheLeoP <eugenio2305@hotmail.com>
Date: Fri, 26 Apr 2024 08:28:22 -0500
Subject: fix(treesitter.foldexpr): check for all insert submodes

---
 runtime/lua/vim/treesitter/_fold.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/_fold.lua b/runtime/lua/vim/treesitter/_fold.lua
index d511bef7a5..09d3f3368f 100644
--- a/runtime/lua/vim/treesitter/_fold.lua
+++ b/runtime/lua/vim/treesitter/_fold.lua
@@ -258,7 +258,7 @@ function FoldInfo:foldupdate(bufnr, srow, erow)
     self.foldupdate_range = { srow, erow }
   end
 
-  if api.nvim_get_mode().mode == 'i' then
+  if api.nvim_get_mode().mode:match('^i') then
     -- foldUpdate() is guarded in insert mode. So update folds on InsertLeave
     if #(api.nvim_get_autocmds({
       group = group,
-- 
cgit 


From 26b5405d181e8c9e75c4b4ec9aae963cc25f285f Mon Sep 17 00:00:00 2001
From: Christian Clason <c.clason@uni-graz.at>
Date: Sun, 28 Apr 2024 16:27:47 +0200
Subject: fix(treesitter): enforce lowercase language names (#28546)

* fix(treesitter): enforce lowercase language names

Problem: On case-insensitive file systems (e.g., macOS), `has_parser`
will return `true` for uppercase aliases, which will then try to inject
the uppercase language unsuccessfully.

Solution: Enforce and assume parser names to be lowercase when
resolving language names.
---
 runtime/lua/vim/treesitter/language.lua     |  3 +++
 runtime/lua/vim/treesitter/languagetree.lua | 12 +-----------
 2 files changed, 4 insertions(+), 11 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua
index 47abf65332..d0a74daa6c 100644
--- a/runtime/lua/vim/treesitter/language.lua
+++ b/runtime/lua/vim/treesitter/language.lua
@@ -88,6 +88,9 @@ function M.add(lang, opts)
     filetype = { filetype, { 'string', 'table' }, true },
   })
 
+  -- parser names are assumed to be lowercase (consistent behavior on case-insensitive file systems)
+  lang = lang:lower()
+
   if vim._ts_has_language(lang) then
     M.register(lang, filetype)
     return
diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index 990debc77b..e618f29f8f 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -758,7 +758,6 @@ local has_parser = vim.func._memoize(1, function(lang)
 end)
 
 --- Return parser name for language (if exists) or filetype (if registered and exists).
---- Also attempts with the input lower-cased.
 ---
 ---@param alias string language or filetype name
 ---@return string? # resolved parser name
@@ -772,19 +771,10 @@ local function resolve_lang(alias)
     return alias
   end
 
-  if has_parser(alias:lower()) then
-    return alias:lower()
-  end
-
   local lang = vim.treesitter.language.get_lang(alias)
   if lang and has_parser(lang) then
     return lang
   end
-
-  lang = vim.treesitter.language.get_lang(alias:lower())
-  if lang and has_parser(lang) then
-    return lang
-  end
 end
 
 ---@private
@@ -808,7 +798,7 @@ function LanguageTree:_get_injection(match, metadata)
       -- Lang should override any other language tag
       if name == 'injection.language' then
         local text = vim.treesitter.get_node_text(node, self._source, { metadata = metadata[id] })
-        lang = resolve_lang(text)
+        lang = resolve_lang(text:lower()) -- language names are always lower case
       elseif name == 'injection.filename' then
         local text = vim.treesitter.get_node_text(node, self._source, { metadata = metadata[id] })
         local ft = vim.filetype.match({ filename = text })
-- 
cgit 


From 037ea6e786b5d05f4a8965e4c2ba6aa60ec7c01a Mon Sep 17 00:00:00 2001
From: Luuk van Baal <luukvbaal@gmail.com>
Date: Wed, 10 Apr 2024 11:42:46 +0200
Subject: feat(api): add nvim__redraw for more granular redrawing

Experimental and subject to future changes.
Add a way to redraw certain elements that are not redrawn while Nvim is waiting
for input, or currently have no API to do so. This API covers all that can be
done with the :redraw* commands, in addition to the following new features:
- Immediately move the cursor to a (non-current) window.
- Target a specific window or buffer to mark for redraw.
- Mark a buffer range for redraw (replaces nvim__buf_redraw_range()).
- Redraw the 'statuscolumn'.
---
 runtime/lua/vim/treesitter/highlighter.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index 3f7e31212c..d2f986b874 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -215,7 +215,7 @@ end
 ---@param start_row integer
 ---@param new_end integer
 function TSHighlighter:on_bytes(_, _, start_row, _, _, _, _, _, new_end)
-  api.nvim__buf_redraw_range(self.bufnr, start_row, start_row + new_end + 1)
+  api.nvim__redraw({ buf = self.bufnr, range = { start_row, start_row + new_end + 1 } })
 end
 
 ---@package
@@ -227,7 +227,7 @@ end
 ---@param changes Range6[]
 function TSHighlighter:on_changedtree(changes)
   for _, ch in ipairs(changes) do
-    api.nvim__buf_redraw_range(self.bufnr, ch[1], ch[4] + 1)
+    api.nvim__redraw({ buf = self.bufnr, range = { ch[1], ch[4] + 1 } })
   end
 end
 
-- 
cgit 


From 3a8265266e0c0fe31f34b7c0192e8ae7d83ae950 Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Fri, 3 May 2024 09:34:02 -0700
Subject: fix(treesitter): escape "\" in :InspectTree #28613

Some parsers for, e.g., LaTeX or PHP have anonymous nodes like `"\"` or `"\text"` that behave wonkily (especially the first example) in the `InspectTree` window, so this PR escapes them by adding another backslash in front of them
---
 runtime/lua/vim/treesitter/dev.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/dev.lua b/runtime/lua/vim/treesitter/dev.lua
index dc2a14d238..5c91f101c0 100644
--- a/runtime/lua/vim/treesitter/dev.lua
+++ b/runtime/lua/vim/treesitter/dev.lua
@@ -226,7 +226,7 @@ function TSTreeView:draw(bufnr)
         text = string.format('(%s', item.node:type())
       end
     else
-      text = string.format('"%s"', item.node:type():gsub('\n', '\\n'):gsub('"', '\\"'))
+      text = string.format('%q', item.node:type()):gsub('\n', 'n')
     end
 
     local next = self:get(i + 1)
-- 
cgit 


From e7f50f43c82225eeecbff531b55d6ed26fad1bf5 Mon Sep 17 00:00:00 2001
From: Jaehwang Jung <tomtomjhj@gmail.com>
Date: Tue, 23 Apr 2024 00:29:14 +0900
Subject: fix(treesitter): clip end row early

Problem:
UINT32_MAX + 1 passed to vim._foldupdate.

Solution:
Clip the end row from treesitter asap to avoid such issues.
---
 runtime/lua/vim/treesitter/_fold.lua | 25 +++++++++----------------
 1 file changed, 9 insertions(+), 16 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/_fold.lua b/runtime/lua/vim/treesitter/_fold.lua
index 09d3f3368f..eecf1ad6b1 100644
--- a/runtime/lua/vim/treesitter/_fold.lua
+++ b/runtime/lua/vim/treesitter/_fold.lua
@@ -103,20 +103,6 @@ local function edit_range(range, srow, erow_old, erow_new)
   range[2] = math.max(range[2], erow_new)
 end
 
---- If a parser doesn't have any ranges explicitly set, treesitter will
---- return a range with end_row and end_bytes with a value of UINT32_MAX,
---- so clip end_row to the max buffer line.
----
---- TODO(lewis6991): Handle this generally
----
---- @param bufnr integer
---- @param erow integer? 0-indexed, exclusive
---- @return integer
-local function normalise_erow(bufnr, erow)
-  local max_erow = api.nvim_buf_line_count(bufnr)
-  return math.min(erow or max_erow, max_erow)
-end
-
 -- TODO(lewis6991): Setup a decor provider so injections folds can be parsed
 -- as the window is redrawn
 ---@param bufnr integer
@@ -126,7 +112,7 @@ end
 ---@param parse_injections? boolean
 local function compute_folds_levels(bufnr, info, srow, erow, parse_injections)
   srow = srow or 0
-  erow = normalise_erow(bufnr, erow)
+  erow = erow or api.nvim_buf_line_count(bufnr)
 
   local parser = ts.get_parser(bufnr)
 
@@ -314,9 +300,16 @@ end
 local function on_changedtree(bufnr, foldinfo, tree_changes)
   schedule_if_loaded(bufnr, function()
     local srow_upd, erow_upd ---@type integer?, integer?
+    local max_erow = api.nvim_buf_line_count(bufnr)
     for _, change in ipairs(tree_changes) do
       local srow, _, erow, ecol = Range.unpack4(change)
-      if ecol > 0 then
+      -- If a parser doesn't have any ranges explicitly set, treesitter will
+      -- return a range with end_row and end_bytes with a value of UINT32_MAX,
+      -- so clip end_row to the max buffer line.
+      -- TODO(lewis6991): Handle this generally
+      if erow > max_erow then
+        erow = max_erow
+      elseif ecol > 0 then
         erow = erow + 1
       end
       -- Start from `srow - foldminlines`, because this edit may have shrunken the fold below limit.
-- 
cgit 


From b6fdde5224250ec5dc3d5dcfec32d62a887173ff Mon Sep 17 00:00:00 2001
From: Jongwook Choi <wookayin@gmail.com>
Date: Sun, 12 May 2024 18:12:03 -0400
Subject: fix(treesitter): text alignment in checkhealth vim.treesitter

Problem: The column width 10 for parser name (lang) is too short.
For example, `markdown_inline` has 15 characters, which results in a
slight misalignment with other lines.

e.g. it looked like:

```
- OK Parser: markdown   ABI: 14, path: .../parser/markdown.so
- OK Parser: markdown_inline  ABI: 14, path: .../parser/markdown_inline.so
- OK Parser: php        ABI: 14, path: .../parser/php.so
```

Solution: Use column width 20. As of now, the longest name among those
available in nvim-treesitter has length 18 (`haskell_persistent`).

e.g.:

```
- OK Parser: markdown             ABI: 14, path: .../parser/markdown.so
- OK Parser: markdown_inline      ABI: 14, path: .../parser/markdown_inline.so
- OK Parser: php                  ABI: 14, path: .../parser/php.so
```
---
 runtime/lua/vim/treesitter/health.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/health.lua b/runtime/lua/vim/treesitter/health.lua
index a9b066d158..ed3616ef46 100644
--- a/runtime/lua/vim/treesitter/health.lua
+++ b/runtime/lua/vim/treesitter/health.lua
@@ -24,7 +24,7 @@ function M.check()
     else
       local lang = ts.language.inspect(parsername)
       health.ok(
-        string.format('Parser: %-10s ABI: %d, path: %s', parsername, lang._abi_version, parser)
+        string.format('Parser: %-20s ABI: %d, path: %s', parsername, lang._abi_version, parser)
       )
     end
   end
-- 
cgit 


From 6a264e08974bcb1b91f891eb65ef374f350d2827 Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Tue, 14 May 2024 07:14:43 -0700
Subject: fix(treesitter): allow optional directive captures (#28664)

---
 runtime/lua/vim/treesitter/query.lua | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index e68acac929..36c78b7f1d 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -529,6 +529,9 @@ local directive_handlers = {
   ['offset!'] = function(match, _, _, pred, metadata)
     local capture_id = pred[2] --[[@as integer]]
     local nodes = match[capture_id]
+    if not nodes or #nodes == 0 then
+      return
+    end
     assert(#nodes == 1, '#offset! does not support captures on multiple nodes')
 
     local node = nodes[1]
@@ -562,6 +565,9 @@ local directive_handlers = {
     assert(type(id) == 'number')
 
     local nodes = match[id]
+    if not nodes or #nodes == 0 then
+      return
+    end
     assert(#nodes == 1, '#gsub! does not support captures on multiple nodes')
     local node = nodes[1]
     local text = vim.treesitter.get_node_text(node, bufnr, { metadata = metadata[id] }) or ''
@@ -584,6 +590,9 @@ local directive_handlers = {
     assert(type(capture_id) == 'number')
 
     local nodes = match[capture_id]
+    if not nodes or #nodes == 0 then
+      return
+    end
     assert(#nodes == 1, '#trim! does not support captures on multiple nodes')
     local node = nodes[1]
 
-- 
cgit 


From 01b6bff7e9bc751be20ce7bb68e7ebe3037441de Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Thu, 2 May 2024 15:57:21 +0200
Subject: docs: news

Set dev_xx.txt help files to use "flow" layout.
---
 runtime/lua/vim/treesitter/languagetree.lua | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index e618f29f8f..270d869e43 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -225,7 +225,10 @@ function LanguageTree:_log(...)
   self._logger('nvim', table.concat(msg, ' '))
 end
 
---- Invalidates this parser and all its children
+--- Invalidates this parser and its children.
+---
+--- Should only be called when the tracked state of the LanguageTree is not valid against the parse
+--- tree in treesitter. Doesn't clear filesystem cache. Called often, so needs to be fast.
 ---@param reload boolean|nil
 function LanguageTree:invalidate(reload)
   self._valid = false
-- 
cgit 


From 4b029163345333a2c6975cd0dace6613b036ae47 Mon Sep 17 00:00:00 2001
From: vanaigr <vanaigranov@gmail.com>
Date: Thu, 16 May 2024 09:57:58 -0500
Subject: perf(treesitter): use child_containing_descendant() in has-ancestor?
 (#28512)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Problem: `has-ancestor?` is O(n²) for the depth of the tree since it iterates over each of the node's ancestors (bottom-up), and each ancestor takes O(n) time.
This happens because tree-sitter's nodes don't store their parent nodes, and the tree is searched (top-down) each time a new parent is requested.

Solution: Make use of new `ts_node_child_containing_descendant()` in tree-sitter v0.22.6 (which is now the minimum required version) to rewrite the `has-ancestor?` predicate in C to become O(n).

For a sample file, decreases the time taken by `has-ancestor?` from 360ms to 6ms.
---
 runtime/lua/vim/treesitter/_meta.lua |  1 +
 runtime/lua/vim/treesitter/query.lua | 13 ++-----------
 2 files changed, 3 insertions(+), 11 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/_meta.lua b/runtime/lua/vim/treesitter/_meta.lua
index 34a51e42f6..177699a207 100644
--- a/runtime/lua/vim/treesitter/_meta.lua
+++ b/runtime/lua/vim/treesitter/_meta.lua
@@ -20,6 +20,7 @@ error('Cannot require a meta file')
 ---@field descendant_for_range fun(self: TSNode, start_row: integer, start_col: integer, end_row: integer, end_col: integer): TSNode?
 ---@field named_descendant_for_range fun(self: TSNode, start_row: integer, start_col: integer, end_row: integer, end_col: integer): TSNode?
 ---@field parent fun(self: TSNode): TSNode?
+---@field child_containing_descendant fun(self: TSNode, descendant: TSNode): TSNode?
 ---@field next_sibling fun(self: TSNode): TSNode?
 ---@field prev_sibling fun(self: TSNode): TSNode?
 ---@field next_named_sibling fun(self: TSNode): TSNode?
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 36c78b7f1d..ef5c2143a7 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -457,17 +457,8 @@ local predicate_handlers = {
     end
 
     for _, node in ipairs(nodes) do
-      local ancestor_types = {} --- @type table<string, boolean>
-      for _, type in ipairs({ unpack(predicate, 3) }) do
-        ancestor_types[type] = true
-      end
-
-      local cur = node:parent()
-      while cur do
-        if ancestor_types[cur:type()] then
-          return true
-        end
-        cur = cur:parent()
+      if node:__has_ancestor(predicate) then
+        return true
       end
     end
     return false
-- 
cgit 


From a664246171569209698c0b17b1d7af831f6603d2 Mon Sep 17 00:00:00 2001
From: dundargoc <gocdundar@gmail.com>
Date: Thu, 16 May 2024 15:22:46 +0200
Subject: feat: remove deprecated features

Remove following functions:
- vim.lsp.util.extract_completion_items
- vim.lsp.util.get_progress_messages
- vim.lsp.util.parse_snippet()
- vim.lsp.util.text_document_completion_list_to_complete_items
- LanguageTree:for_each_child
- health#report_error
- health#report_info
- health#report_ok
- health#report_start
- health#report_warn
- vim.health.report_error
- vim.health.report_info
- vim.health.report_ok
- vim.health.report_start
- vim.health.report_warn
---
 runtime/lua/vim/treesitter/languagetree.lua | 18 ------------------
 1 file changed, 18 deletions(-)

(limited to 'runtime/lua/vim/treesitter')

diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index 270d869e43..b0812123b9 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -463,24 +463,6 @@ function LanguageTree:parse(range)
   return self._trees
 end
 
----@deprecated Misleading name. Use `LanguageTree:children()` (non-recursive) instead,
----            add recursion yourself if needed.
---- Invokes the callback for each |LanguageTree| and its children recursively
----
----@param fn fun(tree: vim.treesitter.LanguageTree, lang: string)
----@param include_self? boolean Whether to include the invoking tree in the results
-function LanguageTree:for_each_child(fn, include_self)
-  vim.deprecate('LanguageTree:for_each_child()', 'LanguageTree:children()', '0.11')
-  if include_self then
-    fn(self, self._lang)
-  end
-
-  for _, child in pairs(self._children) do
-    --- @diagnostic disable-next-line:deprecated
-    child:for_each_child(fn, true)
-  end
-end
-
 --- Invokes the callback for each |LanguageTree| recursively.
 ---
 --- Note: This includes the invoking tree's child trees as well.
-- 
cgit