From b8c75a31e6f4716f542cd2000e4a7c19c1ae9d70 Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Sat, 17 Aug 2024 21:05:09 -0700
Subject: feat(treesitter): #trim! can trim all whitespace

This commit also implements more generic trimming, acting on all
whitespace (charwise) rather than just empty lines.

It will unblock
https://github.com/nvim-treesitter/nvim-treesitter/pull/3442 and allow
for properly concealing markdown bullet markers regardless of indent
width, e.g.
---
 runtime/lua/vim/treesitter/query.lua | 48 +++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 14 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 1677e8d364..3c7bc2eb89 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -572,13 +572,17 @@ local directive_handlers = {
 
     metadata[id].text = text:gsub(pattern, replacement)
   end,
-  -- Trim blank lines from end of the node
-  -- Example: (#trim! @fold)
-  -- TODO(clason): generalize to arbitrary whitespace removal
+  -- Trim whitespace from both sides of the node
+  -- Example: (#trim! @fold 1 1 1 1)
   ['trim!'] = function(match, _, bufnr, pred, metadata)
     local capture_id = pred[2]
     assert(type(capture_id) == 'number')
 
+    local trim_start_lines = pred[3] == '1'
+    local trim_start_cols = pred[4] == '1'
+    local trim_end_lines = pred[5] == '1' or not pred[3] -- default true for backwards compatibility
+    local trim_end_cols = pred[6] == '1'
+
     local nodes = match[capture_id]
     if not nodes or #nodes == 0 then
       return
@@ -588,20 +592,36 @@ local directive_handlers = {
 
     local start_row, start_col, end_row, end_col = node:range()
 
-    -- Don't trim if region ends in middle of a line
-    if end_col ~= 0 then
-      return
-    end
-
-    while end_row >= start_row do
-      -- As we only care when end_col == 0, always inspect one line above end_row.
-      local end_line = api.nvim_buf_get_lines(bufnr, end_row - 1, end_row, true)[1]
+    local node_text = vim.split(vim.treesitter.get_node_text(node, bufnr), '\n')
+    local end_idx = #node_text
+    local start_idx = 1
 
-      if end_line ~= '' then
-        break
+    if trim_end_lines then
+      while end_idx > 0 and node_text[end_idx]:find('^%s*$') do
+        end_idx = end_idx - 1
+        end_row = end_row - 1
       end
+    end
+    if trim_end_cols then
+      if end_idx == 0 then
+        end_row = start_row
+        end_col = start_col
+      else
+        local whitespace_start = node_text[end_idx]:find('(%s*)$')
+        end_col = (whitespace_start - 1) + (end_idx == 1 and start_col or 0)
+      end
+    end
 
-      end_row = end_row - 1
+    if trim_start_lines then
+      while start_idx <= end_idx and node_text[start_idx]:find('^%s*$') do
+        start_idx = start_idx + 1
+        start_row = start_row + 1
+      end
+    end
+    if trim_start_cols and node_text[start_idx] then
+      local _, whitespace_end = node_text[start_idx]:find('^(%s*)')
+      whitespace_end = whitespace_end or 0
+      start_col = (start_idx == 1 and start_col or 0) + whitespace_end
     end
 
     -- If this produces an invalid range, we just skip it.
-- 
cgit 


From c63e49cce2d20cee129a6312319dde8dcea6e3f6 Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Sat, 7 Dec 2024 03:01:59 -0800
Subject: fix(treesitter): #trim! range for nodes ending at col 0 #31488

Problem:
char-wise folding for `#trim!` ranges are improperly calculated for nodes that
end at column 0, due to the way `get_node_text` works.

Solution:
Add the blank line that `get_node_text` removes for for nodes ending at column
0. Also properly set column positions when performing linewise trims.
---
 runtime/lua/vim/treesitter/query.lua | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 3c7bc2eb89..dbe3d54c2f 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -593,6 +593,11 @@ local directive_handlers = {
     local start_row, start_col, end_row, end_col = node:range()
 
     local node_text = vim.split(vim.treesitter.get_node_text(node, bufnr), '\n')
+    if end_col == 0 then
+      -- get_node_text() will ignore the last line if the node ends at column 0
+      node_text[#node_text + 1] = ''
+    end
+
     local end_idx = #node_text
     local start_idx = 1
 
@@ -600,6 +605,9 @@ local directive_handlers = {
       while end_idx > 0 and node_text[end_idx]:find('^%s*$') do
         end_idx = end_idx - 1
         end_row = end_row - 1
+        -- set the end position to the last column of the next line, or 0 if we just trimmed the
+        -- last line
+        end_col = end_idx > 0 and #node_text[end_idx] or 0
       end
     end
     if trim_end_cols then
@@ -616,6 +624,7 @@ local directive_handlers = {
       while start_idx <= end_idx and node_text[start_idx]:find('^%s*$') do
         start_idx = start_idx + 1
         start_row = start_row + 1
+        start_col = 0
       end
     end
     if trim_start_cols and node_text[start_idx] then
-- 
cgit 


From dc692f553aae367a03f286e0d59561247941f96c Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" <justinkz@gmail.com>
Date: Wed, 1 Jan 2025 12:29:51 -0800
Subject: docs: misc #31479

---
 runtime/lua/vim/treesitter/query.lua | 51 ++++++++++++++++++++++--------------
 1 file changed, 32 insertions(+), 19 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index dbe3d54c2f..f9c497337f 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -1,3 +1,6 @@
+--- @brief This Lua |treesitter-query| interface allows you to create queries and use them to parse
+--- text. See |vim.treesitter.query.parse()| for a working example.
+
 local api = vim.api
 local language = require('vim.treesitter.language')
 local memoize = vim.func._memoize
@@ -8,9 +11,9 @@ local M = {}
 ---Parsed query, see |vim.treesitter.query.parse()|
 ---
 ---@class vim.treesitter.Query
----@field lang string name of the language for this parser
+---@field lang string parser language name
 ---@field captures string[] list of (unique) capture names defined in query
----@field info vim.treesitter.QueryInfo contains information used in the query (e.g. captures, predicates, directives)
+---@field info vim.treesitter.QueryInfo query context (e.g. captures, predicates, directives)
 ---@field query TSQuery userdata query object
 local Query = {}
 Query.__index = Query
@@ -228,20 +231,28 @@ M.get = memoize('concat-2', function(lang, query_name)
   return M.parse(lang, query_string)
 end)
 
---- Parse {query} as a string. (If the query is in a file, the caller
---- should read the contents into a string before calling).
----
---- Returns a `Query` (see |lua-treesitter-query|) object which can be used to
---- search nodes in the syntax tree for the patterns defined in {query}
---- using the `iter_captures` and `iter_matches` methods.
+--- Parses a {query} string and returns a `Query` object (|lua-treesitter-query|), which can be used
+--- to search the tree for the query patterns (via |Query:iter_captures()|, |Query:iter_matches()|),
+--- or inspect the query via these fields:
+---   - `captures`: a list of unique capture names defined in the query (alias: `info.captures`).
+---   - `info.patterns`: information about predicates.
 ---
---- Exposes `info` and `captures` with additional context about {query}.
----   - `captures` contains the list of unique capture names defined in {query}.
----   - `info.captures` also points to `captures`.
----   - `info.patterns` contains information about predicates.
+--- Example (select the code then run `:'<,'>lua` to try it):
+--- ```lua
+--- local query = vim.treesitter.query.parse('vimdoc', [[
+---   ; query
+---   ((h1) @str
+---     (#trim! @str 1 1 1 1))
+--- ]])
+--- local tree = vim.treesitter.get_parser():parse()[1]
+--- for id, node, metadata in query:iter_captures(tree:root(), 0) do
+---    -- Print the node name and source text.
+---    vim.print({node:type(), vim.treesitter.get_node_text(node, vim.api.nvim_get_current_buf())})
+--- end
+--- ```
 ---
 ---@param lang string Language to use for the query
----@param query string Query in s-expr syntax
+---@param query string Query text, in s-expr syntax
 ---
 ---@return vim.treesitter.Query : Parsed query
 ---
@@ -847,20 +858,22 @@ local function match_id_hash(_, match)
   return (match:info())
 end
 
---- Iterate over all captures from all matches inside {node}
+--- Iterates over all captures from all matches in {node}.
 ---
---- {source} is needed if the query contains predicates; then the caller
+--- {source} is required if the query contains predicates; then the caller
 --- must ensure to use a freshly parsed tree consistent with the current
 --- text of the buffer (if relevant). {start} and {stop} can be used to limit
 --- matches inside a row range (this is typically used with root node
 --- as the {node}, i.e., to get syntax highlight matches in the current
 --- viewport). When omitted, the {start} and {stop} row values are used from the given node.
 ---
---- The iterator returns four values: a numeric id identifying the capture,
---- the captured node, metadata from any directives processing the match,
---- and the match itself.
---- The following example shows how to get captures by name:
+--- The iterator returns four values:
+--- 1. the numeric id identifying the capture
+--- 2. the captured node
+--- 3. metadata from any directives processing the match
+--- 4. the match itself
 ---
+--- Example: how to get captures by name:
 --- ```lua
 --- for id, node, metadata, match in query:iter_captures(tree:root(), bufnr, first, last) do
 ---   local name = query.captures[id] -- name of the capture in the query
-- 
cgit 


From b61051ccb4c23958d43d285b8b801af11620264f Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Sun, 1 Sep 2024 16:54:30 -0700
Subject: feat(func): allow manual cache invalidation for _memoize

This commit also adds some tests for the existing memoization
functionality.
---
 runtime/lua/vim/treesitter/query.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index f9c497337f..2b3b9096a6 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -902,8 +902,8 @@ function Query:iter_captures(node, source, start, stop)
 
   local cursor = vim._create_ts_querycursor(node, self.query, start, stop, { match_limit = 256 })
 
-  local apply_directives = memoize(match_id_hash, self.apply_directives, true)
-  local match_preds = memoize(match_id_hash, self.match_preds, true)
+  local apply_directives = memoize(match_id_hash, self.apply_directives, false)
+  local match_preds = memoize(match_id_hash, self.match_preds, false)
 
   local function iter(end_line)
     local capture, captured_node, match = cursor:next_capture()
-- 
cgit 


From 8d2ee542a82a0d162198f27de316ddfc81e8761c Mon Sep 17 00:00:00 2001
From: vanaigr <vanaigranov@gmail.com>
Date: Wed, 18 Dec 2024 12:23:28 -0600
Subject: perf(decor): join predicates and matches cache

---
 runtime/lua/vim/treesitter/query.lua | 81 ++++++++++++++++++++----------------
 1 file changed, 45 insertions(+), 36 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 2b3b9096a6..01fdb708eb 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -762,16 +762,7 @@ end
 ---@private
 ---@param match TSQueryMatch
 ---@param source integer|string
-function Query:match_preds(match, source)
-  local _, pattern = match:info()
-  local preds = self.info.patterns[pattern]
-
-  if not preds then
-    return true
-  end
-
-  local captures = match:captures()
-
+function Query:match_preds(preds, pattern, captures, source)
   for _, pred in pairs(preds) do
     -- Here we only want to return if a predicate DOES NOT match, and
     -- continue on the other case. This way unknown predicates will not be considered,
@@ -807,17 +798,9 @@ end
 ---@private
 ---@param match TSQueryMatch
 ---@return vim.treesitter.query.TSMetadata metadata
-function Query:apply_directives(match, source)
+function Query:apply_directives(preds, pattern, captures, source)
   ---@type vim.treesitter.query.TSMetadata
   local metadata = {}
-  local _, pattern = match:info()
-  local preds = self.info.patterns[pattern]
-
-  if not preds then
-    return metadata
-  end
-
-  local captures = match:captures()
 
   for _, pred in pairs(preds) do
     if is_directive(pred[1]) then
@@ -902,8 +885,10 @@ function Query:iter_captures(node, source, start, stop)
 
   local cursor = vim._create_ts_querycursor(node, self.query, start, stop, { match_limit = 256 })
 
-  local apply_directives = memoize(match_id_hash, self.apply_directives, false)
-  local match_preds = memoize(match_id_hash, self.match_preds, false)
+  -- For faster checks that a match is not in the cache.
+  local highest_cached_match_id = -1
+  ---@type table<integer, vim.treesitter.query.TSMetadata>
+  local match_cache = {}
 
   local function iter(end_line)
     local capture, captured_node, match = cursor:next_capture()
@@ -912,16 +897,35 @@ function Query:iter_captures(node, source, start, stop)
       return
     end
 
-    if not match_preds(self, match, source) then
-      local match_id = match:info()
-      cursor:remove_match(match_id)
-      if end_line and captured_node:range() > end_line then
-        return nil, captured_node, nil, nil
-      end
-      return iter(end_line) -- tail call: try next match
+    local match_id, pattern = match:info()
+
+    --- @type vim.treesitter.query.TSMetadata
+    local metadata
+    if match_id <= highest_cached_match_id then
+      metadata = match_cache[match_id]
     end
 
-    local metadata = apply_directives(self, match, source)
+    if not metadata then
+      local preds = self.info.patterns[pattern]
+      if preds then
+        local captures = match:captures()
+
+        if not self:match_preds(preds, pattern, captures, source) then
+          cursor:remove_match(match_id)
+          if end_line and captured_node:range() > end_line then
+            return nil, captured_node, nil, nil
+          end
+          return iter(end_line) -- tail call: try next match
+        end
+
+        metadata = self:apply_directives(preds, pattern, captures, source)
+      else
+        metadata = {}
+      end
+
+      highest_cached_match_id = math.max(highest_cached_match_id, match_id)
+      match_cache[match_id] = metadata
+    end
 
     return capture, captured_node, metadata, match
   end
@@ -985,16 +989,21 @@ function Query:iter_matches(node, source, start, stop, opts)
     end
 
     local match_id, pattern = match:info()
+    local preds = self.info.patterns[pattern]
+    local captures = match:captures()
 
-    if not self:match_preds(match, source) then
-      cursor:remove_match(match_id)
-      return iter() -- tail call: try next match
+    --- @type vim.treesitter.query.TSMetadata
+    local metadata
+    if preds then
+      if not self:match_preds(preds, pattern, captures, source) then
+        cursor:remove_match(match_id)
+        return iter() -- tail call: try next match
+      end
+      metadata = self:apply_directives(preds, pattern, captures, source)
+    else
+      metadata = {}
     end
 
-    local metadata = self:apply_directives(match, source)
-
-    local captures = match:captures()
-
     if opts.all == false then
       -- Convert the match table into the old buggy version for backward
       -- compatibility. This is slow, but we only do it when the caller explicitly opted into it by
-- 
cgit 


From dd234135ad20119917831fd8ffcb19d8562022ca Mon Sep 17 00:00:00 2001
From: vanaigr <vanaigranov@gmail.com>
Date: Wed, 18 Dec 2024 01:06:41 -0600
Subject: refactor: split predicates and directives

---
 runtime/lua/vim/treesitter/query.lua | 172 +++++++++++++++++++++--------------
 1 file changed, 103 insertions(+), 69 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 01fdb708eb..1fc001b39f 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -7,6 +7,59 @@ local memoize = vim.func._memoize
 
 local M = {}
 
+local function is_directive(name)
+  return string.sub(name, -1) == '!'
+end
+
+---@nodoc
+---@class vim.treesitter.query.ProcessedPredicate
+---@field [1] string predicate name
+---@field [2] boolean should match
+---@field [3] (integer|string)[] the original predicate
+
+---@alias vim.treesitter.query.ProcessedDirective (integer|string)[]
+
+---@nodoc
+---@class vim.treesitter.query.ProcessedPattern {
+---@field predicates vim.treesitter.query.ProcessedPredicate[]
+---@field directives vim.treesitter.query.ProcessedDirective[]
+
+--- Splits the query patterns into predicates and directives.
+---@param patterns table<integer, (integer|string)[][]>
+---@return table<integer, vim.treesitter.query.ProcessedPattern>
+local function process_patterns(patterns)
+  ---@type table<integer, vim.treesitter.query.ProcessedPattern>
+  local processed_patterns = {}
+
+  for k, pattern_list in pairs(patterns) do
+    ---@type vim.treesitter.query.ProcessedPredicate[]
+    local predicates = {}
+    ---@type vim.treesitter.query.ProcessedDirective[]
+    local directives = {}
+
+    for _, pattern in ipairs(pattern_list) do
+      -- Note: tree-sitter strips the leading # from predicates for us.
+      local pred_name = pattern[1]
+      ---@cast pred_name string
+
+      if is_directive(pred_name) then
+        table.insert(directives, pattern)
+      else
+        local should_match = true
+        if pred_name:match('^not%-') then
+          pred_name = pred_name:sub(5)
+          should_match = false
+        end
+        table.insert(predicates, { pred_name, should_match, pattern })
+      end
+    end
+
+    processed_patterns[k] = { predicates = predicates, directives = directives }
+  end
+
+  return processed_patterns
+end
+
 ---@nodoc
 ---Parsed query, see |vim.treesitter.query.parse()|
 ---
@@ -15,6 +68,7 @@ local M = {}
 ---@field captures string[] list of (unique) capture names defined in query
 ---@field info vim.treesitter.QueryInfo query context (e.g. captures, predicates, directives)
 ---@field query TSQuery userdata query object
+---@field private _processed_patterns table<integer, vim.treesitter.query.ProcessedPattern>
 local Query = {}
 Query.__index = Query
 
@@ -33,6 +87,7 @@ function Query.new(lang, ts_query)
     patterns = query_info.patterns,
   }
   self.captures = self.info.captures
+  self._processed_patterns = process_patterns(self.info.patterns)
   return self
 end
 
@@ -751,67 +806,50 @@ function M.list_predicates()
   return vim.tbl_keys(predicate_handlers)
 end
 
-local function xor(x, y)
-  return (x or y) and not (x and y)
-end
-
-local function is_directive(name)
-  return string.sub(name, -1) == '!'
-end
-
 ---@private
----@param match TSQueryMatch
+---@param pattern_i integer
+---@param predicates vim.treesitter.query.ProcessedPredicate[]
+---@param captures table<integer, TSNode[]>
 ---@param source integer|string
-function Query:match_preds(preds, pattern, captures, source)
-  for _, pred in pairs(preds) do
-    -- Here we only want to return if a predicate DOES NOT match, and
-    -- continue on the other case. This way unknown predicates will not be considered,
-    -- which allows some testing and easier user extensibility (#12173).
-    -- Also, tree-sitter strips the leading # from predicates for us.
-    local is_not = false
-
-    -- Skip over directives... they will get processed after all the predicates.
-    if not is_directive(pred[1]) then
-      local pred_name = pred[1]
-      if pred_name:match('^not%-') then
-        pred_name = pred_name:sub(5)
-        is_not = true
-      end
-
-      local handler = predicate_handlers[pred_name]
-
-      if not handler then
-        error(string.format('No handler for %s', pred[1]))
-        return false
-      end
-
-      local pred_matches = handler(captures, pattern, source, pred)
+---@return boolean whether the predicates match
+function Query:_match_predicates(predicates, pattern_i, captures, source)
+  for _, predicate in ipairs(predicates) do
+    local processed_name = predicate[1]
+    local should_match = predicate[2]
+    local orig_predicate = predicate[3]
+
+    local handler = predicate_handlers[processed_name]
+    if not handler then
+      error(string.format('No handler for %s', orig_predicate[1]))
+      return false
+    end
 
-      if not xor(is_not, pred_matches) then
-        return false
-      end
+    local does_match = handler(captures, pattern_i, source, orig_predicate)
+    if does_match ~= should_match then
+      return false
     end
   end
   return true
 end
 
 ---@private
----@param match TSQueryMatch
+---@param pattern_i integer
+---@param directives vim.treesitter.query.ProcessedDirective[]
+---@param source integer|string
+---@param captures table<integer, TSNode[]>
 ---@return vim.treesitter.query.TSMetadata metadata
-function Query:apply_directives(preds, pattern, captures, source)
+function Query:_apply_directives(directives, pattern_i, captures, source)
   ---@type vim.treesitter.query.TSMetadata
   local metadata = {}
 
-  for _, pred in pairs(preds) do
-    if is_directive(pred[1]) then
-      local handler = directive_handlers[pred[1]]
+  for _, directive in pairs(directives) do
+    local handler = directive_handlers[directive[1]]
 
-      if not handler then
-        error(string.format('No handler for %s', pred[1]))
-      end
-
-      handler(captures, pattern, source, pred, metadata)
+    if not handler then
+      error(string.format('No handler for %s', directive[1]))
     end
+
+    handler(captures, pattern_i, source, directive, metadata)
   end
 
   return metadata
@@ -835,12 +873,6 @@ local function value_or_node_range(start, stop, node)
   return start, stop
 end
 
---- @param match TSQueryMatch
---- @return integer
-local function match_id_hash(_, match)
-  return (match:info())
-end
-
 --- Iterates over all captures from all matches in {node}.
 ---
 --- {source} is required if the query contains predicates; then the caller
@@ -897,7 +929,7 @@ function Query:iter_captures(node, source, start, stop)
       return
     end
 
-    local match_id, pattern = match:info()
+    local match_id, pattern_i = match:info()
 
     --- @type vim.treesitter.query.TSMetadata
     local metadata
@@ -906,11 +938,14 @@ function Query:iter_captures(node, source, start, stop)
     end
 
     if not metadata then
-      local preds = self.info.patterns[pattern]
-      if preds then
+      metadata = {}
+
+      local processed_pattern = self._processed_patterns[pattern_i]
+      if processed_pattern then
         local captures = match:captures()
 
-        if not self:match_preds(preds, pattern, captures, source) then
+        local predicates = processed_pattern.predicates
+        if not self:_match_predicates(predicates, pattern_i, captures, source) then
           cursor:remove_match(match_id)
           if end_line and captured_node:range() > end_line then
             return nil, captured_node, nil, nil
@@ -918,9 +953,8 @@ function Query:iter_captures(node, source, start, stop)
           return iter(end_line) -- tail call: try next match
         end
 
-        metadata = self:apply_directives(preds, pattern, captures, source)
-      else
-        metadata = {}
+        local directives = processed_pattern.directives
+        metadata = self:_apply_directives(directives, pattern_i, captures, source)
       end
 
       highest_cached_match_id = math.max(highest_cached_match_id, match_id)
@@ -988,20 +1022,20 @@ function Query:iter_matches(node, source, start, stop, opts)
       return
     end
 
-    local match_id, pattern = match:info()
-    local preds = self.info.patterns[pattern]
+    local match_id, pattern_i = match:info()
+    local processed_pattern = self._processed_patterns[pattern_i]
     local captures = match:captures()
 
     --- @type vim.treesitter.query.TSMetadata
-    local metadata
-    if preds then
-      if not self:match_preds(preds, pattern, captures, source) then
+    local metadata = {}
+    if processed_pattern then
+      local predicates = processed_pattern.predicates
+      if not self:_match_predicates(predicates, pattern_i, captures, source) then
         cursor:remove_match(match_id)
         return iter() -- tail call: try next match
       end
-      metadata = self:apply_directives(preds, pattern, captures, source)
-    else
-      metadata = {}
+      local directives = processed_pattern.directives
+      metadata = self:_apply_directives(directives, pattern_i, captures, source)
     end
 
     if opts.all == false then
@@ -1012,11 +1046,11 @@ function Query:iter_matches(node, source, start, stop, opts)
       for k, v in pairs(captures or {}) do
         old_match[k] = v[#v]
       end
-      return pattern, old_match, metadata
+      return pattern_i, old_match, metadata
     end
 
     -- TODO(lewis6991): create a new function that returns {match, metadata}
-    return pattern, captures, metadata
+    return pattern_i, captures, metadata
   end
   return iter
 end
-- 
cgit 


From 0c296ab22484b4c009d119908d1614a6c6d96b2c Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Thu, 9 Jan 2025 08:36:16 -0800
Subject: feat(docs): "yxx" runs Lua/Vimscript code examples #31904

`yxx` in Normal mode over a Lua or Vimscript code block section will execute the code.

Co-authored-by: Justin M. Keyes <justinkz@gmail.com>
---
 runtime/lua/vim/treesitter/query.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 1fc001b39f..b9bcbe9a80 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -292,7 +292,7 @@ end)
 ---   - `captures`: a list of unique capture names defined in the query (alias: `info.captures`).
 ---   - `info.patterns`: information about predicates.
 ---
---- Example (select the code then run `:'<,'>lua` to try it):
+--- Example (to try it, use `yxx` or select the code then run `:'<,'>lua`):
 --- ```lua
 --- local query = vim.treesitter.query.parse('vimdoc', [[
 ---   ; query
@@ -983,7 +983,7 @@ end
 ---       -- `node` was captured by the `name` capture in the match
 ---
 ---       local node_data = metadata[id] -- Node level metadata
----       ... use the info here ...
+---       -- ... use the info here ...
 ---     end
 ---   end
 --- end
-- 
cgit 


From 3fdc4302415972eb5d98ba832372236be3d22572 Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Sat, 11 Jan 2025 15:44:07 -0800
Subject: perf(treesitter): cache queries strongly

**Problem:** Query parsing uses a weak cache which is invalidated
frequently

**Solution:** Make the cache strong, and invalidate it manually when
necessary (that is, when `rtp` is changed or `query.set()` is called)

Co-authored-by: Christian Clason <c.clason@uni-graz.at>
---
 runtime/lua/vim/treesitter/query.lua | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index b9bcbe9a80..b0b0fecd38 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -262,6 +262,7 @@ local explicit_queries = setmetatable({}, {
 ---@param query_name string Name of the query (e.g., "highlights")
 ---@param text string Query text (unparsed).
 function M.set(lang, query_name, text)
+  M.get:clear(lang, query_name)
   explicit_queries[lang][query_name] = M.parse(lang, text)
 end
 
@@ -284,7 +285,15 @@ M.get = memoize('concat-2', function(lang, query_name)
   end
 
   return M.parse(lang, query_string)
-end)
+end, false)
+
+api.nvim_create_autocmd('OptionSet', {
+  pattern = { 'runtimepath' },
+  group = api.nvim_create_augroup('ts_query_cache_reset', { clear = true }),
+  callback = function()
+    M.get:clear()
+  end,
+})
 
 --- Parses a {query} string and returns a `Query` object (|lua-treesitter-query|), which can be used
 --- to search the tree for the query patterns (via |Query:iter_captures()|, |Query:iter_matches()|),
@@ -316,7 +325,7 @@ M.parse = memoize('concat-2', function(lang, query)
   assert(language.add(lang))
   local ts_query = vim._ts_parse_query(lang, query)
   return Query.new(lang, ts_query)
-end)
+end, false)
 
 --- Implementations of predicates that can optionally be prefixed with "any-".
 ---
-- 
cgit 


From 45e606b1fddbfeee8fe28385b5371ca6f2fba71b Mon Sep 17 00:00:00 2001
From: Riley Bruins <ribru17@hotmail.com>
Date: Wed, 18 Dec 2024 10:48:33 -0800
Subject: feat(treesitter): async parsing

**Problem:** Parsing can be slow for large files, and it is a blocking
operation which can be disruptive and annoying.

**Solution:** Provide a function for asynchronous parsing, which accepts
a callback to be run after parsing completes.

Co-authored-by: Lewis Russell <lewis6991@gmail.com>
Co-authored-by: Luuk van Baal <luukvbaal@gmail.com>
Co-authored-by: VanaIgr <vanaigranov@gmail.com>
---
 runtime/lua/vim/treesitter/query.lua | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index b0b0fecd38..66ab0d52f0 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -913,8 +913,8 @@ end
 ---@param start? integer Starting line for the search. Defaults to `node:start()`.
 ---@param stop? integer Stopping line for the search (end-exclusive). Defaults to `node:end_()`.
 ---
----@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, TSQueryMatch):
----        capture id, capture node, metadata, match
+---@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata, TSQueryMatch, TSTree):
+---        capture id, capture node, metadata, match, tree
 ---
 ---@note Captures are only returned if the query pattern of a specific capture contained predicates.
 function Query:iter_captures(node, source, start, stop)
@@ -924,6 +924,8 @@ function Query:iter_captures(node, source, start, stop)
 
   start, stop = value_or_node_range(start, stop, node)
 
+  -- Copy the tree to ensure it is valid during the entire lifetime of the iterator
+  local tree = node:tree():copy()
   local cursor = vim._create_ts_querycursor(node, self.query, start, stop, { match_limit = 256 })
 
   -- For faster checks that a match is not in the cache.
@@ -970,7 +972,7 @@ function Query:iter_captures(node, source, start, stop)
       match_cache[match_id] = metadata
     end
 
-    return capture, captured_node, metadata, match
+    return capture, captured_node, metadata, match, tree
   end
   return iter
 end
@@ -1011,7 +1013,7 @@ end
 ---   (last) node instead of the full list of matching nodes. This option is only for backward
 ---   compatibility and will be removed in a future release.
 ---
----@return (fun(): integer, table<integer, TSNode[]>, vim.treesitter.query.TSMetadata): pattern id, match, metadata
+---@return (fun(): integer, table<integer, TSNode[]>, vim.treesitter.query.TSMetadata, TSTree): pattern id, match, metadata, tree
 function Query:iter_matches(node, source, start, stop, opts)
   opts = opts or {}
   opts.match_limit = opts.match_limit or 256
@@ -1022,6 +1024,8 @@ function Query:iter_matches(node, source, start, stop, opts)
 
   start, stop = value_or_node_range(start, stop, node)
 
+  -- Copy the tree to ensure it is valid during the entire lifetime of the iterator
+  local tree = node:tree():copy()
   local cursor = vim._create_ts_querycursor(node, self.query, start, stop, opts)
 
   local function iter()
@@ -1059,7 +1063,7 @@ function Query:iter_matches(node, source, start, stop, opts)
     end
 
     -- TODO(lewis6991): create a new function that returns {match, metadata}
-    return pattern_i, captures, metadata
+    return pattern_i, captures, metadata, tree
   end
   return iter
 end
-- 
cgit 


From 09e01437c968be4c6e9f6bb3ac8811108c58008c Mon Sep 17 00:00:00 2001
From: Maria José Solano <majosolano99@gmail.com>
Date: Mon, 13 Jan 2025 19:45:11 -0800
Subject: refactor: use nvim.foo.bar format for autocommand groups

---
 runtime/lua/vim/treesitter/query.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 66ab0d52f0..ad648f36cc 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -289,7 +289,7 @@ end, false)
 
 api.nvim_create_autocmd('OptionSet', {
   pattern = { 'runtimepath' },
-  group = api.nvim_create_augroup('ts_query_cache_reset', { clear = true }),
+  group = api.nvim_create_augroup('nvim.treesitter.query_cache_reset', { clear = true }),
   callback = function()
     M.get:clear()
   end,
-- 
cgit 


From 09bcb310681e3b87d5b8c5eb547b182554cff7b4 Mon Sep 17 00:00:00 2001
From: Evgeni Chasnovski <evgeni.chasnovski@gmail.com>
Date: Wed, 15 Jan 2025 12:36:00 +0200
Subject: fix(docs): replace `yxx` mappings with `g==` #31947

Problem:
`yx` uses "y" prefix, which shadows a builtin operator.

Solution:
Use `g=` (in the form of `g==` currently), drawing from precedent of
CTRL-= and 'tpope/vim-scriptease'.
---
 runtime/lua/vim/treesitter/query.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index ad648f36cc..e43d0a8ad4 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -301,7 +301,7 @@ api.nvim_create_autocmd('OptionSet', {
 ---   - `captures`: a list of unique capture names defined in the query (alias: `info.captures`).
 ---   - `info.patterns`: information about predicates.
 ---
---- Example (to try it, use `yxx` or select the code then run `:'<,'>lua`):
+--- Example (to try it, use `g==` or select the code then run `:'<,'>lua`):
 --- ```lua
 --- local query = vim.treesitter.query.parse('vimdoc', [[
 ---   ; query
-- 
cgit 


From 6aa42e8f92bd8bea49b7b2accfe4ab67a5344e41 Mon Sep 17 00:00:00 2001
From: Lewis Russell <lewis6991@gmail.com>
Date: Fri, 24 Jan 2025 13:01:25 +0000
Subject: fix: resolve all remaining LuaLS diagnostics

---
 runtime/lua/vim/treesitter/query.lua | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index e43d0a8ad4..8055270a7f 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -262,6 +262,7 @@ local explicit_queries = setmetatable({}, {
 ---@param query_name string Name of the query (e.g., "highlights")
 ---@param text string Query text (unparsed).
 function M.set(lang, query_name, text)
+  --- @diagnostic disable-next-line: undefined-field LuaLS bad at generics
   M.get:clear(lang, query_name)
   explicit_queries[lang][query_name] = M.parse(lang, text)
 end
@@ -291,6 +292,7 @@ api.nvim_create_autocmd('OptionSet', {
   pattern = { 'runtimepath' },
   group = api.nvim_create_augroup('nvim.treesitter.query_cache_reset', { clear = true }),
   callback = function()
+    --- @diagnostic disable-next-line: undefined-field LuaLS bad at generics
     M.get:clear()
   end,
 })
-- 
cgit 


From da0ae953490098c28bad4791e08e2cc4c2e385e2 Mon Sep 17 00:00:00 2001
From: Maria José Solano <majosolano99@gmail.com>
Date: Tue, 28 Jan 2025 23:59:28 -0800
Subject: feat(treesitter): support modelines in `query.set()` (#30257)

---
 runtime/lua/vim/treesitter/query.lua | 66 +++++++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 12 deletions(-)

(limited to 'runtime/lua/vim/treesitter/query.lua')

diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 8055270a7f..10fb82e533 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -5,6 +5,9 @@ local api = vim.api
 local language = require('vim.treesitter.language')
 local memoize = vim.func._memoize
 
+local MODELINE_FORMAT = '^;+%s*inherits%s*:?%s*([a-z_,()]+)%s*$'
+local EXTENDS_FORMAT = '^;+%s*extends%s*$'
+
 local M = {}
 
 local function is_directive(name)
@@ -167,9 +170,6 @@ function M.get_files(lang, query_name, is_included)
   -- ;+ inherits: ({language},)*{language}
   --
   -- {language} ::= {lang} | ({lang})
-  local MODELINE_FORMAT = '^;+%s*inherits%s*:?%s*([a-z_,()]+)%s*$'
-  local EXTENDS_FORMAT = '^;+%s*extends%s*$'
-
   for _, filename in ipairs(lang_files) do
     local file, err = io.open(filename, 'r')
     if not file then
@@ -242,8 +242,8 @@ local function read_query_files(filenames)
   return table.concat(contents, '')
 end
 
--- The explicitly set queries from |vim.treesitter.query.set()|
----@type table<string,table<string,vim.treesitter.Query>>
+-- The explicitly set query strings from |vim.treesitter.query.set()|
+---@type table<string,table<string,string>>
 local explicit_queries = setmetatable({}, {
   __index = function(t, k)
     local lang_queries = {}
@@ -255,16 +255,27 @@ local explicit_queries = setmetatable({}, {
 
 --- Sets the runtime query named {query_name} for {lang}
 ---
---- This allows users to override any runtime files and/or configuration
+--- This allows users to override or extend any runtime files and/or configuration
 --- set by plugins.
 ---
+--- For example, you could enable spellchecking of `C` identifiers with the
+--- following code:
+--- ```lua
+--- vim.treesitter.query.set(
+---   'c',
+---   'highlights',
+---   [[;inherits c
+---   (identifier) @spell]])
+--- ]])
+--- ```
+---
 ---@param lang string Language to use for the query
 ---@param query_name string Name of the query (e.g., "highlights")
 ---@param text string Query text (unparsed).
 function M.set(lang, query_name, text)
   --- @diagnostic disable-next-line: undefined-field LuaLS bad at generics
   M.get:clear(lang, query_name)
-  explicit_queries[lang][query_name] = M.parse(lang, text)
+  explicit_queries[lang][query_name] = text
 end
 
 --- Returns the runtime query {query_name} for {lang}.
@@ -274,12 +285,43 @@ end
 ---
 ---@return vim.treesitter.Query? : Parsed query. `nil` if no query files are found.
 M.get = memoize('concat-2', function(lang, query_name)
+  local query_string ---@type string
+
   if explicit_queries[lang][query_name] then
-    return explicit_queries[lang][query_name]
-  end
+    local query_files = {}
+    local base_langs = {} ---@type string[]
+
+    for line in explicit_queries[lang][query_name]:gmatch('([^\n]*)\n?') do
+      if not vim.startswith(line, ';') then
+        break
+      end
+
+      local lang_list = line:match(MODELINE_FORMAT)
+      if lang_list then
+        for _, incl_lang in ipairs(vim.split(lang_list, ',')) do
+          local is_optional = incl_lang:match('%(.*%)')
 
-  local query_files = M.get_files(lang, query_name)
-  local query_string = read_query_files(query_files)
+          if is_optional then
+            add_included_lang(base_langs, lang, incl_lang:sub(2, #incl_lang - 1))
+          else
+            add_included_lang(base_langs, lang, incl_lang)
+          end
+        end
+      elseif line:match(EXTENDS_FORMAT) then
+        table.insert(base_langs, lang)
+      end
+    end
+
+    for _, base_lang in ipairs(base_langs) do
+      local base_files = M.get_files(base_lang, query_name, true)
+      vim.list_extend(query_files, base_files)
+    end
+
+    query_string = read_query_files(query_files) .. explicit_queries[lang][query_name]
+  else
+    local query_files = M.get_files(lang, query_name)
+    query_string = read_query_files(query_files)
+  end
 
   if #query_string == 0 then
     return nil
@@ -303,7 +345,7 @@ api.nvim_create_autocmd('OptionSet', {
 ---   - `captures`: a list of unique capture names defined in the query (alias: `info.captures`).
 ---   - `info.patterns`: information about predicates.
 ---
---- Example (to try it, use `g==` or select the code then run `:'<,'>lua`):
+--- Example:
 --- ```lua
 --- local query = vim.treesitter.query.parse('vimdoc', [[
 ---   ; query
-- 
cgit