From 2f9ee9b6cfc61a0504fc0bc22bdf481828e2ea91 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 9 Jan 2024 17:36:46 +0000 Subject: fix(doc): improve doc generation of types using lpeg Added a lpeg grammar for LuaCATS and use it in lua2dox.lua --- runtime/lua/vim/treesitter/query.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 8cbbffcd60..cc8fe319e8 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -793,7 +793,7 @@ end --- of the query file, e.g., if the path ends in `/lua/highlights.scm`, the parser for the --- `lua` language will be used. ---@param buf (integer) Buffer handle ----@param opts (QueryLinterOpts|nil) Optional keyword arguments: +---@param opts? QueryLinterOpts (table) Optional keyword arguments: --- - langs (string|string[]|nil) Language(s) to use for checking the query. --- If multiple languages are specified, queries are validated for all of them --- - clear (boolean) if `true`, just clear current lint errors -- cgit From 617d1b28d6d3241199a9ea4f7e68b650549d3251 Mon Sep 17 00:00:00 2001 From: Phạm Huy Hoàng Date: Tue, 23 Jan 2024 18:06:54 +0900 Subject: fix(treesitter): prefix treesitter types with vim --- runtime/lua/vim/treesitter/query.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index cc8fe319e8..23e5ff1e6b 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -12,7 +12,7 @@ Query.__index = Query ---@field captures table ---@field patterns table ----@class TSQueryModule +---@class vim.treesitter.query local M = {} ---@param files string[] -- cgit From 2e982f1aad9f1a03562b7a451d642f76b04c37cb Mon Sep 17 00:00:00 2001 From: dundargoc Date: Mon, 22 Jan 2024 18:23:28 +0100 Subject: refactor: create function for deferred loading The benefit of this is that users only pay for what they use. If e.g. only `vim.lsp.buf_get_clients()` is called then they don't need to load all modules under `vim.lsp` which could lead to significant startuptime saving. Also `vim.lsp.module` is a bit nicer to user compared to `require("vim.lsp.module")`. This isn't used for some nested modules such as `filetype` as it breaks tests with error messages such as "attempt to index field 'detect'". It's not entirely certain the reason for this, but it is likely it is due to filetype being precompiled which would imply deferred loading isn't needed for performance reasons. --- runtime/lua/vim/treesitter/query.lua | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 23e5ff1e6b..63d4a9382a 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -12,7 +12,6 @@ Query.__index = Query ---@field captures table ---@field patterns table ----@class vim.treesitter.query local M = {} ---@param files string[] @@ -799,9 +798,9 @@ end --- - clear (boolean) if `true`, just clear current lint errors function M.lint(buf, opts) if opts and opts.clear then - require('vim.treesitter._query_linter').clear(buf) + vim.treesitter._query_linter.clear(buf) else - require('vim.treesitter._query_linter').lint(buf, opts) + vim.treesitter._query_linter.lint(buf, opts) end end @@ -814,7 +813,7 @@ end --- ``` --- function M.omnifunc(findstart, base) - return require('vim.treesitter._query_linter').omnifunc(findstart, base) + return vim.treesitter._query_linter.omnifunc(findstart, base) end --- Opens a live editor to query the buffer you started from. @@ -827,7 +826,7 @@ end --- --- @param lang? string language to open the query editor for. If omitted, inferred from the current buffer's filetype. function M.edit(lang) - require('vim.treesitter.dev').edit_query(lang) + vim.treesitter.dev.edit_query(lang) end return M -- cgit From 800134ea5ec60338a40280c8536db6a6a4a10249 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 25 Jan 2024 13:27:48 -0500 Subject: refactor(treesitter): typing for Query, TSQuery, and TSQueryInfo - `TSQuery`: userdata object for parsed query. - `vim.treesitter.Query`: renamed from `Query`. - Add a new field `lang`. - `TSQueryInfo`: - Move to `vim/treesitter/_meta.lua`, because C code owns it. - Correct typing for `patterns`, should be a map from `integer` (pattern_id) to `(integer|string)[][]` (list of predicates or directives). - `vim.treesitter.QueryInfo` is added. - This currently has the same structure as `TSQueryInfo` (exported from C code). - Document the fields (see `TSQuery:inspect`). - Add typing for `vim._ts_parse_query()`. --- runtime/lua/vim/treesitter/query.lua | 68 +++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 20 deletions(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 63d4a9382a..7631c6c4a2 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -1,18 +1,48 @@ local api = vim.api local language = require('vim.treesitter.language') ----@class Query ----@field captures string[] List of captures used in query ----@field info TSQueryInfo Contains used queries, predicates, directives ----@field query userdata Parsed query +local M = {} + +---Parsed query, see |vim.treesitter.query.parse()| +--- +---@class vim.treesitter.Query +---@field lang string name of the language for this parser +---@field captures string[] list of (unique) capture names defined in query +---@field info vim.treesitter.QueryInfo contains information used in the query (e.g. captures, predicates, directives) +---@field query TSQuery userdata query object local Query = {} Query.__index = Query ----@class TSQueryInfo ----@field captures table ----@field patterns table +---@package +---@see vim.treesitter.query.parse +---@param lang string +---@param ts_query TSQuery +---@return vim.treesitter.Query +function Query.new(lang, ts_query) + local self = setmetatable({}, Query) + local query_info = ts_query:inspect() ---@type TSQueryInfo + self.query = ts_query + self.lang = lang + self.info = { + captures = query_info.captures, + patterns = query_info.patterns, + } + self.captures = self.info.captures + return self +end -local M = {} +---Information for Query, see |vim.treesitter.query.parse()| +---@class vim.treesitter.QueryInfo +--- +---List of (unique) capture names defined in query. +---@field captures string[] +--- +---Contains information about predicates and directives. +---Key is pattern id, and value is list of predicates or directives defined in the pattern. +---A predicate or directive is a list of (integer|string); integer represents `capture_id`, and +---string represents (literal) arguments to predicate/directive. See |treesitter-predicates| +---and |treesitter-directives| for more details. +---@field patterns table ---@param files string[] ---@return string[] @@ -162,7 +192,7 @@ local function read_query_files(filenames) end -- The explicitly set queries from |vim.treesitter.query.set()| ----@type table> +---@type table> local explicit_queries = setmetatable({}, { __index = function(t, k) local lang_queries = {} @@ -201,7 +231,7 @@ end ---@param lang string Language to use for the query ---@param query_name string Name of the query (e.g. "highlights") --- ----@return Query|nil Parsed query +---@return vim.treesitter.Query|nil -- Parsed query. `nil` if no query files are found. M.get = vim.func._memoize('concat-2', function(lang, query_name) if explicit_queries[lang][query_name] then return explicit_queries[lang][query_name] @@ -228,26 +258,24 @@ end --- --- Returns a `Query` (see |lua-treesitter-query|) object which can be used to --- search nodes in the syntax tree for the patterns defined in {query} ---- using `iter_*` methods below. +--- using the `iter_captures` and `iter_matches` methods. --- --- Exposes `info` and `captures` with additional context about {query}. ---- - `captures` contains the list of unique capture names defined in ---- {query}. ---- -` info.captures` also points to `captures`. +--- - `captures` contains the list of unique capture names defined in {query}. +--- - `info.captures` also points to `captures`. --- - `info.patterns` contains information about predicates. --- ---@param lang string Language to use for the query ---@param query string Query in s-expr syntax --- ----@return Query Parsed query +---@return vim.treesitter.Query Parsed query +--- +---@see |vim.treesitter.query.get()| M.parse = vim.func._memoize('concat-2', function(lang, query) language.add(lang) - local self = setmetatable({}, Query) - self.query = vim._ts_parse_query(lang, query) - self.info = self.query:inspect() - self.captures = self.info.captures - return self + local ts_query = vim._ts_parse_query(lang, query) + return Query.new(lang, ts_query) end) ---@deprecated -- cgit From d0e9e36a7841c28f82e5c7ae2bde1fa21319f2ac Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Fri, 2 Feb 2024 01:51:35 -0500 Subject: refactor(treesitter): {start,stop} are optional in Query:iter_* methods Document that the `start` and `stop` parameters in `Query:iter_captures()` and `Query:iter_matches()` are optional. The tree-sitter lib has been bumped up to 0.20.9, so we also no longer need "Requires treesitter >= 0.20.9". --- runtime/lua/vim/treesitter/query.lua | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 7631c6c4a2..cd65c0d7f6 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -672,14 +672,16 @@ end --- Returns the start and stop value if set else the node's range. -- When the node's range is used, the stop is incremented by 1 -- to make the search inclusive. ----@param start integer ----@param stop integer +---@param start integer|nil +---@param stop integer|nil ---@param node TSNode ---@return integer, integer local function value_or_node_range(start, stop, node) - if start == nil and stop == nil then - local node_start, _, node_stop, _ = node:range() - return node_start, node_stop + 1 -- Make stop inclusive + if start == nil then + start = node:start() + end + if stop == nil then + stop = node:end_() + 1 -- Make stop inclusive end return start, stop @@ -710,8 +712,8 @@ end --- ---@param node TSNode under which the search will occur ---@param source (integer|string) Source buffer or string to extract text from ----@param start integer Starting line for the search ----@param stop integer Stopping line for the search (end-exclusive) +---@param start? integer Starting line for the search. Defaults to `node:start()`. +---@param stop? integer Stopping line for the search (end-exclusive). Defaults to `node:end_()`. --- ---@return (fun(end_line: integer|nil): integer, TSNode, TSMetadata): --- capture id, capture node, metadata @@ -769,12 +771,11 @@ end --- ---@param node TSNode under which the search will occur ---@param source (integer|string) Source buffer or string to search ----@param start integer Starting line for the search ----@param stop integer Stopping line for the search (end-exclusive) ----@param opts table|nil Options: +---@param start? integer Starting line for the search. Defaults to `node:start()`. +---@param stop? integer Stopping line for the search (end-exclusive). Defaults to `node:end_()`. +---@param opts? table Optional keyword arguments: --- - max_start_depth (integer) if non-zero, sets the maximum start depth --- for each match. This is used to prevent traversing too deep into a tree. ---- Requires treesitter >= 0.20.9. --- ---@return (fun(): integer, table, table): pattern id, match, metadata function Query:iter_matches(node, source, start, stop, opts) -- cgit From bd5008de07d29a6457ddc7fe13f9f85c9c4619d2 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Fri, 16 Feb 2024 18:54:47 +0100 Subject: fix(treesitter): correctly handle query quantifiers (#24738) Query patterns can contain quantifiers (e.g. (foo)+ @bar), so a single capture can map to multiple nodes. The iter_matches API can not handle this situation because the match table incorrectly maps capture indices to a single node instead of to an array of nodes. The match table should be updated to map capture indices to an array of nodes. However, this is a massively breaking change, so must be done with a proper deprecation period. `iter_matches`, `add_predicate` and `add_directive` must opt-in to the correct behavior for backward compatibility. This is done with a new "all" option. This option will become the default and removed after the 0.10 release. Co-authored-by: Christian Clason Co-authored-by: MDeiml Co-authored-by: Gregory Anders --- runtime/lua/vim/treesitter/query.lua | 422 +++++++++++++++++++++++++---------- 1 file changed, 300 insertions(+), 122 deletions(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index cd65c0d7f6..5bb9e07a82 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -290,47 +290,71 @@ function M.get_node_text(...) return vim.treesitter.get_node_text(...) end ----@alias TSMatch table - ----@alias TSPredicate fun(match: TSMatch, _, _, predicate: any[]): boolean - --- Predicate handler receive the following arguments --- (match, pattern, bufnr, predicate) ----@type table -local predicate_handlers = { - ['eq?'] = function(match, _, source, predicate) - local node = match[predicate[2]] - if not node then +--- Implementations of predicates that can optionally be prefixed with "any-". +--- +--- These functions contain the implementations for each predicate, correctly +--- handling the "any" vs "all" semantics. They are called from the +--- predicate_handlers table with the appropriate arguments for each predicate. +local impl = { + --- @param match TSMatch + --- @param source integer|string + --- @param predicate any[] + --- @param any boolean + ['eq'] = function(match, source, predicate, any) + local nodes = match[predicate[2]] + if not nodes or #nodes == 0 then return true end - local node_text = vim.treesitter.get_node_text(node, source) - local str ---@type string - if type(predicate[3]) == 'string' then - -- (#eq? @aa "foo") - str = predicate[3] - else - -- (#eq? @aa @bb) - str = vim.treesitter.get_node_text(match[predicate[3]], source) - end + for _, node in ipairs(nodes) do + local node_text = vim.treesitter.get_node_text(node, source) + + local str ---@type string + if type(predicate[3]) == 'string' then + -- (#eq? @aa "foo") + str = predicate[3] + else + -- (#eq? @aa @bb) + local other = assert(match[predicate[3]]) + assert(#other == 1, '#eq? does not support comparison with captures on multiple nodes') + str = vim.treesitter.get_node_text(other[1], source) + end - if node_text ~= str or str == nil then - return false + local res = str ~= nil and node_text == str + if any and res then + return true + elseif not any and not res then + return false + end end - return true + return not any end, - ['lua-match?'] = function(match, _, source, predicate) - local node = match[predicate[2]] - if not node then + --- @param match TSMatch + --- @param source integer|string + --- @param predicate any[] + --- @param any boolean + ['lua-match'] = function(match, source, predicate, any) + local nodes = match[predicate[2]] + if not nodes or #nodes == 0 then return true end - local regex = predicate[3] - return string.find(vim.treesitter.get_node_text(node, source), regex) ~= nil + + for _, node in ipairs(nodes) do + local regex = predicate[3] + local res = string.find(vim.treesitter.get_node_text(node, source), regex) ~= nil + if any and res then + return true + elseif not any and not res then + return false + end + end + + return not any end, - ['match?'] = (function() + ['match'] = (function() local magic_prefixes = { ['\\v'] = true, ['\\m'] = true, ['\\M'] = true, ['\\V'] = true } local function check_magic(str) if string.len(str) < 2 or magic_prefixes[string.sub(str, 1, 2)] then @@ -347,85 +371,160 @@ local predicate_handlers = { end, }) - return function(match, _, source, pred) - ---@cast match TSMatch - local node = match[pred[2]] - if not node then + --- @param match TSMatch + --- @param source integer|string + --- @param predicate any[] + --- @param any boolean + return function(match, source, predicate, any) + local nodes = match[predicate[2]] + if not nodes or #nodes == 0 then return true end - ---@diagnostic disable-next-line no-unknown - local regex = compiled_vim_regexes[pred[3]] - return regex:match_str(vim.treesitter.get_node_text(node, source)) + + for _, node in ipairs(nodes) do + local regex = compiled_vim_regexes[predicate[3]] ---@type vim.regex + local res = regex:match_str(vim.treesitter.get_node_text(node, source)) + if any and res then + return true + elseif not any and not res then + return false + end + end + return not any end end)(), - ['contains?'] = function(match, _, source, predicate) - local node = match[predicate[2]] - if not node then + --- @param match TSMatch + --- @param source integer|string + --- @param predicate any[] + --- @param any boolean + ['contains'] = function(match, source, predicate, any) + local nodes = match[predicate[2]] + if not nodes or #nodes == 0 then return true end - local node_text = vim.treesitter.get_node_text(node, source) - for i = 3, #predicate do - if string.find(node_text, predicate[i], 1, true) then - return true + for _, node in ipairs(nodes) do + local node_text = vim.treesitter.get_node_text(node, source) + + for i = 3, #predicate do + local res = string.find(node_text, predicate[i], 1, true) + if any and res then + return true + elseif not any and not res then + return false + end end end - return false + return not any + end, +} + +---@class TSMatch +---@field pattern? integer +---@field active? boolean +---@field [integer] TSNode[] + +---@alias TSPredicate fun(match: TSMatch, pattern: integer, source: integer|string, predicate: any[]): boolean + +-- Predicate handler receive the following arguments +-- (match, pattern, bufnr, predicate) +---@type table +local predicate_handlers = { + ['eq?'] = function(match, _, source, predicate) + return impl['eq'](match, source, predicate, false) + end, + + ['any-eq?'] = function(match, _, source, predicate) + return impl['eq'](match, source, predicate, true) + end, + + ['lua-match?'] = function(match, _, source, predicate) + return impl['lua-match'](match, source, predicate, false) + end, + + ['any-lua-match?'] = function(match, _, source, predicate) + return impl['lua-match'](match, source, predicate, true) + end, + + ['match?'] = function(match, _, source, predicate) + return impl['match'](match, source, predicate, false) + end, + + ['any-match?'] = function(match, _, source, predicate) + return impl['match'](match, source, predicate, true) + end, + + ['contains?'] = function(match, _, source, predicate) + return impl['contains'](match, source, predicate, false) + end, + + ['any-contains?'] = function(match, _, source, predicate) + return impl['contains'](match, source, predicate, true) end, ['any-of?'] = function(match, _, source, predicate) - local node = match[predicate[2]] - if not node then + local nodes = match[predicate[2]] + if not nodes or #nodes == 0 then return true end - local node_text = vim.treesitter.get_node_text(node, source) - -- Since 'predicate' will not be used by callers of this function, use it - -- to store a string set built from the list of words to check against. - local string_set = predicate['string_set'] - if not string_set then - string_set = {} - for i = 3, #predicate do - ---@diagnostic disable-next-line:no-unknown - string_set[predicate[i]] = true + for _, node in ipairs(nodes) do + local node_text = vim.treesitter.get_node_text(node, source) + + -- Since 'predicate' will not be used by callers of this function, use it + -- to store a string set built from the list of words to check against. + local string_set = predicate['string_set'] --- @type table + if not string_set then + string_set = {} + for i = 3, #predicate do + string_set[predicate[i]] = true + end + predicate['string_set'] = string_set + end + + if string_set[node_text] then + return true end - predicate['string_set'] = string_set end - return string_set[node_text] + return false end, ['has-ancestor?'] = function(match, _, _, predicate) - local node = match[predicate[2]] - if not node then + local nodes = match[predicate[2]] + if not nodes or #nodes == 0 then return true end - local ancestor_types = {} - for _, type in ipairs({ unpack(predicate, 3) }) do - ancestor_types[type] = true - end + for _, node in ipairs(nodes) do + local ancestor_types = {} --- @type table + for _, type in ipairs({ unpack(predicate, 3) }) do + ancestor_types[type] = true + end - node = node:parent() - while node do - if ancestor_types[node:type()] then - return true + local cur = node:parent() + while cur do + if ancestor_types[cur:type()] then + return true + end + cur = cur:parent() end - node = node:parent() end return false end, ['has-parent?'] = function(match, _, _, predicate) - local node = match[predicate[2]] - if not node then + local nodes = match[predicate[2]] + if not nodes or #nodes == 0 then return true end - if vim.list_contains({ unpack(predicate, 3) }, node:parent():type()) then - return true + for _, node in ipairs(nodes) do + if vim.list_contains({ unpack(predicate, 3) }, node:parent():type()) then + return true + end end return false end, @@ -433,6 +532,7 @@ local predicate_handlers = { -- As we provide lua-match? also expose vim-match? predicate_handlers['vim-match?'] = predicate_handlers['match?'] +predicate_handlers['any-vim-match?'] = predicate_handlers['any-match?'] ---@class TSMetadata ---@field range? Range @@ -468,13 +568,17 @@ local directive_handlers = { -- Shifts the range of a node. -- Example: (#offset! @_node 0 1 0 -1) ['offset!'] = function(match, _, _, pred, metadata) - ---@cast pred integer[] - local capture_id = pred[2] + local capture_id = pred[2] --[[@as integer]] + local nodes = match[capture_id] + assert(#nodes == 1, '#offset! does not support captures on multiple nodes') + + local node = nodes[1] + if not metadata[capture_id] then metadata[capture_id] = {} end - local range = metadata[capture_id].range or { match[capture_id]:range() } + local range = metadata[capture_id].range or { node:range() } local start_row_offset = pred[3] or 0 local start_col_offset = pred[4] or 0 local end_row_offset = pred[5] or 0 @@ -498,7 +602,9 @@ local directive_handlers = { local id = pred[2] assert(type(id) == 'number') - local node = match[id] + local nodes = match[id] + assert(#nodes == 1, '#gsub! does not support captures on multiple nodes') + local node = nodes[1] local text = vim.treesitter.get_node_text(node, bufnr, { metadata = metadata[id] }) or '' if not metadata[id] then @@ -518,10 +624,9 @@ local directive_handlers = { local capture_id = pred[2] assert(type(capture_id) == 'number') - local node = match[capture_id] - if not node then - return - end + local nodes = match[capture_id] + assert(#nodes == 1, '#trim! does not support captures on multiple nodes') + local node = nodes[1] local start_row, start_col, end_row, end_col = node:range() @@ -552,38 +657,93 @@ local directive_handlers = { --- Adds a new predicate to be used in queries --- ---@param name string Name of the predicate, without leading # ----@param handler function(match:table, pattern:string, bufnr:integer, predicate:string[]) +---@param handler function(match: table, pattern: integer, source: integer|string, predicate: any[], metadata: table) --- - see |vim.treesitter.query.add_directive()| for argument meanings ----@param force boolean|nil -function M.add_predicate(name, handler, force) - if predicate_handlers[name] and not force then - error(string.format('Overriding %s', name)) +---@param opts table Optional options: +--- - force (boolean): Override an existing +--- predicate of the same name +--- - all (boolean): Use the correct +--- implementation of the match table where +--- capture IDs map to a list of nodes instead +--- of a single node. Defaults to false (for +--- backward compatibility). This option will +--- eventually become the default and removed. +function M.add_predicate(name, handler, opts) + -- Backward compatibility: old signature had "force" as boolean argument + if type(opts) == 'boolean' then + opts = { force = opts } end - predicate_handlers[name] = handler + opts = opts or {} + + if predicate_handlers[name] and not opts.force then + error(string.format('Overriding existing predicate %s', name)) + end + + if opts.all then + predicate_handlers[name] = handler + else + --- @param match table + local function wrapper(match, ...) + local m = {} ---@type table + for k, v in pairs(match) do + if type(k) == 'number' then + m[k] = v[#v] + end + end + return handler(m, ...) + end + predicate_handlers[name] = wrapper + end end --- Adds a new directive to be used in queries --- --- Handlers can set match level data by setting directly on the ---- metadata object `metadata.key = value`, additionally, handlers +--- metadata object `metadata.key = value`. Additionally, handlers --- can set node level data by using the capture id on the --- metadata table `metadata[capture_id].key = value` --- ---@param name string Name of the directive, without leading # ----@param handler function(match:table, pattern:string, bufnr:integer, predicate:string[], metadata:table) ---- - match: see |treesitter-query| ---- - node-level data are accessible via `match[capture_id]` ---- - pattern: see |treesitter-query| +---@param handler function(match: table, pattern: integer, source: integer|string, predicate: any[], metadata: table) +--- - match: A table mapping capture IDs to a list of captured nodes +--- - pattern: the index of the matching pattern in the query file --- - predicate: list of strings containing the full directive being called, e.g. --- `(node (#set! conceal "-"))` would get the predicate `{ "#set!", "conceal", "-" }` ----@param force boolean|nil -function M.add_directive(name, handler, force) - if directive_handlers[name] and not force then - error(string.format('Overriding %s', name)) +---@param opts table Optional options: +--- - force (boolean): Override an existing +--- predicate of the same name +--- - all (boolean): Use the correct +--- implementation of the match table where +--- capture IDs map to a list of nodes instead +--- of a single node. Defaults to false (for +--- backward compatibility). This option will +--- eventually become the default and removed. +function M.add_directive(name, handler, opts) + -- Backward compatibility: old signature had "force" as boolean argument + if type(opts) == 'boolean' then + opts = { force = opts } + end + + opts = opts or {} + + if directive_handlers[name] and not opts.force then + error(string.format('Overriding existing directive %s', name)) end - directive_handlers[name] = handler + if opts.all then + directive_handlers[name] = handler + else + --- @param match table + local function wrapper(match, ...) + local m = {} ---@type table + for k, v in pairs(match) do + m[k] = v[#v] + end + handler(m, ...) + end + directive_handlers[name] = wrapper + end end --- Lists the currently available directives to use in queries. @@ -608,7 +768,7 @@ end ---@private ---@param match TSMatch ----@param pattern string +---@param pattern integer ---@param source integer|string function Query:match_preds(match, pattern, source) local preds = self.info.patterns[pattern] @@ -618,18 +778,14 @@ function Query:match_preds(match, pattern, source) -- continue on the other case. This way unknown predicates will not be considered, -- which allows some testing and easier user extensibility (#12173). -- Also, tree-sitter strips the leading # from predicates for us. - local pred_name ---@type string - - local is_not ---@type boolean + local is_not = false -- Skip over directives... they will get processed after all the predicates. if not is_directive(pred[1]) then - if string.sub(pred[1], 1, 4) == 'not-' then - pred_name = string.sub(pred[1], 5) + local pred_name = pred[1] + if pred_name:match('^not%-') then + pred_name = pred_name:sub(5) is_not = true - else - pred_name = pred[1] - is_not = false end local handler = predicate_handlers[pred_name] @@ -724,7 +880,7 @@ function Query:iter_captures(node, source, start, stop) start, stop = value_or_node_range(start, stop, node) - local raw_iter = node:_rawquery(self.query, true, start, stop) + local raw_iter = node:_rawquery(self.query, true, start, stop) ---@type fun(): integer, TSNode, TSMatch local function iter(end_line) local capture, captured_node, match = raw_iter() local metadata = {} @@ -748,27 +904,34 @@ end --- Iterates the matches of self on a given range. --- ---- Iterate over all matches within a {node}. The arguments are the same as ---- for |Query:iter_captures()| but the iterated values are different: ---- an (1-based) index of the pattern in the query, a table mapping ---- capture indices to nodes, and metadata from any directives processing the match. ---- If the query has more than one pattern, the capture table might be sparse ---- and e.g. `pairs()` method should be used over `ipairs`. ---- Here is an example iterating over all captures in every match: +--- Iterate over all matches within a {node}. The arguments are the same as for +--- |Query:iter_captures()| but the iterated values are different: an (1-based) +--- index of the pattern in the query, a table mapping capture indices to a list +--- of nodes, and metadata from any directives processing the match. +--- +--- WARNING: Set `all=true` to ensure all matching nodes in a match are +--- returned, otherwise only the last node in a match is returned, breaking captures +--- involving quantifiers such as `(comment)+ @comment`. The default option +--- `all=false` is only provided for backward compatibility and will be removed +--- after Nvim 0.10. +--- +--- Example: --- --- ```lua ---- for pattern, match, metadata in cquery:iter_matches(tree:root(), bufnr, first, last) do ---- for id, node in pairs(match) do +--- for pattern, match, metadata in cquery:iter_matches(tree:root(), bufnr, 0, -1, { all = true }) do +--- for id, nodes in pairs(match) do --- local name = query.captures[id] ---- -- `node` was captured by the `name` capture in the match +--- for _, node in ipairs(nodes) do +--- -- `node` was captured by the `name` capture in the match --- ---- local node_data = metadata[id] -- Node level metadata ---- ---- -- ... use the info here ... +--- local node_data = metadata[id] -- Node level metadata +--- ... use the info here ... +--- end --- end --- end --- ``` --- +--- ---@param node TSNode under which the search will occur ---@param source (integer|string) Source buffer or string to search ---@param start? integer Starting line for the search. Defaults to `node:start()`. @@ -776,17 +939,20 @@ end ---@param opts? table Optional keyword arguments: --- - max_start_depth (integer) if non-zero, sets the maximum start depth --- for each match. This is used to prevent traversing too deep into a tree. +--- - all (boolean) When set, the returned match table maps capture IDs to a list of nodes. +--- Older versions of iter_matches incorrectly mapped capture IDs to a single node, which is +--- incorrect behavior. This option will eventually become the default and removed. --- ----@return (fun(): integer, table, table): pattern id, match, metadata +---@return (fun(): integer, table, table): pattern id, match, metadata function Query:iter_matches(node, source, start, stop, opts) + local all = opts and opts.all if type(source) == 'number' and source == 0 then source = api.nvim_get_current_buf() end start, stop = value_or_node_range(start, stop, node) - local raw_iter = node:_rawquery(self.query, false, start, stop, opts) - ---@cast raw_iter fun(): string, any + local raw_iter = node:_rawquery(self.query, false, start, stop, opts) ---@type fun(): integer, TSMatch local function iter() local pattern, match = raw_iter() local metadata = {} @@ -799,6 +965,18 @@ function Query:iter_matches(node, source, start, stop, opts) self:apply_directives(match, pattern, source, metadata) end + + if not all then + -- Convert the match table into the old buggy version for backward + -- compatibility. This is slow. Plugin authors, if you're reading this, set the "all" + -- option! + local old_match = {} ---@type table + for k, v in pairs(match or {}) do + old_match[k] = v[#v] + end + return pattern, old_match, metadata + end + return pattern, match, metadata end return iter -- cgit From 9beb40a4db5613601fc1a4b828a44e5977eca046 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Thu, 15 Feb 2024 17:16:04 +0000 Subject: feat(docs): replace lua2dox.lua Problem: The documentation flow (`gen_vimdoc.py`) has several issues: - it's not very versatile - depends on doxygen - doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C. - The intermediate XML files and filters makes it too much like a rube goldberg machine. Solution: Re-implement the flow using Lua, LPEG and treesitter. - `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic. - `lua2dox.lua` is gone! - No more XML files. - Doxygen is now longer used and instead we now use: - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`). - LPEG for C parsing (see `scripts/cdoc_parser.lua`) - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`). - Treesitter for Markdown parsing (see `scripts/text_utils.lua`). - The generated `runtime/doc/*.mpack` files have been removed. - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly. - Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change). --- runtime/lua/vim/treesitter/query.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 5bb9e07a82..57272dbd60 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -231,7 +231,7 @@ end ---@param lang string Language to use for the query ---@param query_name string Name of the query (e.g. "highlights") --- ----@return vim.treesitter.Query|nil -- Parsed query. `nil` if no query files are found. +---@return vim.treesitter.Query|nil : Parsed query. `nil` if no query files are found. M.get = vim.func._memoize('concat-2', function(lang, query_name) if explicit_queries[lang][query_name] then return explicit_queries[lang][query_name] @@ -1019,6 +1019,8 @@ end --- vim.bo.omnifunc = 'v:lua.vim.treesitter.query.omnifunc' --- ``` --- +--- @param findstart 0|1 +--- @param base string function M.omnifunc(findstart, base) return vim.treesitter._query_linter.omnifunc(findstart, base) end -- cgit From a5fe8f59d98398d04bed8586cee73864bbcdde92 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 27 Feb 2024 15:20:32 +0000 Subject: docs: improve/add documentation of Lua types - Added `@inlinedoc` so single use Lua types can be inlined into the functions docs. E.g. ```lua --- @class myopts --- @inlinedoc --- --- Documentation for some field --- @field somefield integer --- @param opts myOpts function foo(opts) end ``` Will be rendered as ``` foo(opts) Parameters: - {opts} (table) Object with the fields: - somefield (integer) Documentation for some field ``` - Marked many classes with with `@nodoc` or `(private)`. We can eventually introduce these when we want to. --- runtime/lua/vim/treesitter/query.lua | 52 +++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 22 deletions(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 57272dbd60..6c7f713fd7 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -3,6 +3,7 @@ local language = require('vim.treesitter.language') local M = {} +---@nodoc ---Parsed query, see |vim.treesitter.query.parse()| --- ---@class vim.treesitter.Query @@ -31,6 +32,7 @@ function Query.new(lang, ts_query) return self end +---@nodoc ---Information for Query, see |vim.treesitter.query.parse()| ---@class vim.treesitter.QueryInfo --- @@ -296,7 +298,7 @@ end --- handling the "any" vs "all" semantics. They are called from the --- predicate_handlers table with the appropriate arguments for each predicate. local impl = { - --- @param match TSMatch + --- @param match vim.treesitter.query.TSMatch --- @param source integer|string --- @param predicate any[] --- @param any boolean @@ -331,7 +333,7 @@ local impl = { return not any end, - --- @param match TSMatch + --- @param match vim.treesitter.query.TSMatch --- @param source integer|string --- @param predicate any[] --- @param any boolean @@ -371,7 +373,7 @@ local impl = { end, }) - --- @param match TSMatch + --- @param match vim.treesitter.query.TSMatch --- @param source integer|string --- @param predicate any[] --- @param any boolean @@ -394,7 +396,7 @@ local impl = { end end)(), - --- @param match TSMatch + --- @param match vim.treesitter.query.TSMatch --- @param source integer|string --- @param predicate any[] --- @param any boolean @@ -421,12 +423,13 @@ local impl = { end, } ----@class TSMatch +---@nodoc +---@class vim.treesitter.query.TSMatch ---@field pattern? integer ---@field active? boolean ---@field [integer] TSNode[] ----@alias TSPredicate fun(match: TSMatch, pattern: integer, source: integer|string, predicate: any[]): boolean +---@alias TSPredicate fun(match: vim.treesitter.query.TSMatch, pattern: integer, source: integer|string, predicate: any[]): boolean -- Predicate handler receive the following arguments -- (match, pattern, bufnr, predicate) @@ -534,13 +537,14 @@ local predicate_handlers = { predicate_handlers['vim-match?'] = predicate_handlers['match?'] predicate_handlers['any-vim-match?'] = predicate_handlers['any-match?'] ----@class TSMetadata +---@nodoc +---@class vim.treesitter.query.TSMetadata ---@field range? Range ---@field conceal? string ----@field [integer] TSMetadata +---@field [integer] vim.treesitter.query.TSMetadata ---@field [string] integer|string ----@alias TSDirective fun(match: TSMatch, _, _, predicate: (string|integer)[], metadata: TSMetadata) +---@alias TSDirective fun(match: vim.treesitter.query.TSMatch, _, _, predicate: (string|integer)[], metadata: vim.treesitter.query.TSMetadata) -- Predicate handler receive the following arguments -- (match, pattern, bufnr, predicate) @@ -767,7 +771,7 @@ local function is_directive(name) end ---@private ----@param match TSMatch +---@param match vim.treesitter.query.TSMatch ---@param pattern integer ---@param source integer|string function Query:match_preds(match, pattern, source) @@ -806,8 +810,8 @@ function Query:match_preds(match, pattern, source) end ---@private ----@param match TSMatch ----@param metadata TSMetadata +---@param match vim.treesitter.query.TSMatch +---@param metadata vim.treesitter.query.TSMetadata function Query:apply_directives(match, pattern, source, metadata) local preds = self.info.patterns[pattern] @@ -871,7 +875,7 @@ end ---@param start? integer Starting line for the search. Defaults to `node:start()`. ---@param stop? integer Stopping line for the search (end-exclusive). Defaults to `node:end_()`. --- ----@return (fun(end_line: integer|nil): integer, TSNode, TSMetadata): +---@return (fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata): --- capture id, capture node, metadata function Query:iter_captures(node, source, start, stop) if type(source) == 'number' and source == 0 then @@ -880,7 +884,7 @@ function Query:iter_captures(node, source, start, stop) start, stop = value_or_node_range(start, stop, node) - local raw_iter = node:_rawquery(self.query, true, start, stop) ---@type fun(): integer, TSNode, TSMatch + local raw_iter = node:_rawquery(self.query, true, start, stop) ---@type fun(): integer, TSNode, vim.treesitter.query.TSMatch local function iter(end_line) local capture, captured_node, match = raw_iter() local metadata = {} @@ -952,7 +956,7 @@ function Query:iter_matches(node, source, start, stop, opts) start, stop = value_or_node_range(start, stop, node) - local raw_iter = node:_rawquery(self.query, false, start, stop, opts) ---@type fun(): integer, TSMatch + local raw_iter = node:_rawquery(self.query, false, start, stop, opts) ---@type fun(): integer, vim.treesitter.query.TSMatch local function iter() local pattern, match = raw_iter() local metadata = {} @@ -982,9 +986,16 @@ function Query:iter_matches(node, source, start, stop, opts) return iter end ----@class QueryLinterOpts ----@field langs (string|string[]|nil) ----@field clear (boolean) +--- Optional keyword arguments: +--- @class vim.treesitter.query.lint.Opts +--- @inlinedoc +--- +--- Language(s) to use for checking the query. +--- If multiple languages are specified, queries are validated for all of them +--- @field langs? string|string[] +--- +--- Just clear current lint errors +--- @field clear boolean --- Lint treesitter queries using installed parser, or clear lint errors. --- @@ -999,10 +1010,7 @@ end --- of the query file, e.g., if the path ends in `/lua/highlights.scm`, the parser for the --- `lua` language will be used. ---@param buf (integer) Buffer handle ----@param opts? QueryLinterOpts (table) Optional keyword arguments: ---- - langs (string|string[]|nil) Language(s) to use for checking the query. ---- If multiple languages are specified, queries are validated for all of them ---- - clear (boolean) if `true`, just clear current lint errors +---@param opts? vim.treesitter.query.lint.Opts function M.lint(buf, opts) if opts and opts.clear then vim.treesitter._query_linter.clear(buf) -- cgit From 649dd00fe2e54183cc210f24d36504a61e5ea605 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Fri, 8 Mar 2024 11:23:17 +0100 Subject: feat!: remove deprecated functions --- runtime/lua/vim/treesitter/query.lua | 40 ------------------------------------ 1 file changed, 40 deletions(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 6c7f713fd7..85e477be58 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -84,16 +84,6 @@ local function add_included_lang(base_langs, lang, ilang) return false end ----@deprecated -function M.get_query_files(...) - vim.deprecate( - 'vim.treesitter.query.get_query_files()', - 'vim.treesitter.query.get_files()', - '0.10' - ) - return M.get_files(...) -end - --- Gets the list of files used to make up a query --- ---@param lang string Language to get query for @@ -204,12 +194,6 @@ local explicit_queries = setmetatable({}, { end, }) ----@deprecated -function M.set_query(...) - vim.deprecate('vim.treesitter.query.set_query()', 'vim.treesitter.query.set()', '0.10') - M.set(...) -end - --- Sets the runtime query named {query_name} for {lang} --- --- This allows users to override any runtime files and/or configuration @@ -222,12 +206,6 @@ function M.set(lang, query_name, text) explicit_queries[lang][query_name] = M.parse(lang, text) end ----@deprecated -function M.get_query(...) - vim.deprecate('vim.treesitter.query.get_query()', 'vim.treesitter.query.get()', '0.10') - return M.get(...) -end - --- Returns the runtime query {query_name} for {lang}. --- ---@param lang string Language to use for the query @@ -249,12 +227,6 @@ M.get = vim.func._memoize('concat-2', function(lang, query_name) return M.parse(lang, query_string) end) ----@deprecated -function M.parse_query(...) - vim.deprecate('vim.treesitter.query.parse_query()', 'vim.treesitter.query.parse()', '0.10') - return M.parse(...) -end - --- Parse {query} as a string. (If the query is in a file, the caller --- should read the contents into a string before calling). --- @@ -280,18 +252,6 @@ M.parse = vim.func._memoize('concat-2', function(lang, query) return Query.new(lang, ts_query) end) ----@deprecated -function M.get_range(...) - vim.deprecate('vim.treesitter.query.get_range()', 'vim.treesitter.get_range()', '0.10') - return vim.treesitter.get_range(...) -end - ----@deprecated -function M.get_node_text(...) - vim.deprecate('vim.treesitter.query.get_node_text()', 'vim.treesitter.get_node_text()', '0.10') - return vim.treesitter.get_node_text(...) -end - --- Implementations of predicates that can optionally be prefixed with "any-". --- --- These functions contain the implementations for each predicate, correctly -- cgit From ade1b12f49c3b3914c74847d791eb90ea90b56b7 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 8 Mar 2024 12:25:18 +0000 Subject: docs: support inline markdown - Tags are now created with `[tag]()` - References are now created with `[tag]` - Code spans are no longer wrapped --- runtime/lua/vim/treesitter/query.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/query.lua') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 85e477be58..a086f5e876 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -995,7 +995,7 @@ end --- Opens a live editor to query the buffer you started from. --- ---- Can also be shown with *:EditQuery*. +--- Can also be shown with [:EditQuery](). --- --- If you move the cursor to a capture name ("@foo"), text matching the capture is highlighted in --- the source buffer. The query editor is a scratch buffer, use `:write` to save it. You can find -- cgit