From 3c1d70f20b5d5bad3bec121e589187d15f325a9b Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Mon, 25 Jul 2022 12:23:04 +0200 Subject: feat(treesitter): allow customizing language symbol name --- runtime/lua/vim/treesitter/language.lua | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index dfb6f5be84..d14b825603 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -6,10 +6,11 @@ local M = {} --- --- Parsers are searched in the `parser` runtime directory. --- ----@param lang The language the parser should parse ----@param path Optional path the parser is located at ----@param silent Don't throw an error if language not found -function M.require_language(lang, path, silent) +---@param lang string The language the parser should parse +---@param path string|nil Optional path the parser is located at +---@param silent boolean|nil Don't throw an error if language not found +---@param symbol_name string|nil Internal symbol name for the language to load +function M.require_language(lang, path, silent, symbol_name) if vim._ts_has_language(lang) then return true end @@ -21,7 +22,6 @@ function M.require_language(lang, path, silent) return false end - -- TODO(bfredl): help tag? error("no parser for '" .. lang .. "' language, see :help treesitter-parsers") end path = paths[1] @@ -29,10 +29,10 @@ function M.require_language(lang, path, silent) if silent then return pcall(function() - vim._ts_add_language(path, lang) + vim._ts_add_language(path, lang, symbol_name) end) else - vim._ts_add_language(path, lang) + vim._ts_add_language(path, lang, symbol_name) end return true -- cgit From 733b2e12b86a34c00aa07e0491762f88582792a5 Mon Sep 17 00:00:00 2001 From: Quentin Rasmont Date: Tue, 26 Apr 2022 22:42:48 +0200 Subject: feat(treesitter): add opts.concat to query.get_text_node As part of the upstream of utility functions from nvim-treesitter, this option when set to false allows to return a table (downstream behavior). Effectively making the switch from the downstream to the upstream function much easier. --- runtime/lua/vim/treesitter/query.lua | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 103e85abfd..697e2e7691 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -181,9 +181,14 @@ end --- Gets the text corresponding to a given node --- ----@param node the node ----@param source The buffer or string from which the node is extracted -function M.get_node_text(node, source) +---@param node table The node +---@param source table The buffer or string from which the node is extracted +---@param opts table Optional parameters. +--- - concat: (boolean default true) Concatenate result in a string +function M.get_node_text(node, source, opts) + opts = opts or {} + local concat = vim.F.if_nil(opts.concat, true) + local start_row, start_col, start_byte = node:start() local end_row, end_col, end_byte = node:end_() @@ -210,7 +215,7 @@ function M.get_node_text(node, source) end end - return table.concat(lines, '\n') + return concat and table.concat(lines, '\n') or lines elseif type(source) == 'string' then return source:sub(start_byte + 1, end_byte) end -- cgit From 6b2d42eb0352d01923e4bf2e3ce0824c662b7be4 Mon Sep 17 00:00:00 2001 From: Quentin Rasmont Date: Sat, 30 Apr 2022 10:43:26 +0200 Subject: feat(treesitter): add ability to retreive a tree/node given a range --- runtime/lua/vim/treesitter/languagetree.lua | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua index 4d3b0631a2..f87a66ddab 100644 --- a/runtime/lua/vim/treesitter/languagetree.lua +++ b/runtime/lua/vim/treesitter/languagetree.lua @@ -549,6 +549,44 @@ function LanguageTree:contains(range) return false end +--- Gets the tree that contains {range} +--- +---@param range table A text range +---@param opts table Options table +---@param opts.ignore_injections boolean (default true) Ignore injected languages. +function LanguageTree:tree_for_range(range, opts) + opts = opts or {} + local ignore = vim.F.if_nil(opts.ignore_injections, true) + + if not ignore then + for _, child in pairs(self._children) do + for _, tree in pairs(child:trees()) do + if tree_contains(tree, range) then + return tree + end + end + end + end + + for _, tree in pairs(self._trees) do + if tree_contains(tree, range) then + return tree + end + end + + return nil +end + +--- Gets the smallest named node that contains {range} +--- +---@param range table A text range +---@param opts table Options table +---@param opts.ignore_injections boolean (default true) Ignore injected languages. +function LanguageTree:named_node_for_range(range, opts) + local tree = self:tree_for_range(range, opts) + return tree:root():named_descendant_for_range(unpack(range)) +end + --- Gets the appropriate language that contains {range} --- ---@param range A text range, see |LanguageTree:contains| -- cgit From 244a115e494bce8e8205c04a6e5f3ab74ec4ed65 Mon Sep 17 00:00:00 2001 From: Quentin Rasmont Date: Sun, 24 Jul 2022 20:49:33 +0200 Subject: feat(treesitter): clarify similar 'get_node_range' functions The private 'get_node_range' function from the languagetree module has been renamed and remains private as it serve a purpose that is only relevant inside the languagetree module. The 'get_node_range' upstreamed from nvim-treesitter in the treesitter module has been made public as it is in itself a utlity function. --- runtime/lua/vim/treesitter/languagetree.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua index f87a66ddab..70317a9f94 100644 --- a/runtime/lua/vim/treesitter/languagetree.lua +++ b/runtime/lua/vim/treesitter/languagetree.lua @@ -299,7 +299,7 @@ function LanguageTree:included_regions() end ---@private -local function get_node_range(node, id, metadata) +local function get_range_from_metadata(node, id, metadata) if metadata[id] and metadata[id].range then return metadata[id].range end @@ -362,7 +362,7 @@ function LanguageTree:_get_injections() elseif name == 'combined' then combined = true elseif name == 'content' and #ranges == 0 then - table.insert(ranges, get_node_range(node, id, metadata)) + table.insert(ranges, get_range_from_metadata(node, id, metadata)) -- Ignore any tags that start with "_" -- Allows for other tags to be used in matches elseif string.sub(name, 1, 1) ~= '_' then @@ -371,7 +371,7 @@ function LanguageTree:_get_injections() end if #ranges == 0 then - table.insert(ranges, get_node_range(node, id, metadata)) + table.insert(ranges, get_range_from_metadata(node, id, metadata)) end end end -- cgit From 030b422d1e9517ed1b1c70fd8002b74881c80650 Mon Sep 17 00:00:00 2001 From: bfredl Date: Wed, 24 Aug 2022 23:48:52 +0200 Subject: feat(treesitter)!: use @foo.bar style highlight groups This removes the support for defining links via vim.treesitter.highlighter.hl_map (never documented, but plugins did anyway), or the uppercase-only `@FooGroup.Bar` to `FooGroup` rule. The fallback is now strictly `@foo.bar.lang` to `@foo.bar` to `@foo`, and casing is irrelevant (as it already was outside of treesitter) For compatibility, define default links to builting syntax groups as defined by pre-existing color schemes --- runtime/lua/vim/treesitter/highlighter.lua | 109 ++--------------------------- 1 file changed, 4 insertions(+), 105 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua index e27a5fa9c3..1f242b0fdd 100644 --- a/runtime/lua/vim/treesitter/highlighter.lua +++ b/runtime/lua/vim/treesitter/highlighter.lua @@ -12,105 +12,18 @@ TSHighlighterQuery.__index = TSHighlighterQuery local ns = a.nvim_create_namespace('treesitter/highlighter') -local _default_highlights = {} -local _link_default_highlight_once = function(from, to) - if not _default_highlights[from] then - _default_highlights[from] = true - a.nvim_set_hl(0, from, { link = to, default = true }) - end - - return from -end - --- If @definition.special does not exist use @definition instead -local subcapture_fallback = { - __index = function(self, capture) - local rtn - local shortened = capture - while not rtn and shortened do - shortened = shortened:match('(.*)%.') - rtn = shortened and rawget(self, shortened) - end - rawset(self, capture, rtn or '__notfound') - return rtn - end, -} - -TSHighlighter.hl_map = setmetatable({ - ['error'] = 'Error', - ['text.underline'] = 'Underlined', - ['todo'] = 'Todo', - ['debug'] = 'Debug', - - -- Miscs - ['comment'] = 'Comment', - ['punctuation.delimiter'] = 'Delimiter', - ['punctuation.bracket'] = 'Delimiter', - ['punctuation.special'] = 'Delimiter', - - -- Constants - ['constant'] = 'Constant', - ['constant.builtin'] = 'Special', - ['constant.macro'] = 'Define', - ['define'] = 'Define', - ['macro'] = 'Macro', - ['string'] = 'String', - ['string.regex'] = 'String', - ['string.escape'] = 'SpecialChar', - ['character'] = 'Character', - ['character.special'] = 'SpecialChar', - ['number'] = 'Number', - ['boolean'] = 'Boolean', - ['float'] = 'Float', - - -- Functions - ['function'] = 'Function', - ['function.special'] = 'Function', - ['function.builtin'] = 'Special', - ['function.macro'] = 'Macro', - ['parameter'] = 'Identifier', - ['method'] = 'Function', - ['field'] = 'Identifier', - ['property'] = 'Identifier', - ['constructor'] = 'Special', - - -- Keywords - ['conditional'] = 'Conditional', - ['repeat'] = 'Repeat', - ['label'] = 'Label', - ['operator'] = 'Operator', - ['keyword'] = 'Keyword', - ['exception'] = 'Exception', - - ['type'] = 'Type', - ['type.builtin'] = 'Type', - ['type.qualifier'] = 'Type', - ['type.definition'] = 'Typedef', - ['storageclass'] = 'StorageClass', - ['structure'] = 'Structure', - ['include'] = 'Include', - ['preproc'] = 'PreProc', -}, subcapture_fallback) - ----@private -local function is_highlight_name(capture_name) - local firstc = string.sub(capture_name, 1, 1) - return firstc ~= string.lower(firstc) -end - ---@private function TSHighlighterQuery.new(lang, query_string) local self = setmetatable({}, { __index = TSHighlighterQuery }) self.hl_cache = setmetatable({}, { __index = function(table, capture) - local hl, is_vim_highlight = self:_get_hl_from_capture(capture) - if not is_vim_highlight then - hl = _link_default_highlight_once(lang .. hl, hl) + local name = self._query.captures[capture] + local id = 0 + if not vim.startswith(name, '_') then + id = a.nvim_get_hl_id_by_name('@' .. name .. '.' .. lang) end - local id = a.nvim_get_hl_id_by_name(hl) - rawset(table, capture, id) return id end, @@ -130,20 +43,6 @@ function TSHighlighterQuery:query() return self._query end ----@private ---- Get the hl from capture. ---- Returns a tuple { highlight_name: string, is_builtin: bool } -function TSHighlighterQuery:_get_hl_from_capture(capture) - local name = self._query.captures[capture] - - if is_highlight_name(name) then - -- From "Normal.left" only keep "Normal" - return vim.split(name, '.', true)[1], true - else - return TSHighlighter.hl_map[name] or 0, false - end -end - --- Creates a new highlighter using @param tree --- ---@param tree The language tree to use for highlighting -- cgit From 75adfefc85bcf0d62d2c0f51a951e6003b595cea Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Mon, 18 Jul 2022 14:21:40 +0200 Subject: feat(extmarks,ts,spell): full support for spelling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added 'spell' option to extmarks: Extmarks with this set will have the region spellchecked. - Added 'noplainbuffer' option to 'spelloptions': This is used to tell Neovim not to spellcheck the buffer. The old behaviour was to spell check the whole buffer unless :syntax was set. - Added spelling support to the treesitter highlighter: @spell captures in highlights.scm are used to define regions which should be spell checked. - Added support for navigating spell errors for extmarks: Works for both ephemeral and static extmarks - Added '_on_spell_nav' callback for decoration providers: Since ephemeral callbacks are only drawn for the visible screen, providers must implement this callback to instruct Neovim which regions in the buffer need can be spell checked. The callback takes a start position and an end position. Note: this callback is subject to change hence the _ prefix. - Added spell captures for built-in support languages Co-authored-by: Lewis Russell Co-authored-by: Björn Linse --- runtime/lua/vim/treesitter/highlighter.lua | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua index 1f242b0fdd..9e95af98db 100644 --- a/runtime/lua/vim/treesitter/highlighter.lua +++ b/runtime/lua/vim/treesitter/highlighter.lua @@ -97,6 +97,7 @@ function TSHighlighter.new(tree, opts) if vim.g.syntax_on ~= 1 then vim.api.nvim_command('runtime! syntax/synload.vim') end + vim.bo[self.bufnr].spelloptions = 'noplainbuffer' self.tree:parse() @@ -156,7 +157,7 @@ function TSHighlighter:get_query(lang) end ---@private -local function on_line_impl(self, buf, line) +local function on_line_impl(self, buf, line, spell) self.tree:for_each_tree(function(tstree, tree) if not tstree then return @@ -193,7 +194,9 @@ local function on_line_impl(self, buf, line) local start_row, start_col, end_row, end_col = node:range() local hl = highlighter_query.hl_cache[capture] - if hl and end_row >= line then + local is_spell = highlighter_query:query().captures[capture] == 'spell' + + if hl and end_row >= line and (not spell or is_spell) then a.nvim_buf_set_extmark(buf, ns, start_row, start_col, { end_line = end_row, end_col = end_col, @@ -201,6 +204,7 @@ local function on_line_impl(self, buf, line) ephemeral = true, priority = tonumber(metadata.priority) or 100, -- Low but leaves room below conceal = metadata.conceal, + spell = is_spell, }) end if start_row > line then @@ -217,7 +221,21 @@ function TSHighlighter._on_line(_, _win, buf, line, _) return end - on_line_impl(self, buf, line) + on_line_impl(self, buf, line, false) +end + +---@private +function TSHighlighter._on_spell_nav(_, _, buf, srow, _, erow, _) + local self = TSHighlighter.active[buf] + if not self then + return + end + + self:reset_highlight_state() + + for row = srow, erow do + on_line_impl(self, buf, row, true) + end end ---@private @@ -244,6 +262,7 @@ a.nvim_set_decoration_provider(ns, { on_buf = TSHighlighter._on_buf, on_win = TSHighlighter._on_win, on_line = TSHighlighter._on_line, + _on_spell_nav = TSHighlighter._on_spell_nav, }) return TSHighlighter -- cgit From 707edfc9e6a1745f268d1a9184183da521dc86b8 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 6 Sep 2022 19:22:05 +0100 Subject: fix(ts): do not clobber spelloptions (#20095) --- runtime/lua/vim/treesitter/highlighter.lua | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua index 9e95af98db..1e625eddb8 100644 --- a/runtime/lua/vim/treesitter/highlighter.lua +++ b/runtime/lua/vim/treesitter/highlighter.lua @@ -86,7 +86,7 @@ function TSHighlighter.new(tree, opts) end end - a.nvim_buf_set_option(self.bufnr, 'syntax', '') + vim.bo[self.bufnr].syntax = '' TSHighlighter.active[self.bufnr] = self @@ -95,9 +95,12 @@ function TSHighlighter.new(tree, opts) -- syntax FileType autocmds. Later on we should integrate with the -- `:syntax` and `set syntax=...` machinery properly. if vim.g.syntax_on ~= 1 then - vim.api.nvim_command('runtime! syntax/synload.vim') + vim.cmd.runtime({ 'syntax/synload.vim', bang = true }) end - vim.bo[self.bufnr].spelloptions = 'noplainbuffer' + + a.nvim_buf_call(self.bufnr, function() + vim.opt_local.spelloptions:append('noplainbuffer') + end) self.tree:parse() -- cgit From fd1595514b747d8b083f78007579d869ccfbe89c Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Wed, 7 Sep 2022 08:39:56 +0200 Subject: Use weak tables in tree-sitter code (#17117) feat(treesitter): use weak tables when possible Also add the defaulttable function to create a table whose values are created when a key is missing. --- runtime/lua/vim/treesitter/query.lua | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 697e2e7691..78042a74bf 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -140,12 +140,9 @@ function M.get_query(lang, query_name) end end -local query_cache = setmetatable({}, { - __index = function(tbl, key) - rawset(tbl, key, {}) - return rawget(tbl, key) - end, -}) +local query_cache = vim.defaulttable(function() + return setmetatable({}, { __mode = 'v' }) +end) --- Parse {query} as a string. (If the query is in a file, the caller --- should read the contents into a string before calling). -- cgit From 0405594399741babd6d935d581eac2584b289f92 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Thu, 8 Sep 2022 09:47:36 +0200 Subject: feat(treesitter)!: do not merge queries by default (#20105) Problem: Treesitter queries for a given language in runtime were merged together, leading to errors if they targeted different parser versions (e.g., bundled viml queries and those shipped by nvim-treesitter). Solution: Runtime queries now work as follows: * The last query in the rtp without `; extends` in the header will be used as the base query * All queries (without a specific order) with `; extends` are concatenated with the base query BREAKING CHANGE: queries need to be updated if they are meant to extend other queries --- runtime/lua/vim/treesitter/query.lua | 37 ++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 78042a74bf..be5b24fd95 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -47,6 +47,9 @@ function M.get_query_files(lang, query_name, is_included) return {} end + local base_query = nil + local extensions = {} + local base_langs = {} -- Now get the base languages by looking at the first line of every file @@ -55,13 +58,26 @@ function M.get_query_files(lang, query_name, is_included) -- -- {language} ::= {lang} | ({lang}) local MODELINE_FORMAT = '^;+%s*inherits%s*:?%s*([a-z_,()]+)%s*$' + local EXTENDS_FORMAT = '^;+%s*extends%s*$' - for _, file in ipairs(lang_files) do - local modeline = safe_read(file, '*l') + for _, filename in ipairs(lang_files) do + local file, err = io.open(filename, 'r') + if not file then + error(err) + end - if modeline then - local langlist = modeline:match(MODELINE_FORMAT) + local extension = false + + for modeline in + function() + return file:read('*l') + end + do + if not vim.startswith(modeline, ';') then + break + end + local langlist = modeline:match(MODELINE_FORMAT) if langlist then for _, incllang in ipairs(vim.split(langlist, ',', true)) do local is_optional = incllang:match('%(.*%)') @@ -74,16 +90,25 @@ function M.get_query_files(lang, query_name, is_included) table.insert(base_langs, incllang) end end + elseif modeline:match(EXTENDS_FORMAT) then + extension = true end end + + if extension then + table.insert(extensions, filename) + else + base_query = filename + end + io.close(file) end - local query_files = {} + local query_files = { base_query } for _, base_lang in ipairs(base_langs) do local base_files = M.get_query_files(base_lang, query_name, true) vim.list_extend(query_files, base_files) end - vim.list_extend(query_files, lang_files) + vim.list_extend(query_files, extensions) return query_files end -- cgit From 893b659e88c61a8c3ce5b140ab475cd67e0ca6bc Mon Sep 17 00:00:00 2001 From: bfredl Date: Thu, 8 Sep 2022 11:17:29 +0200 Subject: fix(treesitter): use the right loading order for base queries (#20117) Use the first, not last, query for a language on runtimepath. Typically, this implies that a user query will override a site plugin query, which will override a bundled runtime query. --- runtime/lua/vim/treesitter/query.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index be5b24fd95..2f6227af8e 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -97,7 +97,7 @@ function M.get_query_files(lang, query_name, is_included) if extension then table.insert(extensions, filename) - else + elseif base_query == nil then base_query = filename end io.close(file) -- cgit From 1939518ebab72878f2a8ca0cb85c09f7e70d1093 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Sat, 10 Sep 2022 11:26:23 +0200 Subject: fix(treesitter): prevent endless loop on self-inheritence Fixes #20139 --- runtime/lua/vim/treesitter/query.lua | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 2f6227af8e..90ed2a357c 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -34,6 +34,18 @@ local function safe_read(filename, read_quantifier) return content end +---@private +--- Adds @p ilang to @p base_langs, only if @p ilang is different than @lang +--- +---@return boolean true it lang == ilang +local function add_included_lang(base_langs, lang, ilang) + if lang == ilang then + return true + end + table.insert(base_langs, ilang) + return false +end + --- Gets the list of files used to make up a query --- ---@param lang The language @@ -84,10 +96,14 @@ function M.get_query_files(lang, query_name, is_included) if is_optional then if not is_included then - table.insert(base_langs, incllang:sub(2, #incllang - 1)) + if add_included_lang(base_langs, lang, incllang:sub(2, #incllang - 1)) then + extension = true + end end else - table.insert(base_langs, incllang) + if add_included_lang(base_langs, lang, incllang) then + extension = true + end end end elseif modeline:match(EXTENDS_FORMAT) then -- cgit