From 613068071e02ddf5958fa82974373c370112c5e6 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Tue, 14 Jul 2020 21:50:57 +0200 Subject: treesitter: refactor and use lua regexes --- runtime/lua/vim/treesitter/highlighter.lua | 160 +++++++++++++++++++++++++++++ runtime/lua/vim/treesitter/language.lua | 26 +++++ runtime/lua/vim/treesitter/query.lua | 133 ++++++++++++++++++++++++ 3 files changed, 319 insertions(+) create mode 100644 runtime/lua/vim/treesitter/highlighter.lua create mode 100644 runtime/lua/vim/treesitter/language.lua create mode 100644 runtime/lua/vim/treesitter/query.lua (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua new file mode 100644 index 0000000000..b410f01092 --- /dev/null +++ b/runtime/lua/vim/treesitter/highlighter.lua @@ -0,0 +1,160 @@ +local a = vim.api + +-- support reload for quick experimentation +local TSHighlighter = rawget(vim.treesitter, 'TSHighlighter') or {} +TSHighlighter.__index = TSHighlighter +local ts_hs_ns = a.nvim_create_namespace("treesitter_hl") + +-- These are conventions defined by tree-sitter, though it +-- needs to be user extensible also. +-- TODO(bfredl): this is very much incomplete, we will need to +-- go through a few tree-sitter provided queries and decide +-- on translations that makes the most sense. +TSHighlighter.hl_map = { + ["error"] = "Error", + +-- Miscs + ["comment"] = "Comment", + ["punctuation.delimiter"] = "Delimiter", + ["punctuation.bracket"] = "Delimiter", + ["punctuation.special"] = "Delimiter", + +-- Constants + ["constant"] = "Constant", + ["constant.builtin"] = "Special", + ["constant.macro"] = "Define", + ["string"] = "String", + ["string.regex"] = "String", + ["string.escape"] = "SpecialChar", + ["character"] = "Character", + ["number"] = "Number", + ["boolean"] = "Boolean", + ["float"] = "Float", + +-- Functions + ["function"] = "Function", + ["function.special"] = "Function", + ["function.builtin"] = "Special", + ["function.macro"] = "Macro", + ["parameter"] = "Identifier", + ["method"] = "Function", + ["field"] = "Identifier", + ["property"] = "Identifier", + ["constructor"] = "Special", + +-- Keywords + ["conditional"] = "Conditional", + ["repeat"] = "Repeat", + ["label"] = "Label", + ["operator"] = "Operator", + ["keyword"] = "Keyword", + ["exception"] = "Exception", + + ["type"] = "Type", + ["type.builtin"] = "Type", + ["structure"] = "Structure", + ["include"] = "Include", +} + +function TSHighlighter.new(query, bufnr, ft) + local self = setmetatable({}, TSHighlighter) + self.parser = vim.treesitter.get_parser( + bufnr, + ft, + { + on_changedtree = function(...) self:on_changedtree(...) end, + on_lines = function() self.root = self.parser:parse():root() end + } + ) + + self.buf = self.parser.bufnr + + local tree = self.parser:parse() + self.root = tree:root() + self:set_query(query) + self.edit_count = 0 + self.redraw_count = 0 + self.line_count = {} + a.nvim_buf_set_option(self.buf, "syntax", "") + + -- Tricky: if syntax hasn't been enabled, we need to reload color scheme + -- but use synload.vim rather than syntax.vim to not enable + -- syntax FileType autocmds. Later on we should integrate with the + -- `:syntax` and `set syntax=...` machinery properly. + if vim.g.syntax_on ~= 1 then + vim.api.nvim_command("runtime! syntax/synload.vim") + end + return self +end + +local function is_highlight_name(capture_name) + local firstc = string.sub(capture_name, 1, 1) + return firstc ~= string.lower(firstc) +end + +function TSHighlighter:get_hl_from_capture(capture) + + local name = self.query.captures[capture] + + if is_highlight_name(name) then + -- From "Normal.left" only keep "Normal" + return vim.split(name, '.', true)[1] + else + -- Default to false to avoid recomputing + return TSHighlighter.hl_map[name] + end +end + +function TSHighlighter:set_query(query) + if type(query) == "string" then + query = vim.treesitter.parse_query(self.parser.lang, query) + elseif query == nil then + query = vim.treesitter.get_query(self.parser.lang, 'highlights') + + if query == nil then + a.err_writeln("No highlights.scm query found for " .. self.parser.lang) + + if query == nil then + query = vim.treesitter.parse_query(self.parser.lang, "") + end + end + end + + self.query = query + + self.hl_cache = setmetatable({}, { + __index = function(table, capture) + local hl = self:get_hl_from_capture(capture) + rawset(table, capture, hl) + + return hl + end + }) + + self:on_changedtree({{self.root:range()}}) +end + +function TSHighlighter:on_changedtree(changes) + -- Get a fresh root + self.root = self.parser.tree:root() + + for _, ch in ipairs(changes or {}) do + -- Try to be as exact as possible + local changed_node = self.root:descendant_for_range(ch[1], ch[2], ch[3], ch[4]) + + a.nvim_buf_clear_namespace(self.buf, ts_hs_ns, ch[1], ch[3]) + + for capture, node in self.query:iter_captures(changed_node, self.buf, ch[1], ch[3] + 1) do + local start_row, start_col, end_row, end_col = node:range() + local hl = self.hl_cache[capture] + if hl then + a.nvim__buf_add_decoration(self.buf, ts_hs_ns, hl, + start_row, start_col, + end_row, end_col, + {}) + end + end + end +end + +return TSHighlighter diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua new file mode 100644 index 0000000000..b4817de91e --- /dev/null +++ b/runtime/lua/vim/treesitter/language.lua @@ -0,0 +1,26 @@ +local a = vim.api + +local M = {} + +function M.require_language(lang, path) + if vim._ts_has_language(lang) then + return true + end + if path == nil then + local fname = 'parser/' .. lang .. '.*' + local paths = a.nvim_get_runtime_file(fname, false) + if #paths == 0 then + -- TODO(bfredl): help tag? + error("no parser for '"..lang.."' language") + end + path = paths[1] + end + vim._ts_add_language(path, lang) +end + +function M.inspect_language(lang) + M.require_language(lang) + return vim._ts_inspect_language(lang) +end + +return M diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua new file mode 100644 index 0000000000..914c266426 --- /dev/null +++ b/runtime/lua/vim/treesitter/query.lua @@ -0,0 +1,133 @@ +local a = vim.api +local language = require'vim.treesitter.language' + +-- query: pattern matching on trees +-- predicate matching is implemented in lua +local Query = {} +Query.__index = Query + +local M = {} + +--- Parses a query. +-- +-- @param language The language +-- @param query A string containing the query (s-expr syntax) +-- +-- @returns The query +function M.parse_query(lang, query) + language.require_language(lang) + local self = setmetatable({}, Query) + self.query = vim._ts_parse_query(lang, vim.fn.escape(query,'\\')) + self.info = self.query:inspect() + self.captures = self.info.captures + return self +end + +-- TODO(vigoux): support multiline nodes too +local function get_node_text(node, bufnr) + local start_row, start_col, end_row, end_col = node:range() + if start_row ~= end_row then + return nil + end + local line = a.nvim_buf_get_lines(bufnr, start_row, start_row+1, true)[1] + return string.sub(line, start_col+1, end_col) +end + +-- Predicate handler receive the following arguments +-- (match, pattern, bufnr, regexes, index, predicate) +local predicate_handlers = { + ["eq?"] = function(match, _, bufnr, predicate) + local node = match[predicate[2]] + local node_text = get_node_text(node, bufnr) + + local str + if type(predicate[3]) == "string" then + -- (#eq? @aa "foo") + str = predicate[3] + else + -- (#eq? @aa @bb) + str = get_node_text(match[predicate[3]], bufnr) + end + + if node_text ~= str or str == nil then + return false + end + + return true + end, + ["match?"] = function(match, _, bufnr, predicate) + local node = match[predicate[2]] + local regex = predicate[3] + local start_row, _, end_row, _ = node:range() + if start_row ~= end_row then + return false + end + + return string.find(get_node_text(node, bufnr), regex) + end, +} + +function M.add_predicate(name, handler) + if predicate_handlers[name] then + a.nvim_err_writeln("It is recomended to not overwrite predicates.") + end + + predicate_handlers[name] = handler +end + +function Query:match_preds(match, pattern, bufnr) + local preds = self.info.patterns[pattern] + if not preds then + return true + end + for _, pred in pairs(preds) do + -- Here we only want to return if a predicate DOES NOT match, and + -- continue on the other case. This way unknown predicates will not be considered, + -- which allows some testing and easier user extensibility (#12173). + -- Also, tree-sitter strips the leading # from predicates for us. + if predicate_handlers[pred[1]] and + not predicate_handlers[pred[1]](match, pattern, bufnr, pred) then + return false + end + end + return true +end + +function Query:iter_captures(node, bufnr, start, stop) + if bufnr == 0 then + bufnr = vim.api.nvim_get_current_buf() + end + local raw_iter = node:_rawquery(self.query, true, start, stop) + local function iter() + local capture, captured_node, match = raw_iter() + if match ~= nil then + local active = self:match_preds(match, match.pattern, bufnr) + match.active = active + if not active then + return iter() -- tail call: try next match + end + end + return capture, captured_node + end + return iter +end + +function Query:iter_matches(node, bufnr, start, stop) + if bufnr == 0 then + bufnr = vim.api.nvim_get_current_buf() + end + local raw_iter = node:_rawquery(self.query, false, start, stop) + local function iter() + local pattern, match = raw_iter() + if match ~= nil then + local active = self:match_preds(match, pattern, bufnr) + if not active then + return iter() -- tail call: try next match + end + end + return pattern, match + end + return iter +end + +return M -- cgit From 18c0e775286aa6621d9190db8cf02e65a3f3d61f Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Thu, 16 Jul 2020 17:17:42 +0200 Subject: treesitter(docs): update and refresh docs --- runtime/lua/vim/treesitter/highlighter.lua | 10 ++-------- runtime/lua/vim/treesitter/language.lua | 13 ++++++++++++- runtime/lua/vim/treesitter/query.lua | 23 +++++++++++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua index b410f01092..681d2c6324 100644 --- a/runtime/lua/vim/treesitter/highlighter.lua +++ b/runtime/lua/vim/treesitter/highlighter.lua @@ -7,9 +7,6 @@ local ts_hs_ns = a.nvim_create_namespace("treesitter_hl") -- These are conventions defined by tree-sitter, though it -- needs to be user extensible also. --- TODO(bfredl): this is very much incomplete, we will need to --- go through a few tree-sitter provided queries and decide --- on translations that makes the most sense. TSHighlighter.hl_map = { ["error"] = "Error", @@ -112,11 +109,8 @@ function TSHighlighter:set_query(query) query = vim.treesitter.get_query(self.parser.lang, 'highlights') if query == nil then - a.err_writeln("No highlights.scm query found for " .. self.parser.lang) - - if query == nil then - query = vim.treesitter.parse_query(self.parser.lang, "") - end + a.nvim_err_writeln("No highlights.scm query found for " .. self.parser.lang) + query = vim.treesitter.parse_query(self.parser.lang, "") end end diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index b4817de91e..a7e36a0b89 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -2,6 +2,12 @@ local a = vim.api local M = {} +--- Asserts that the provided language is installed, and optionnaly provide a path for the parser +-- +-- Parsers are searched in the `parser` runtime directory. +-- +-- @param lang The language the parser should parse +-- @param path Optionnal path the parser is located at function M.require_language(lang, path) if vim._ts_has_language(lang) then return true @@ -11,13 +17,18 @@ function M.require_language(lang, path) local paths = a.nvim_get_runtime_file(fname, false) if #paths == 0 then -- TODO(bfredl): help tag? - error("no parser for '"..lang.."' language") + error("no parser for '"..lang.."' language, see :help treesitter-parsers") end path = paths[1] end vim._ts_add_language(path, lang) end +--- Inspects the provided language. +-- +-- Inspecting provides some useful informations on the language like node names, ... +-- +-- @param lang The language. function M.inspect_language(lang) M.require_language(lang) return vim._ts_inspect_language(lang) diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 914c266426..d16e1c3662 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -67,6 +67,11 @@ local predicate_handlers = { end, } +--- Adds a new predicates to be used in queries +-- +-- @param name the name of the predicate, without leading # +-- @param handler the handler function to be used +-- signature will be (match, pattern, bufnr, predicate) function M.add_predicate(name, handler) if predicate_handlers[name] then a.nvim_err_writeln("It is recomended to not overwrite predicates.") @@ -93,6 +98,15 @@ function Query:match_preds(match, pattern, bufnr) return true end +--- Iterates of the captures of self on a given range. +-- +-- @param node The node under witch the search will occur +-- @param buffer The source buffer to search +-- @param start The starting line of the search +-- @param stop The stoping line of the search (end-exclusive) +-- +-- @returns The matching capture id +-- @returns The captured node function Query:iter_captures(node, bufnr, start, stop) if bufnr == 0 then bufnr = vim.api.nvim_get_current_buf() @@ -112,6 +126,15 @@ function Query:iter_captures(node, bufnr, start, stop) return iter end +--- Iterates of the matches of self on a given range. +-- +-- @param node The node under witch the search will occur +-- @param buffer The source buffer to search +-- @param start The starting line of the search +-- @param stop The stoping line of the search (end-exclusive) +-- +-- @returns The matching pattern id +-- @returns The matching match function Query:iter_matches(node, bufnr, start, stop) if bufnr == 0 then bufnr = vim.api.nvim_get_current_buf() -- cgit From 58e37d7df8ab3afc4d77e6ff1248d26a2559399e Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Mon, 10 Aug 2020 09:41:57 +0200 Subject: treesitter: add contains? predicate --- runtime/lua/vim/treesitter/query.lua | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index d16e1c3662..b30bf5fb6b 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -65,6 +65,19 @@ local predicate_handlers = { return string.find(get_node_text(node, bufnr), regex) end, + + ["contains?"] = function(match, _, bufnr, predicate) + local node = match[predicate[2]] + local node_text = get_node_text(node, bufnr) + + for i=3,#predicate do + if string.find(node_text, predicate[i], 1, true) then + return true + end + end + + return false + end } --- Adds a new predicates to be used in queries -- cgit From d7b12e58dfc7303dbc06381a9bedd5c3539d5413 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Mon, 10 Aug 2020 18:25:52 +0200 Subject: treesitter: add and test vim-match? predicate --- runtime/lua/vim/treesitter/query.lua | 46 +++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index b30bf5fb6b..b43c28b0ab 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -24,7 +24,11 @@ function M.parse_query(lang, query) end -- TODO(vigoux): support multiline nodes too -local function get_node_text(node, bufnr) + +--- Gets the text corresponding to a given node +-- @param node the node +-- @param bufnr the buffer from which the node in extracted. +function M.get_node_text(node, bufnr) local start_row, start_col, end_row, end_col = node:range() if start_row ~= end_row then return nil @@ -34,11 +38,11 @@ local function get_node_text(node, bufnr) end -- Predicate handler receive the following arguments --- (match, pattern, bufnr, regexes, index, predicate) +-- (match, pattern, bufnr, predicate) local predicate_handlers = { ["eq?"] = function(match, _, bufnr, predicate) local node = match[predicate[2]] - local node_text = get_node_text(node, bufnr) + local node_text = M.get_node_text(node, bufnr) local str if type(predicate[3]) == "string" then @@ -46,7 +50,7 @@ local predicate_handlers = { str = predicate[3] else -- (#eq? @aa @bb) - str = get_node_text(match[predicate[3]], bufnr) + str = M.get_node_text(match[predicate[3]], bufnr) end if node_text ~= str or str == nil then @@ -63,12 +67,42 @@ local predicate_handlers = { return false end - return string.find(get_node_text(node, bufnr), regex) + return string.find(M.get_node_text(node, bufnr), regex) end, + ["vim-match?"] = (function() + + local magic_prefixes = {['\\v']=true, ['\\m']=true, ['\\M']=true, ['\\V']=true} + local function check_magic(str) + if string.len(str) < 2 or magic_prefixes[string.sub(str,1,2)] then + return str + end + return '\\v'..str + end + + local compiled_vim_regexes = setmetatable({}, { + __index = function(t, pattern) + local res = vim.regex(check_magic(pattern)) + rawset(t, pattern, res) + return res + end + }) + + return function(match, _, bufnr, pred) + local node = match[pred[2]] + local start_row, start_col, end_row, end_col = node:range() + if start_row ~= end_row then + return false + end + + local regex = compiled_vim_regexes[pred[3]] + return regex:match_line(bufnr, start_row, start_col, end_col) + end + end)(), + ["contains?"] = function(match, _, bufnr, predicate) local node = match[predicate[2]] - local node_text = get_node_text(node, bufnr) + local node_text = M.get_node_text(node, bufnr) for i=3,#predicate do if string.find(node_text, predicate[i], 1, true) then -- cgit From 9564803d1ad2c618176a917f974fb47ce1c598a9 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Tue, 11 Aug 2020 23:21:15 +0200 Subject: treesitter: add predicate negation --- runtime/lua/vim/treesitter/query.lua | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index b43c28b0ab..0665a837e9 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -59,6 +59,7 @@ local predicate_handlers = { return true end, + ["match?"] = function(match, _, bufnr, predicate) local node = match[predicate[2]] local regex = predicate[3] @@ -71,7 +72,6 @@ local predicate_handlers = { end, ["vim-match?"] = (function() - local magic_prefixes = {['\\v']=true, ['\\m']=true, ['\\M']=true, ['\\V']=true} local function check_magic(str) if string.len(str) < 2 or magic_prefixes[string.sub(str,1,2)] then @@ -121,7 +121,7 @@ local predicate_handlers = { -- signature will be (match, pattern, bufnr, predicate) function M.add_predicate(name, handler) if predicate_handlers[name] then - a.nvim_err_writeln("It is recomended to not overwrite predicates.") + a.nvim_err_writeln(string.format("Overriding %s", name)) end predicate_handlers[name] = handler @@ -137,7 +137,14 @@ function Query:match_preds(match, pattern, bufnr) -- continue on the other case. This way unknown predicates will not be considered, -- which allows some testing and easier user extensibility (#12173). -- Also, tree-sitter strips the leading # from predicates for us. - if predicate_handlers[pred[1]] and + if string.sub(pred[1], 1, 4) == "not-" then + local pred_name = string.sub(pred[1], 5) + if predicate_handlers[pred_name] and + predicate_handlers[pred_name](match, pattern, bufnr, pred) then + return false + end + + elseif predicate_handlers[pred[1]] and not predicate_handlers[pred[1]](match, pattern, bufnr, pred) then return false end -- cgit From 6a8dcfab4b2bada9c68379ee17235974fa8ad411 Mon Sep 17 00:00:00 2001 From: Thomas Vigouroux Date: Thu, 13 Aug 2020 20:40:40 +0200 Subject: treesitter: allow to force predicate addition --- runtime/lua/vim/treesitter/query.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime/lua/vim/treesitter') diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua index 0665a837e9..17f61b24f1 100644 --- a/runtime/lua/vim/treesitter/query.lua +++ b/runtime/lua/vim/treesitter/query.lua @@ -119,8 +119,8 @@ local predicate_handlers = { -- @param name the name of the predicate, without leading # -- @param handler the handler function to be used -- signature will be (match, pattern, bufnr, predicate) -function M.add_predicate(name, handler) - if predicate_handlers[name] then +function M.add_predicate(name, handler, force) + if predicate_handlers[name] and not force then a.nvim_err_writeln(string.format("Overriding %s", name)) end -- cgit