diff options
44 files changed, 5963 insertions, 397 deletions
diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt index c0da06ffe3..1c3a7f70c9 100644 --- a/runtime/doc/lua.txt +++ b/runtime/doc/lua.txt @@ -594,6 +594,102 @@ tsnode:named_descendant_for_range(start_row, start_col, end_row, end_col) Get the smallest named node within this node that spans the given range of (row, column) positions +Query methods *lua-treesitter-query* + +Tree-sitter queries are supported, with some limitations. Currently, the only +supported match predicate is `eq?` (both comparing a capture against a string +and two captures against each other). + +vim.treesitter.parse_query(lang, query) + *vim.treesitter.parse_query(()* + Parse the query as a string. (If the query is in a file, the caller + should read the contents into a string before calling). + +query:iter_captures(node, bufnr, start_row, end_row) + *query:iter_captures()* + Iterate over all captures from all matches inside a `node`. + `bufnr` is needed if the query contains predicates, then the caller + must ensure to use a freshly parsed tree consistent with the current + text of the buffer. `start_row` and `end_row` can be used to limit + matches inside a row range (this is typically used with root node + as the node, i e to get syntax highlight matches in the current + viewport) + + The iterator returns two values, a numeric id identifying the capture + and the captured node. The following example shows how to get captures + by name: +> + for id, node in query:iter_captures(tree:root(), bufnr, first, last) do + local name = query.captures[id] -- name of the capture in the query + -- typically useful info about the node: + local type = node:type() -- type of the captured node + local row1, col1, row2, col2 = node:range() -- range of the capture + ... use the info here ... + end +< +query:iter_matches(node, bufnr, start_row, end_row) + *query:iter_matches()* + Iterate over all matches within a node. The arguments are the same as + for |query:iter_captures()| but the iterated values are different: + an (1-based) index of the pattern in the query, and a table mapping + capture indices to nodes. If the query has more than one pattern + the capture table might be sparse, and e.g. `pairs` should be used and not + `ipairs`. Here an example iterating over all captures in + every match: +> + for pattern, match in cquery:iter_matches(tree:root(), bufnr, first, last) do + for id,node in pairs(match) do + local name = query.captures[id] + -- `node` was captured by the `name` capture in the match + ... use the info here ... + end + end +> +Treesitter syntax highlighting (WIP) *lua-treesitter-highlight* + +NOTE: This is a partially implemented feature, and not usable as a default +solution yet. What is documented here is a temporary interface indented +for those who want to experiment with this feature and contribute to +its development. + +Highlights are defined in the same query format as in the tree-sitter highlight +crate, which some limitations and additions. Set a highlight query for a +buffer with this code: > + + local query = [[ + "for" @keyword + "if" @keyword + "return" @keyword + + (string_literal) @string + (number_literal) @number + (comment) @comment + + (preproc_function_def name: (identifier) @function) + + ; ... more definitions + ]] + + highlighter = vim.treesitter.TSHighlighter.new(query, bufnr, lang) + -- alternatively, to use the current buffer and its filetype: + -- highlighter = vim.treesitter.TSHighlighter.new(query) + + -- Don't recreate the highlighter for the same buffer, instead + -- modify the query like this: + local query2 = [[ ... ]] + highlighter:set_query(query2) + +As mentioned above the supported predicate is currently only `eq?`. `match?` +predicates behave like matching always fails. As an addition a capture which +begin with an upper-case letter like `@WarningMsg` will map directly to this +highlight group, if defined. Also if the predicate begins with upper-case and +contains a dot only the part before the first will be interpreted as the +highlight group. As an example, this warns of a binary expression with two +identical identifiers, highlighting both as |hl-WarningMsg|: > + + ((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right) + (eq? @WarningMsg.left @WarningMsg.right)) + ------------------------------------------------------------------------------ VIM *lua-builtin* diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua index e0202927bb..aa8b8fcdd1 100644 --- a/runtime/lua/vim/treesitter.lua +++ b/runtime/lua/vim/treesitter.lua @@ -12,9 +12,13 @@ function Parser:parse() if self.valid then return self.tree end - self.tree = self._parser:parse_buf(self.bufnr) + local changes + self.tree, changes = self._parser:parse_buf(self.bufnr) self.valid = true - return self.tree + for _, cb in ipairs(self.change_cbs) do + cb(changes) + end + return self.tree, changes end function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_size) @@ -26,17 +30,28 @@ function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_ self.valid = false end -local module = { +local M = { add_language=vim._ts_add_language, inspect_language=vim._ts_inspect_language, + parse_query = vim._ts_parse_query, } -function module.create_parser(bufnr, ft, id) +setmetatable(M, { + __index = function (t, k) + if k == "TSHighlighter" then + t[k] = require'vim.tshighlighter' + return t[k] + end + end + }) + +function M.create_parser(bufnr, ft, id) if bufnr == 0 then bufnr = a.nvim_get_current_buf() end - local self = setmetatable({bufnr=bufnr, valid=false}, Parser) + local self = setmetatable({bufnr=bufnr, lang=ft, valid=false}, Parser) self._parser = vim._create_ts_parser(ft) + self.change_cbs = {} self:parse() -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is -- using it. @@ -55,7 +70,7 @@ function module.create_parser(bufnr, ft, id) return self end -function module.get_parser(bufnr, ft) +function M.get_parser(bufnr, ft, cb) if bufnr == nil or bufnr == 0 then bufnr = a.nvim_get_current_buf() end @@ -65,9 +80,98 @@ function module.get_parser(bufnr, ft) local id = tostring(bufnr)..'_'..ft if parsers[id] == nil then - parsers[id] = module.create_parser(bufnr, ft, id) + parsers[id] = M.create_parser(bufnr, ft, id) + end + if cb ~= nil then + table.insert(parsers[id].change_cbs, cb) end return parsers[id] end -return module +-- query: pattern matching on trees +-- predicate matching is implemented in lua +local Query = {} +Query.__index = Query + +function M.parse_query(lang, query) + local self = setmetatable({}, Query) + self.query = vim._ts_parse_query(lang, query) + self.info = self.query:inspect() + self.captures = self.info.captures + return self +end + +local function get_node_text(node, bufnr) + local start_row, start_col, end_row, end_col = node:range() + if start_row ~= end_row then + return nil + end + local line = a.nvim_buf_get_lines(bufnr, start_row, start_row+1, true)[1] + return string.sub(line, start_col+1, end_col) +end + +local function match_preds(match, preds, bufnr) + for _, pred in pairs(preds) do + if pred[1] == "eq?" then + local node = match[pred[2]] + local node_text = get_node_text(node, bufnr) + + local str + if type(pred[3]) == "string" then + -- (eq? @aa "foo") + str = pred[3] + else + -- (eq? @aa @bb) + str = get_node_text(match[pred[3]], bufnr) + end + + if node_text ~= str or str == nil then + return false + end + else + return false + end + end + return true +end + +function Query:iter_captures(node, bufnr, start, stop) + if bufnr == 0 then + bufnr = vim.api.nvim_get_current_buf() + end + local raw_iter = node:_rawquery(self.query,true,start,stop) + local function iter() + local capture, captured_node, match = raw_iter() + if match ~= nil then + local preds = self.info.patterns[match.pattern] + local active = match_preds(match, preds, bufnr) + match.active = active + if not active then + return iter() -- tail call: try next match + end + end + return capture, captured_node + end + return iter +end + +function Query:iter_matches(node, bufnr, start, stop) + if bufnr == 0 then + bufnr = vim.api.nvim_get_current_buf() + end + local raw_iter = node:_rawquery(self.query,false,start,stop) + local function iter() + local pattern, match = raw_iter() + if match ~= nil then + local preds = self.info.patterns[pattern] + local active = (not preds) or match_preds(match, preds, bufnr) + if not active then + return iter() -- tail call: try next match + end + end + return pattern, match + end + return iter +end + +return M diff --git a/runtime/lua/vim/tshighlighter.lua b/runtime/lua/vim/tshighlighter.lua new file mode 100644 index 0000000000..1544ecbf49 --- /dev/null +++ b/runtime/lua/vim/tshighlighter.lua @@ -0,0 +1,142 @@ +local a = vim.api + +-- support reload for quick experimentation +local TSHighlighter = rawget(vim.treesitter, 'TSHighlighter') or {} +TSHighlighter.__index = TSHighlighter + +-- These are conventions defined by tree-sitter, though it +-- needs to be user extensible also. +-- TODO(bfredl): this is very much incomplete, we will need to +-- go through a few tree-sitter provided queries and decide +-- on translations that makes the most sense. +TSHighlighter.hl_map = { + keyword="Keyword", + string="String", + type="Type", + comment="Comment", + constant="Constant", + operator="Operator", + number="Number", + label="Label", + ["function"]="Function", + ["function.special"]="Function", +} + +function TSHighlighter.new(query, bufnr, ft) + local self = setmetatable({}, TSHighlighter) + self.parser = vim.treesitter.get_parser(bufnr, ft, function(...) self:on_change(...) end) + self.buf = self.parser.bufnr + -- TODO(bfredl): perhaps on_start should be called uncondionally, instead for only on mod? + local tree = self.parser:parse() + self.root = tree:root() + self:set_query(query) + self.edit_count = 0 + self.redraw_count = 0 + self.line_count = {} + a.nvim_buf_set_option(self.buf, "syntax", "") + a.nvim__buf_set_luahl(self.buf, { + on_start=function(...) return self:on_start(...) end, + on_window=function(...) return self:on_window(...) end, + on_line=function(...) return self:on_line(...) end, + }) + + -- Tricky: if syntax hasn't been enabled, we need to reload color scheme + -- but use synload.vim rather than syntax.vim to not enable + -- syntax FileType autocmds. Later on we should integrate with the + -- `:syntax` and `set syntax=...` machinery properly. + if vim.g.syntax_on ~= 1 then + vim.api.nvim_command("runtime! syntax/synload.vim") + end + return self +end + +function TSHighlighter:set_query(query) + if type(query) == "string" then + query = vim.treesitter.parse_query(self.parser.lang, query) + end + self.query = query + + self.id_map = {} + for i, capture in ipairs(self.query.captures) do + local hl = 0 + local firstc = string.sub(capture, 1, 1) + local hl_group = self.hl_map[capture] + if firstc ~= string.lower(firstc) then + hl_group = vim.split(capture, '.', true)[1] + end + if hl_group then + hl = a.nvim_get_hl_id_by_name(hl_group) + end + self.id_map[i] = hl + end +end + +function TSHighlighter:on_change(changes) + for _, ch in ipairs(changes or {}) do + a.nvim__buf_redraw_range(self.buf, ch[1], ch[3]+1) + end + self.edit_count = self.edit_count + 1 +end + +function TSHighlighter:on_start(_, _buf, _tick) + local tree = self.parser:parse() + self.root = tree:root() +end + +function TSHighlighter:on_window(_, _win, _buf, _topline, botline) + self.iter = nil + self.active_nodes = {} + self.nextrow = 0 + self.botline = botline + self.redraw_count = self.redraw_count + 1 +end + +function TSHighlighter:on_line(_, _win, buf, line) + if self.iter == nil then + self.iter = self.query:iter_captures(self.root,buf,line,self.botline) + end + while line >= self.nextrow do + local capture, node, match = self.iter() + local active = true + if capture == nil then + break + end + if match ~= nil then + active = self:run_pred(match) + match.active = active + end + local start_row, start_col, end_row, end_col = node:range() + local hl = self.id_map[capture] + if hl > 0 and active then + if start_row == line and end_row == line then + a.nvim__put_attr(hl, start_col, end_col) + elseif end_row >= line then + -- TODO(bfredl): this is quite messy. Togheter with multiline bufhl we should support + -- luahl generating multiline highlights (and other kinds of annotations) + self.active_nodes[{hl=hl, start_row=start_row, start_col=start_col, end_row=end_row, end_col=end_col}] = true + end + end + if start_row > line then + self.nextrow = start_row + end + end + for node,_ in pairs(self.active_nodes) do + if node.start_row <= line and node.end_row >= line then + local start_col, end_col = node.start_col, node.end_col + if node.start_row < line then + start_col = 0 + end + if node.end_row > line then + end_col = 9000 + end + a.nvim__put_attr(node.hl, start_col, end_col) + end + if node.end_row <= line then + self.active_nodes[node] = nil + end + end + self.line_count[line] = (self.line_count[line] or 0) + 1 + --return tostring(self.line_count[line]) +end + +return TSHighlighter diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c index 8f5718d97e..e6f8f73b9d 100644 --- a/src/nvim/api/buffer.c +++ b/src/nvim/api/buffer.c @@ -169,21 +169,21 @@ Boolean nvim_buf_attach(uint64_t channel_id, goto error; } cb.on_lines = v->data.luaref; - v->data.integer = LUA_NOREF; + v->data.luaref = LUA_NOREF; } else if (is_lua && strequal("on_changedtick", k.data)) { if (v->type != kObjectTypeLuaRef) { api_set_error(err, kErrorTypeValidation, "callback is not a function"); goto error; } cb.on_changedtick = v->data.luaref; - v->data.integer = LUA_NOREF; + v->data.luaref = LUA_NOREF; } else if (is_lua && strequal("on_detach", k.data)) { if (v->type != kObjectTypeLuaRef) { api_set_error(err, kErrorTypeValidation, "callback is not a function"); goto error; } cb.on_detach = v->data.luaref; - v->data.integer = LUA_NOREF; + v->data.luaref = LUA_NOREF; } else if (is_lua && strequal("utf_sizes", k.data)) { if (v->type != kObjectTypeBoolean) { api_set_error(err, kErrorTypeValidation, "utf_sizes must be boolean"); @@ -231,6 +231,90 @@ Boolean nvim_buf_detach(uint64_t channel_id, return true; } +static void buf_clear_luahl(buf_T *buf, bool force) +{ + if (buf->b_luahl || force) { + executor_free_luaref(buf->b_luahl_start); + executor_free_luaref(buf->b_luahl_window); + executor_free_luaref(buf->b_luahl_line); + executor_free_luaref(buf->b_luahl_end); + } + buf->b_luahl_start = LUA_NOREF; + buf->b_luahl_window = LUA_NOREF; + buf->b_luahl_line = LUA_NOREF; + buf->b_luahl_end = LUA_NOREF; +} + +/// Unstabilized interface for defining syntax hl in lua. +/// +/// This is not yet safe for general use, lua callbacks will need to +/// be restricted, like textlock and probably other stuff. +/// +/// The API on_line/nvim__put_attr is quite raw and not intended to be the +/// final shape. Ideally this should operate on chunks larger than a single +/// line to reduce interpreter overhead, and generate annotation objects +/// (bufhl/virttext) on the fly but using the same representation. +void nvim__buf_set_luahl(uint64_t channel_id, Buffer buffer, + DictionaryOf(LuaRef) opts, Error *err) + FUNC_API_LUA_ONLY +{ + buf_T *buf = find_buffer_by_handle(buffer, err); + + if (!buf) { + return; + } + + redraw_buf_later(buf, NOT_VALID); + buf_clear_luahl(buf, false); + + for (size_t i = 0; i < opts.size; i++) { + String k = opts.items[i].key; + Object *v = &opts.items[i].value; + if (strequal("on_start", k.data)) { + if (v->type != kObjectTypeLuaRef) { + api_set_error(err, kErrorTypeValidation, "callback is not a function"); + goto error; + } + buf->b_luahl_start = v->data.luaref; + v->data.luaref = LUA_NOREF; + } else if (strequal("on_window", k.data)) { + if (v->type != kObjectTypeLuaRef) { + api_set_error(err, kErrorTypeValidation, "callback is not a function"); + goto error; + } + buf->b_luahl_window = v->data.luaref; + v->data.luaref = LUA_NOREF; + } else if (strequal("on_line", k.data)) { + if (v->type != kObjectTypeLuaRef) { + api_set_error(err, kErrorTypeValidation, "callback is not a function"); + goto error; + } + buf->b_luahl_line = v->data.luaref; + v->data.luaref = LUA_NOREF; + } else { + api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data); + goto error; + } + } + buf->b_luahl = true; + return; +error: + buf_clear_luahl(buf, true); + buf->b_luahl = false; +} + +void nvim__buf_redraw_range(Buffer buffer, Integer first, Integer last, + Error *err) + FUNC_API_LUA_ONLY +{ + buf_T *buf = find_buffer_by_handle(buffer, err); + if (!buf) { + return; + } + + redraw_buf_range_later(buf, (linenr_T)first+1, (linenr_T)last); +} + /// Sets a buffer line /// /// @deprecated use nvim_buf_set_lines instead. @@ -1112,7 +1196,6 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start, return rv; } limit = v->data.integer; - v->data.integer = LUA_NOREF; } else { api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data); return rv; diff --git a/src/nvim/api/private/helpers.h b/src/nvim/api/private/helpers.h index 8930f252f6..048b937136 100644 --- a/src/nvim/api/private/helpers.h +++ b/src/nvim/api/private/helpers.h @@ -60,6 +60,12 @@ #define ADD(array, item) \ kv_push(array, item) +#define FIXED_TEMP_ARRAY(name, fixsize) \ + Array name = ARRAY_DICT_INIT; \ + Object name##__items[fixsize]; \ + args.size = fixsize; \ + args.items = name##__items; \ + #define STATIC_CSTR_AS_STRING(s) ((String) {.data = s, .size = sizeof(s) - 1}) /// Create a new String instance, putting data in allocated memory diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 19601b6539..2f59edee33 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -189,6 +189,15 @@ Dictionary nvim_get_hl_by_id(Integer hl_id, Boolean rgb, Error *err) return hl_get_attr_by_id(attrcode, rgb, err); } +/// Gets a highlight group by name +/// +/// similar to |hlID()|, but allocates a new ID if not present. +Integer nvim_get_hl_id_by_name(String name) + FUNC_API_SINCE(7) +{ + return syn_check_group((const char_u *)name.data, (int)name.size); +} + /// Sends input-keys to Nvim, subject to various quirks controlled by `mode` /// flags. This is a blocking call, unlike |nvim_input()|. /// @@ -2546,3 +2555,27 @@ Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Error *err) } return ret; } + +/// Set attrs in nvim__buf_set_lua_hl callbacks +/// +/// TODO(bfredl): This is rather pedestrian. The final +/// interface should probably be derived from a reformed +/// bufhl/virttext interface with full support for multi-line +/// ranges etc +void nvim__put_attr(Integer id, Integer c0, Integer c1) + FUNC_API_LUA_ONLY +{ + if (!lua_attr_active) { + return; + } + if (id == 0 || syn_get_final_id((int)id) == 0) { + return; + } + int attr = syn_id2attr((int)id); + c0 = MAX(c0, 0); + c1 = MIN(c1, (Integer)lua_attr_bufsize); + for (Integer c = c0; c < c1; c++) { + lua_attr_buf[c] = (sattr_T)hl_combine_attr(lua_attr_buf[c], (int)attr); + } + return; +} diff --git a/src/nvim/buffer_defs.h b/src/nvim/buffer_defs.h index 700d8b82e6..bc4fb2997d 100644 --- a/src/nvim/buffer_defs.h +++ b/src/nvim/buffer_defs.h @@ -832,6 +832,12 @@ struct file_buffer { // The number for times the current line has been flushed in the memline. int flush_count; + bool b_luahl; + LuaRef b_luahl_start; + LuaRef b_luahl_window; + LuaRef b_luahl_line; + LuaRef b_luahl_end; + int b_diff_failed; // internal diff failed for this buffer }; diff --git a/src/nvim/buffer_updates.c b/src/nvim/buffer_updates.c index 3604578b50..d12527d6ac 100644 --- a/src/nvim/buffer_updates.c +++ b/src/nvim/buffer_updates.c @@ -157,7 +157,7 @@ void buf_updates_unregister_all(buf_T *buf) args.items[0] = BUFFER_OBJ(buf->handle); textlock++; - executor_exec_lua_cb(cb.on_detach, "detach", args, false); + executor_exec_lua_cb(cb.on_detach, "detach", args, false, NULL); textlock--; } free_update_callbacks(cb); @@ -265,7 +265,7 @@ void buf_updates_send_changes(buf_T *buf, args.items[7] = INTEGER_OBJ((Integer)deleted_codeunits); } textlock++; - Object res = executor_exec_lua_cb(cb.on_lines, "lines", args, true); + Object res = executor_exec_lua_cb(cb.on_lines, "lines", args, true, NULL); textlock--; if (res.type == kObjectTypeBoolean && res.data.boolean == true) { @@ -293,10 +293,7 @@ void buf_updates_changedtick(buf_T *buf) BufUpdateCallbacks cb = kv_A(buf->update_callbacks, i); bool keep = true; if (cb.on_changedtick != LUA_NOREF) { - Array args = ARRAY_DICT_INIT; - Object items[2]; - args.size = 2; - args.items = items; + FIXED_TEMP_ARRAY(args, 2); // the first argument is always the buffer handle args.items[0] = BUFFER_OBJ(buf->handle); @@ -306,7 +303,7 @@ void buf_updates_changedtick(buf_T *buf) textlock++; Object res = executor_exec_lua_cb(cb.on_changedtick, "changedtick", - args, true); + args, true, NULL); textlock--; if (res.type == kObjectTypeBoolean && res.data.boolean == true) { diff --git a/src/nvim/func_attr.h b/src/nvim/func_attr.h index 14b8158c7f..38b51b5564 100644 --- a/src/nvim/func_attr.h +++ b/src/nvim/func_attr.h @@ -211,6 +211,8 @@ # define FUNC_API_NOEXPORT /// API function not exposed in VimL/eval. # define FUNC_API_REMOTE_ONLY +/// API function not exposed in VimL/remote. +# define FUNC_API_LUA_ONLY /// API function introduced at the given API level. # define FUNC_API_SINCE(X) /// API function deprecated since the given API level. diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua index 1aa8223da0..de098b7a7b 100644 --- a/src/nvim/generators/c_grammar.lua +++ b/src/nvim/generators/c_grammar.lua @@ -42,6 +42,7 @@ local c_proto = Ct( (fill * Cg((P('FUNC_API_FAST') * Cc(true)), 'fast') ^ -1) * (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) * (fill * Cg((P('FUNC_API_REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) * + (fill * Cg((P('FUNC_API_LUA_ONLY') * Cc(true)), 'lua_only') ^ -1) * (fill * Cg((P('FUNC_API_REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) * (fill * Cg((P('FUNC_API_BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) * (fill * Cg((P('FUNC_API_COMPOSITOR_IMPL') * Cc(true)), 'compositor_impl') ^ -1) * diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index 76dcf849d1..e861cfda35 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -192,7 +192,7 @@ end -- the real API. for i = 1, #functions do local fn = functions[i] - if fn.impl_name == nil then + if fn.impl_name == nil and not fn.lua_only then local args = {} output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)') @@ -310,12 +310,13 @@ void msgpack_rpc_init_method_table(void) for i = 1, #functions do local fn = functions[i] - output:write(' msgpack_rpc_add_method_handler('.. - '(String) {.data = "'..fn.name..'", '.. - |