From c8f861b739b4703b1198dc1f88b09edbeb0d9f2e Mon Sep 17 00:00:00 2001 From: Björn Linse Date: Sat, 15 Jun 2019 12:10:12 +0200 Subject: tree-sitter: rename tree_sitter => treesitter for consistency --- runtime/lua/tree_sitter_demo.lua | 57 ---- runtime/lua/treesitter_demo.lua | 58 ++++ runtime/lua/vim/tree_sitter.lua | 57 ---- runtime/lua/vim/treesitter.lua | 57 ++++ runtime/plugin/ts_test.vim | 4 +- src/nvim/CMakeLists.txt | 2 +- src/nvim/lua/executor.c | 2 +- src/nvim/lua/tree_sitter.c | 524 ------------------------------- src/nvim/lua/tree_sitter.h | 10 - src/nvim/lua/treesitter.c | 524 +++++++++++++++++++++++++++++++ src/nvim/lua/treesitter.h | 10 + src/nvim/lua/vim.lua | 6 +- test/functional/lua/tree_sitter_spec.lua | 67 ---- test/functional/lua/treesitter_spec.lua | 67 ++++ 14 files changed, 723 insertions(+), 722 deletions(-) delete mode 100644 runtime/lua/tree_sitter_demo.lua create mode 100644 runtime/lua/treesitter_demo.lua delete mode 100644 runtime/lua/vim/tree_sitter.lua create mode 100644 runtime/lua/vim/treesitter.lua delete mode 100644 src/nvim/lua/tree_sitter.c delete mode 100644 src/nvim/lua/tree_sitter.h create mode 100644 src/nvim/lua/treesitter.c create mode 100644 src/nvim/lua/treesitter.h delete mode 100644 test/functional/lua/tree_sitter_spec.lua create mode 100644 test/functional/lua/treesitter_spec.lua diff --git a/runtime/lua/tree_sitter_demo.lua b/runtime/lua/tree_sitter_demo.lua deleted file mode 100644 index bbfd69109d..0000000000 --- a/runtime/lua/tree_sitter_demo.lua +++ /dev/null @@ -1,57 +0,0 @@ -local a = vim.api -_G.a = vim.api - -if __treesitter_rt_ns == nil then - __treesitter_rt_ns = a.nvim_create_namespace("treesitter_demp") -end -local my_ns = __treesitter_rt_ns - -function ts_inspect_pos(row,col) - local tree = theparser:parse_tree() - local root = tree:root() - local node = root:descendant_for_point_range(row,col,row,col) - show_node(node) -end - -function show_node(node) - if node == nil then - return - end - a.nvim_buf_clear_highlight(0, my_ns, 0, -1) - shown_node = node - print(node:type()) - local start_row, start_col, end_row, end_col = node:range() - - a.nvim_buf_add_highlight(0, my_ns, "ErrorMsg", start_row, start_col, start_col+1) - - if end_col >= 1 then - end_col = end_col - 1 - end - a.nvim_buf_add_highlight(0, my_ns, "ErrorMsg", end_row, end_col, end_col+1) -end - -function ts_expand_node() - if shown_node == nil then - return - end - parent = shown_node:parent() - show_node(parent) -end - -function ts_cursor() - local row, col = unpack(a.nvim_win_get_cursor(0)) - ts_inspect_pos(row-1, col) -end - -if false then - ctree = theparser.tree - root = ctree:root() - cursor = root:to_cursor() - node = cursor:forward(5000) if true then return node end - print(#root) - c = root:child(50) - print(require'inspect'{c:extent()}) - type(ctree.__tostring) - root:__tostring() - print(_tslua_debug()) -end diff --git a/runtime/lua/treesitter_demo.lua b/runtime/lua/treesitter_demo.lua new file mode 100644 index 0000000000..82c36f94c0 --- /dev/null +++ b/runtime/lua/treesitter_demo.lua @@ -0,0 +1,58 @@ +-- TODO: externalize this +local a = vim.api +_G.a = vim.api + +if __treesitter_rt_ns == nil then + __treesitter_rt_ns = a.nvim_create_namespace("treesitter_demp") +end +local my_ns = __treesitter_rt_ns + +function ts_inspect_pos(row,col) + local tree = theparser:parse_tree() + local root = tree:root() + local node = root:descendant_for_point_range(row,col,row,col) + show_node(node) +end + +function show_node(node) + if node == nil then + return + end + a.nvim_buf_clear_highlight(0, my_ns, 0, -1) + shown_node = node + print(node:type()) + local start_row, start_col, end_row, end_col = node:range() + + a.nvim_buf_add_highlight(0, my_ns, "ErrorMsg", start_row, start_col, start_col+1) + + if end_col >= 1 then + end_col = end_col - 1 + end + a.nvim_buf_add_highlight(0, my_ns, "ErrorMsg", end_row, end_col, end_col+1) +end + +function ts_expand_node() + if shown_node == nil then + return + end + parent = shown_node:parent() + show_node(parent) +end + +function ts_cursor() + local row, col = unpack(a.nvim_win_get_cursor(0)) + ts_inspect_pos(row-1, col) +end + +if false then + ctree = theparser.tree + root = ctree:root() + cursor = root:to_cursor() + node = cursor:forward(5000) if true then return node end + print(#root) + c = root:child(50) + print(require'inspect'{c:extent()}) + type(ctree.__tostring) + root:__tostring() + print(_tslua_debug()) +end diff --git a/runtime/lua/vim/tree_sitter.lua b/runtime/lua/vim/tree_sitter.lua deleted file mode 100644 index 1b5f416b67..0000000000 --- a/runtime/lua/vim/tree_sitter.lua +++ /dev/null @@ -1,57 +0,0 @@ -local a = vim.api - -local Parser = {} -Parser.__index = Parser - -function Parser:parse_tree(force) - if self.valid and not force then - return self.tree - end - self.tree = self._parser:parse_buf(self.bufnr) - self.valid = true - return self.tree -end - -local function change_cb(self, ev, bufnr, tick, start_row, oldstopline, stop_row) - local start_byte = a.nvim_buf_get_offset(bufnr,start_row) - -- a bit messy, should we expose edited but not reparsed tree? - -- are multiple edits safe in general? - local root = self._parser:tree():root() - -- TODO: add proper lookup function! - local inode = root:descendant_for_point_range(oldstopline+9000,0, oldstopline,0) - if inode == nil then - local stop_byte = a.nvim_buf_get_offset(bufnr,stop_row) - self._parser:edit(start_byte,stop_byte,stop_byte,start_row,0,stop_row,0,stop_row,0) - else - local fakeoldstoprow, fakeoldstopcol, fakebyteoldstop = inode:start() - local fake_rows = fakeoldstoprow-oldstopline - local fakestop = stop_row+fake_rows - local fakebytestop = a.nvim_buf_get_offset(bufnr,fakestop)+fakeoldstopcol - self._parser:edit(start_byte,fakebyteoldstop,fakebytestop,start_row,0,fakeoldstoprow,fakeoldstopcol,fakestop,fakeoldstopcol) - end - self.valid = false -end - -local function create_parser(bufnr, ft) - if bufnr == 0 then - bufnr = a.nvim_get_current_buf() - end - if ft == nil then - ft = a.nvim_buf_get_option(bufnr, "filetype") - end - local self = setmetatable({bufnr=bufnr, valid=false}, Parser) - self._parser = vim._create_ts_parser(ft) - self:parse_tree() - local function cb(ev, ...) - -- TODO: use weakref to self, so that the parser is free'd is no plugin is - -- using it. - return change_cb(self, ev, ...) - end - a.nvim_buf_attach(self.bufnr, false, {on_lines=cb}) - return self -end - --- TODO: weak table with reusable parser per buffer. - -return {create_parser=create_parser, add_language=vim._ts_add_language} - diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua new file mode 100644 index 0000000000..1b5f416b67 --- /dev/null +++ b/runtime/lua/vim/treesitter.lua @@ -0,0 +1,57 @@ +local a = vim.api + +local Parser = {} +Parser.__index = Parser + +function Parser:parse_tree(force) + if self.valid and not force then + return self.tree + end + self.tree = self._parser:parse_buf(self.bufnr) + self.valid = true + return self.tree +end + +local function change_cb(self, ev, bufnr, tick, start_row, oldstopline, stop_row) + local start_byte = a.nvim_buf_get_offset(bufnr,start_row) + -- a bit messy, should we expose edited but not reparsed tree? + -- are multiple edits safe in general? + local root = self._parser:tree():root() + -- TODO: add proper lookup function! + local inode = root:descendant_for_point_range(oldstopline+9000,0, oldstopline,0) + if inode == nil then + local stop_byte = a.nvim_buf_get_offset(bufnr,stop_row) + self._parser:edit(start_byte,stop_byte,stop_byte,start_row,0,stop_row,0,stop_row,0) + else + local fakeoldstoprow, fakeoldstopcol, fakebyteoldstop = inode:start() + local fake_rows = fakeoldstoprow-oldstopline + local fakestop = stop_row+fake_rows + local fakebytestop = a.nvim_buf_get_offset(bufnr,fakestop)+fakeoldstopcol + self._parser:edit(start_byte,fakebyteoldstop,fakebytestop,start_row,0,fakeoldstoprow,fakeoldstopcol,fakestop,fakeoldstopcol) + end + self.valid = false +end + +local function create_parser(bufnr, ft) + if bufnr == 0 then + bufnr = a.nvim_get_current_buf() + end + if ft == nil then + ft = a.nvim_buf_get_option(bufnr, "filetype") + end + local self = setmetatable({bufnr=bufnr, valid=false}, Parser) + self._parser = vim._create_ts_parser(ft) + self:parse_tree() + local function cb(ev, ...) + -- TODO: use weakref to self, so that the parser is free'd is no plugin is + -- using it. + return change_cb(self, ev, ...) + end + a.nvim_buf_attach(self.bufnr, false, {on_lines=cb}) + return self +end + +-- TODO: weak table with reusable parser per buffer. + +return {create_parser=create_parser, add_language=vim._ts_add_language} + diff --git a/runtime/plugin/ts_test.vim b/runtime/plugin/ts_test.vim index 9420c2c9d3..15192d8dda 100644 --- a/runtime/plugin/ts_test.vim +++ b/runtime/plugin/ts_test.vim @@ -6,8 +6,8 @@ func! TSTest() return end " TODO: module! - lua theparser = vim.tree_sitter.create_parser(0) - lua require'tree_sitter_demo' + lua theparser = vim.treesitter.create_parser(0) + lua require'treesitter_demo' let g:has_ts = v:true endfunc diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index 3056c108b4..27977e3a40 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -145,7 +145,7 @@ set(CONV_SOURCES ex_cmds.c ex_docmd.c fileio.c - lua/tree_sitter.c + lua/treesitter.c mbyte.c memline.c message.c diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index 23c4fcabbc..8a35f11c71 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -31,7 +31,7 @@ #include "nvim/lua/executor.h" #include "nvim/lua/converter.h" -#include "nvim/lua/tree_sitter.h" +#include "nvim/lua/treesitter.h" #include "luv/luv.h" diff --git a/src/nvim/lua/tree_sitter.c b/src/nvim/lua/tree_sitter.c deleted file mode 100644 index f992639955..0000000000 --- a/src/nvim/lua/tree_sitter.c +++ /dev/null @@ -1,524 +0,0 @@ -// This is an open source non-commercial project. Dear PVS-Studio, please check -// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com - -// lua bindings for tree-siter. -// NB: this file should contain a generic lua interface for -// tree-sitter trees and nodes, and could be broken out as a reusable library - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "tree_sitter/api.h" - -// NOT state-safe, delete when GC is confimed working: -static int debug_n_trees = 0, debug_n_cursors = 0; - -#define REG_KEY "tree_sitter-private" - -#include "nvim/lua/tree_sitter.h" -#include "nvim/api/private/handle.h" -#include "nvim/memline.h" - -typedef struct { - TSParser *parser; - TSTree *tree; -} Tslua_parser; - -#ifdef INCLUDE_GENERATED_DECLARATIONS -# include "lua/tree_sitter.c.generated.h" -#endif - -static struct luaL_Reg parser_meta[] = { - {"__gc", parser_gc}, - {"__tostring", parser_tostring}, - {"parse_buf", parser_parse_buf}, - {"edit", parser_edit}, - {"tree", parser_tree}, - {NULL, NULL} -}; - -static struct luaL_Reg tree_meta[] = { - {"__gc", tree_gc}, - {"__tostring", tree_tostring}, - {"root", tree_root}, - {NULL, NULL} -}; - -static struct luaL_Reg node_meta[] = { - {"__tostring", node_tostring}, - {"__len", node_child_count}, - {"range", node_range}, - {"start", node_start}, - {"type", node_type}, - {"symbol", node_symbol}, - {"child_count", node_child_count}, - {"child", node_child}, - {"descendant_for_point_range", node_descendant_for_point_range}, - {"parent", node_parent}, - {NULL, NULL} -}; - -PMap(cstr_t) *langs; - -void build_meta(lua_State *L, const luaL_Reg *meta) -{ - // [env, target] - for (size_t i = 0; meta[i].name != NULL; i++) { - lua_pushcfunction(L, meta[i].func); // [env, target, func] - lua_pushvalue(L, -3); // [env, target, func, env] - lua_setfenv(L, -2); // [env, target, func] - lua_setfield(L, -2, meta[i].name); // [env, target] - } - - lua_pushvalue(L, -1); // [env, target, target] - lua_setfield(L, -2, "__index"); // [env, target] -} - - - -/// init the tslua library -/// -/// all global state is stored in the regirstry of the lua_State -void tslua_init(lua_State *L) -{ - - langs = pmap_new(cstr_t)(); - - lua_createtable(L, 0, 0); - - // type metatables - lua_createtable(L, 0, 0); - build_meta(L, parser_meta); - lua_setfield(L, -2, "parser-meta"); - - lua_createtable(L, 0, 0); - build_meta(L, tree_meta); - lua_setfield(L, -2, "tree-meta"); - - lua_createtable(L, 0, 0); - build_meta(L, node_meta); - lua_setfield(L, -2, "node-meta"); - - lua_setfield(L, LUA_REGISTRYINDEX, REG_KEY); - - lua_pushcfunction(L, tslua_debug); - lua_setglobal(L, "_tslua_debug"); -} - -static int tslua_debug(lua_State *L) -{ - lua_pushinteger(L, debug_n_trees); - lua_pushinteger(L, debug_n_cursors); - return 2; -} - - -int ts_lua_register_lang(lua_State *L) -{ - if (lua_gettop(L) < 2 || !lua_isstring(L, 1) || !lua_isstring(L, 2)) { - return luaL_error(L, "string expected"); - } - - const char *path = lua_tostring(L,1); - const char *lang_name = lua_tostring(L,2); - - if (pmap_has(cstr_t)(langs, lang_name)) { - return 0; - } - - // TODO: unsafe! - char symbol_buf[128] = "tree_sitter_"; - STRCAT(symbol_buf, lang_name); - - // TODO: we should maybe keep the uv_lib_t around, and close them - // at exit, to keep LeakSanitizer happy. - uv_lib_t lib; - if (uv_dlopen(path, &lib)) { - return luaL_error(L, "Failed to load parser: uv_dlopen: %s", uv_dlerror(&lib)); - } - - TSLanguage *(*lang_parser)(void); - if (uv_dlsym(&lib, symbol_buf, (void **)&lang_parser)) { - return luaL_error(L, "Failed to load parser: uv_dlsym: %s", uv_dlerror(&lib)); - } - - TSLanguage *lang = lang_parser(); - if (lang == NULL) { - return luaL_error(L, "Failed to load parser: internal error"); - } - - pmap_put(cstr_t)(langs, xstrdup(lang_name), lang); - - lua_pushboolean(L, true); - return 1; -} - -int tslua_push_parser(lua_State *L, const char *lang_name) -{ - TSParser *parser = ts_parser_new(); - TSLanguage *lang = pmap_get(cstr_t)(langs, lang_name); - if (!lang) { - return luaL_error(L, "no such language: %s", lang_name); - } - - ts_parser_set_language(parser, lang); - Tslua_parser *p = lua_newuserdata(L, sizeof(Tslua_parser)); // [udata] - p->parser = parser; - p->tree = NULL; - - lua_getfield(L, LUA_REGISTRYINDEX, REG_KEY); // [udata, env] - lua_getfield(L, -1, "parser-meta"); // [udata, env, meta] - lua_setmetatable(L, -3); // [udata, env] - lua_pop(L, 1); // [udata] - return 1; -} - -static Tslua_parser *parser_check(lua_State *L) -{ - if (!lua_gettop(L)) { - return 0; - } - if (!lua_isuserdata(L, 1)) { - return 0; - } - // TODO: typecheck! - return lua_touserdata(L, 1); -} - -static int parser_gc(lua_State *L) -{ - Tslua_parser *p = parser_check(L); - if (!p) { - return 0; - } - - ts_parser_delete(p->parser); - if (p->tree) { - ts_tree_delete(p->tree); - } - - return 0; -} - -static int parser_tostring(lua_State *L) -{ - lua_pushstring(L, ""); - return 1; -} - -static const char *input_cb(void *payload, uint32_t byte_index, TSPoint position, uint32_t *bytes_read) -{ - buf_T *bp = payload; - static char buf[200]; - if ((linenr_T)position.row >= bp->b_ml.ml_line_count) { - *bytes_read = 0; - return ""; - } - char_u *line = ml_get_buf(bp, position.row+1, false); - size_t len = STRLEN(line); - size_t tocopy = MIN(len-position.column,200); - - // TODO: translate embedded \n to \000 - memcpy(buf, line+position.column, tocopy); - *bytes_read = (uint32_t)tocopy; - if (tocopy < 200) { - buf[tocopy] = '\n'; - (*bytes_read)++; - } - return buf; -} - -static int parser_parse_buf(lua_State *L) -{ - Tslua_parser *p = parser_check(L); - if (!p) { - return 0; - } - - long bufnr = lua_tointeger(L, 2); - void *payload = handle_get_buffer(bufnr); - TSInput input = {payload, input_cb, TSInputEncodingUTF8}; - TSTree *new_tree = ts_parser_parse(p->parser, p->tree, input); - if (p->tree) { - ts_tree_delete(p->tree); - } - p->tree = new_tree; - - tslua_push_tree(L, ts_tree_copy(p->tree)); - return 1; -} - -static int parser_tree(lua_State *L) -{ - Tslua_parser *p = parser_check(L); - if (!p) { - return 0; - } - - if (p->tree) { - tslua_push_tree(L, ts_tree_copy(p->tree)); - } else { - lua_pushnil(L); - } - return 1; -} - -static int parser_edit(lua_State *L) -{ - if(lua_gettop(L) < 10) { - lua_pushstring(L, "not enough args to parser:edit()"); - lua_error(L); - return 0; // unreachable - } - - Tslua_parser *p = parser_check(L); - if (!p) { - return 0; - } - - if (!p->tree) { - return 0; - } - - long start_byte = lua_tointeger(L, 2); - long old_end_byte = lua_tointeger(L, 3); - long new_end_byte = lua_tointeger(L, 4); - TSPoint start_point = { lua_tointeger(L, 5), lua_tointeger(L, 6) }; - TSPoint old_end_point = { lua_tointeger(L, 7), lua_tointeger(L, 8) }; - TSPoint new_end_point = { lua_tointeger(L, 9), lua_tointeger(L, 10) }; - - TSInputEdit edit = { start_byte, old_end_byte, new_end_byte, - start_point, old_end_point, new_end_point }; - - ts_tree_edit(p->tree, &edit); - - return 0; -} - - -// Tree methods - -/// push tree interface on lua stack. -/// -/// This takes "ownership" of the tree and will free it -/// when the wrapper object is garbage collected -void tslua_push_tree(lua_State *L, TSTree *tree) -{ - TSTree **ud = lua_newuserdata(L, sizeof(TSTree *)); // [udata] - *ud = tree; - lua_getfield(L, LUA_REGISTRYINDEX, REG_KEY); // [udata, env] - lua_getfield(L, -1, "tree-meta"); // [udata, env, meta] - lua_setmetatable(L, -3); // [udata, env] - lua_pop(L, 1); // [udata] - - // table used for node wrappers to keep a reference to tree wrapper - // NB: in lua 5.3 the uservalue for the node could just be the tree, but - // in lua 5.1 the uservalue (fenv) must be a table. - lua_createtable(L, 1, 0); // [udata, reftable] - lua_pushvalue(L, -2); // [udata, reftable, udata] - lua_rawseti(L, -2, 1); // [udata, reftable] - lua_setfenv(L, -2); // [udata] - debug_n_trees++; -} - -static TSTree *tree_check(lua_State *L) -{ - if (!lua_gettop(L)) { - return 0; - } - if (!lua_isuserdata(L, 1)) { - return 0; - } - // TODO: typecheck! - TSTree **ud = lua_touserdata(L, 1); - return *ud; -} - -static int tree_gc(lua_State *L) -{ - TSTree *tree = tree_check(L); - if (!tree) { - return 0; - } - - ts_tree_delete(tree); - debug_n_trees--; - return 0; -} - -static int tree_tostring(lua_State *L) -{ - lua_pushstring(L, ""); - return 1; -} - -static int tree_root(lua_State *L) -{ - TSTree *tree = tree_check(L); - if (!tree) { - return 0; - } - TSNode root = ts_tree_root_node(tree); - push_node(L, root); - return 1; -} - -// Node methods - -/// push node interface on lua stack -/// -/// top of stack must either be the tree this node belongs to or another node -/// of the same tree! This value is not popped. Can only be called inside a -/// cfunction with the tslua environment. -static void push_node(lua_State *L, TSNode node) -{ - if (ts_node_is_null(node)) { - lua_pushnil(L); // [src, nil] - return; - } - TSNode *ud = lua_newuserdata(L, sizeof(TSNode)); // [src, udata] - *ud = node; - lua_getfield(L, LUA_ENVIRONINDEX, "node-meta"); // [src, udata, meta] - lua_setmetatable(L, -2); // [src, udata] - lua_getfenv(L, -2); // [src, udata, reftable] - lua_setfenv(L, -2); // [src, udata] -} - -static bool node_check(lua_State *L, TSNode *res) -{ - if (!lua_gettop(L)) { - return 0; - } - if (!lua_isuserdata(L, 1)) { - return 0; - } - // TODO: typecheck! - TSNode *ud = lua_touserdata(L, 1); - *res = *ud; - return true; -} - - -static int node_tostring(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - lua_pushstring(L, ""); - lua_concat(L, 3); - return 1; -} - -static int node_range(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - TSPoint start = ts_node_start_point(node); - TSPoint end = ts_node_end_point(node); - lua_pushnumber(L, start.row); - lua_pushnumber(L, start.column); - lua_pushnumber(L, end.row); - lua_pushnumber(L, end.column); - return 4; -} - -static int node_start(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - TSPoint start = ts_node_start_point(node); - uint32_t start_byte = ts_node_start_byte(node); - lua_pushnumber(L, start.row); - lua_pushnumber(L, start.column); - lua_pushnumber(L, start_byte); - return 3; -} - -static int node_child_count(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - uint32_t count = ts_node_child_count(node); - lua_pushnumber(L, count); - return 1; -} - -static int node_type(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - lua_pushstring(L, ts_node_type(node)); - return 1; -} - -static int node_symbol(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - TSSymbol symbol = ts_node_symbol(node); - lua_pushnumber(L, symbol); - return 1; -} - -static int node_child(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - long num = lua_tointeger(L, 2); - TSNode child = ts_node_child(node, (uint32_t)num); - - lua_pushvalue(L, 1); - push_node(L, child); - return 1; -} - -static int node_descendant_for_point_range(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - TSPoint start = {(uint32_t)lua_tointeger(L, 2), - (uint32_t)lua_tointeger(L, 3)}; - TSPoint end = {(uint32_t)lua_tointeger(L, 4), - (uint32_t)lua_tointeger(L, 5)}; - TSNode child = ts_node_descendant_for_point_range(node, start, end); - - lua_pushvalue(L, 1); - push_node(L, child); - return 1; -} - -static int node_parent(lua_State *L) -{ - TSNode node; - if (!node_check(L, &node)) { - return 0; - } - TSNode parent = ts_node_parent(node); - push_node(L, parent); - return 1; -} - diff --git a/src/nvim/lua/tree_sitter.h b/src/nvim/lua/tree_sitter.h deleted file mode 100644 index 2ae0ec8371..0000000000 --- a/src/nvim/lua/tree_sitter.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef NVIM_LUA_TREE_SITTER_H -#define NVIM_LUA_TREE_SITTER_H - -#include "tree_sitter/api.h" - -#ifdef INCLUDE_GENERATED_DECLARATIONS -# include "lua/tree_sitter.h.generated.h" -#endif - -#endif // NVIM_LUA_TREE_SITTER_H diff --git a/src/nvim/lua/treesitter.c b/src/nvim/lua/treesitter.c new file mode 100644 index 0000000000..794bdc6749 --- /dev/null +++ b/src/nvim/lua/treesitter.c @@ -0,0 +1,524 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +// lua bindings for tree-siter. +// NB: this file should contain a generic lua interface for +// tree-sitter trees and nodes, and could be broken out as a reusable library + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "tree_sitter/api.h" + +// NOT state-safe, delete when GC is confimed working: +static int debug_n_trees = 0, debug_n_cursors = 0; + +#define REG_KEY "treesitter-private" + +#include "nvim/lua/treesitter.h" +#include "nvim/api/private/handle.h" +#include "nvim/memline.h" + +typedef struct { + TSParser *parser; + TSTree *tree; +} Tslua_parser; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "lua/treesitter.c.generated.h" +#endif + +static struct luaL_Reg parser_meta[] = { + {"__gc", parser_gc}, + {"__tostring", parser_tostring}, + {"parse_buf", parser_parse_buf}, + {"edit", parser_edit}, + {"tree", parser_tree}, + {NULL, NULL} +}; + +static struct luaL_Reg tree_meta[] = { + {"__gc", tree_gc}, + {"__tostring", tree_tostring}, + {"root", tree_root}, + {NULL, NULL} +}; + +static struct luaL_Reg node_meta[] = { + {"__tostring", node_tostring}, + {"__len", node_child_count}, + {"range", node_range}, + {"start", node_start}, + {"type", node_type}, + {"symbol", node_symbol}, + {"child_count", node_child_count}, + {"child", node_child}, + {"descendant_for_point_range", node_descendant_for_point_range}, + {"parent", node_parent}, + {NULL, NULL} +}; + +PMap(cstr_t) *langs; + +void build_meta(lua_State *L, const luaL_Reg *meta) +{ + // [env, target] + for (size_t i = 0; meta[i].name != NULL; i++) { + lua_pushcfunction(L, meta[i].func); // [env, target, func] + lua_pushvalue(L, -3); // [env, target, func, env] + lua_setfenv(L, -2); // [env, target, func] + lua_setfield(L, -2, meta[i].name); // [env, target] + } + + lua_pushvalue(L, -1); // [env, target, target] + lua_setfield(L, -2, "__index"); // [env, target] +} + + + +/// init the tslua library +/// +/// all global state is stored in the regirstry of the lua_State +void tslua_init(lua_State *L) +{ + + langs = pmap_new(cstr_t)(); + + lua_createtable(L, 0, 0); + + // type metatables + lua_createtable(L, 0, 0); + build_meta(L, parser_meta); + lua_setfield(L, -2, "parser-meta"); + + lua_createtable(L, 0, 0); + build_meta(L, tree_meta); + lua_setfield(L, -2, "tree-meta"); + + lua_createtable(L, 0, 0); + build_meta(L, node_meta); + lua_setfield(L, -2, "node-meta"); + + lua_setfield(L, LUA_REGISTRYINDEX, REG_KEY); + + lua_pushcfunction(L, tslua_debug); + lua_setglobal(L, "_tslua_debug"); +} + +static int tslua_debug(lua_State *L) +{ + lua_pushinteger(L, debug_n_trees); + lua_pushinteger(L, debug_n_cursors); + return 2; +} + + +int ts_lua_register_lang(lua_State *L) +{ + if (lua_gettop(L) < 2 || !lua_isstring(L, 1) || !lua_isstring(L, 2)) { + return luaL_error(L, "string expected"); + } + + const char *path = lua_tostring(L,1); + const char *lang_name = lua_tostring(L,2); + + if (pmap_has(cstr_t)(langs, lang_name)) { + return 0; + } + + // TODO: unsafe! + char symbol_buf[128] = "tree_sitter_"; + STRCAT(symbol_buf, lang_name); + + // TODO: we should maybe keep the uv_lib_t around, and close them + // at exit, to keep LeakSanitizer happy. + uv_lib_t lib; + if (uv_dlopen(path, &lib)) { + return luaL_error(L, "Failed to load parser: uv_dlopen: %s", uv_dlerror(&lib)); + } + + TSLanguage *(*lang_parser)(void); + if (uv_dlsym(&lib, symbol_buf, (void **)&lang_parser)) { + return luaL_error(L, "Failed to load parser: uv_dlsym: %s", uv_dlerror(&lib)); + } + + TSLanguage *lang = lang_parser(); + if (lang == NULL) { + return luaL_error(L, "Failed to load parser: internal error"); + } + + pmap_put(cstr_t)(langs, xstrdup(lang_name), lang); + + lua_pushboolean(L, true); + return 1; +} + +int tslua_push_parser(lua_State *L, const char *lang_name) +{ + TSParser *parser = ts_parser_new(); + TSLanguage *lang = pmap_get(cstr_t)(langs, lang_name); + if (!lang) { + return luaL_error(L, "no such language: %s", lang_name); + } + + ts_parser_set_language(parser, lang); + Tslua_parser *p = lua_newuserdata(L, sizeof(Tslua_parser)); // [udata] + p->parser = parser; + p->tree = NULL; + + lua_getfield(L, LUA_REGISTRYINDEX, REG_KEY); // [udata, env] + lua_getfield(L, -1, "parser-meta"); // [udata, env, meta] + lua_setmetatable(L, -3); // [udata, env] + lua_pop(L, 1); // [udata] + return 1; +} + +static Tslua_parser *parser_check(lua_State *L) +{ + if (!lua_gettop(L)) { + return 0; + } + if (!lua_isuserdata(L, 1)) { + return 0; + } + // TODO: typecheck! + return lua_touserdata(L, 1); +} + +static int parser_gc(lua_State *L) +{ + Tslua_parser *p = parser_check(L); + if (!p) { + return 0; + } + + ts_parser_delete(p->parser); + if (p->tree) { + ts_tree_delete(p->tree); + } + + return 0; +} + +static int parser_tostring(lua_State *L) +{ + lua_pushstring(L, ""); + return 1; +} + +static const char *input_cb(void *payload, uint32_t byte_index, TSPoint position, uint32_t *bytes_read) +{ + buf_T *bp = payload; + static char buf[200]; + if ((linenr_T)position.row >= bp->b_ml.ml_line_count) { + *bytes_read = 0; + return ""; + } + char_u *line = ml_get_buf(bp, position.row+1, false); + size_t len = STRLEN(line); + size_t tocopy = MIN(len-position.column,200); + + // TODO: translate embedded \n to \000 + memcpy(buf, line+position.column, tocopy); + *bytes_read = (uint32_t)tocopy; + if (tocopy < 200) { + buf[tocopy] = '\n'; + (*bytes_read)++; + } + return buf; +} + +static int parser_parse_buf(lua_State *L) +{ + Tslua_parser *p = parser_check(L); + if (!p) { + return 0; + } + + long bufnr = lua_tointeger(L, 2); + void *payload = handle_get_buffer(bufnr); + TSInput input = {payload, input_cb, TSInputEncodingUTF8}; + TSTree *new_tree = ts_parser_parse(p->parser, p->tree, input); + if (p->tree) { + ts_tree_delete(p->tree); + } + p->tree = new_tree; + + tslua_push_tree(L, ts_tree_copy(p->tree)); + return 1; +} + +static int parser_tree(lua_State *L) +{ + Tslua_parser *p = parser_check(L); + if (!p) { + return 0; + } + + if (p->tree) { + tslua_push_tree(L, ts_tree_copy(p->tree)); + } else { + lua_pushnil(L); + } + return 1; +} + +static int parser_edit(lua_State *L) +{ + if(lua_gettop(L) < 10) { + lua_pushstring(L, "not enough args to parser:edit()"); + lua_error(L); + return 0; // unreachable + } + + Tslua_parser *p = parser_check(L); + if (!p) { + return 0; + } + + if (!p->tree) { + return 0; + } + + long start_byte = lua_tointeger(L, 2); + long old_end_byte = lua_tointeger(L, 3); + long new_end_byte = lua_tointeger(L, 4); + TSPoint start_point = { lua_tointeger(L, 5), lua_tointeger(L, 6) }; + TSPoint old_end_point = { lua_tointeger(L, 7), lua_tointeger(L, 8) }; + TSPoint new_end_point = { lua_tointeger(L, 9), lua_tointeger(L, 10) }; + + TSInputEdit edit = { start_byte, old_end_byte, new_end_byte, + start_point, old_end_point, new_end_point }; + + ts_tree_edit(p->tree, &edit); + + return 0; +} + + +// Tree methods + +/// push tree interface on lua stack. +/// +/// This takes "ownership" of the tree and will free it +/// when the wrapper object is garbage collected +void tslua_push_tree(lua_State *L, TSTree *tree) +{ + TSTree **ud = lua_newuserdata(L, sizeof(TSTree *)); // [udata] + *ud = tree; + lua_getfield(L, LUA_REGISTRYINDEX, REG_KEY); // [udata, env] + lua_getfield(L, -1, "tree-meta"); // [udata, env, meta] + lua_setmetatable(L, -3); // [udata, env] + lua_pop(L, 1); // [udata] + + // table used for node wrappers to keep a reference to tree wrapper + // NB: in lua 5.3 the uservalue for the node could just be the tree, but + // in lua 5.1 the uservalue (fenv) must be a table. + lua_createtable(L, 1, 0); // [udata, reftable] + lua_pushvalue(L, -2); // [udata, reftable, udata] + lua_rawseti(L, -2, 1); // [udata, reftable] + lua_setfenv(L, -2); // [udata] + debug_n_trees++; +} + +static TSTree *tree_check(lua_State *L) +{ + if (!lua_gettop(L)) { + return 0; + } + if (!lua_isuserdata(L, 1)) { + return 0; + } + // TODO: typecheck! + TSTree **ud = lua_touserdata(L, 1); + return *ud; +} + +static int tree_gc(lua_State *L) +{ + TSTree *tree = tree_check(L); + if (!tree) { + return 0; + } + + ts_tree_delete(tree); + debug_n_trees--; + return 0; +} + +static int tree_tostring(lua_State *L) +{ + lua_pushstring(L, ""); + return 1; +} + +static int tree_root(lua_State *L) +{ + TSTree *tree = tree_check(L); + if (!tree) { + return 0; + } + TSNode root = ts_tree_root_node(tree); + push_node(L, root); + return 1; +} + +// Node methods + +/// push node interface on lua stack +/// +/// top of stack must either be the tree this node belongs to or another node +/// of the same tree! This value is not popped. Can only be called inside a +/// cfunction with the tslua environment. +static void push_node(lua_State *L, TSNode node) +{ + if (ts_node_is_null(node)) { + lua_pushnil(L); // [src, nil] + return; + } + TSNode *ud = lua_newuserdata(L, sizeof(TSNode)); // [src, udata] + *ud = node; + lua_getfield(L, LUA_ENVIRONINDEX, "node-meta"); // [src, udata, meta] + lua_setmetatable(L, -2); // [src, udata] + lua_getfenv(L, -2); // [src, udata, reftable] + lua_setfenv(L, -2); // [src, udata] +} + +static bool node_check(lua_State *L, TSNode *res) +{ + if (!lua_gettop(L)) { + return 0; + } + if (!lua_isuserdata(L, 1)) { + return 0; + } + // TODO: typecheck! + TSNode *ud = lua_touserdata(L, 1); + *res = *ud; + return true; +} + + +static int node_tostring(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + lua_pushstring(L, ""); + lua_concat(L, 3); + return 1; +} + +static int node_range(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSPoint start = ts_node_start_point(node); + TSPoint end = ts_node_end_point(node); + lua_pushnumber(L, start.row); + lua_pushnumber(L, start.column); + lua_pushnumber(L, end.row); + lua_pushnumber(L, end.column); + return 4; +} + +static int node_start(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSPoint start = ts_node_start_point(node); + uint32_t start_byte = ts_node_start_byte(node); + lua_pushnumber(L, start.row); + lua_pushnumber(L, start.column); + lua_pushnumber(L, start_byte); + return 3; +} + +static int node_child_count(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + uint32_t count = ts_node_child_count(node); + lua_pushnumber(L, count); + return 1; +} + +static int node_type(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + lua_pushstring(L, ts_node_type(node)); + return 1; +} + +static int node_symbol(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSSymbol symbol = ts_node_symbol(node); + lua_pushnumber(L, symbol); + return 1; +} + +static int node_child(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + long num = lua_tointeger(L, 2); + TSNode child = ts_node_child(node, (uint32_t)num); + + lua_pushvalue(L, 1); + push_node(L, child); + return 1; +} + +static int node_descendant_for_point_range(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSPoint start = {(uint32_t)lua_tointeger(L, 2), + (uint32_t)lua_tointeger(L, 3)}; + TSPoint end = {(uint32_t)lua_tointeger(L, 4), + (uint32_t)lua_tointeger(L, 5)}; + TSNode child = ts_node_descendant_for_point_range(node, start, end); + + lua_pushvalue(L, 1); + push_node(L, child); + return 1; +} + +static int node_parent(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSNode parent = ts_node_parent(node); + push_node(L, parent); + return 1; +} + diff --git a/src/nvim/lua/treesitter.h b/src/nvim/lua/treesitter.h new file mode 100644 index 0000000000..d59b5e47a0 --- /dev/null +++ b/src/nvim/lua/treesitter.h @@ -0,0 +1,10 @@ +#ifndef NVIM_LUA_TREESITTER_H +#define NVIM_LUA_TREESITTER_H + +#include "tree_sitter/api.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "lua/treesitter.h.generated.h" +#endif + +#endif // NVIM_LUA_TREESITTER_H diff --git a/src/nvim/lua/vim.lua b/src/nvim/lua/vim.lua index c38926fe24..b67762e48e 100644 --- a/src/nvim/lua/vim.lua +++ b/src/nvim/lua/vim.lua @@ -232,9 +232,9 @@ local function __index(t, key) if key == 'inspect' then t.inspect = require('vim.inspect') return t.inspect - elseif key == 'tree_sitter' then - t.tree_sitter = require('vim.tree_sitter') - return t.tree_sitter + elseif key == 'treesitter' then + t.treesitter = require('vim.treesitter') + return t.treesitter elseif require('vim.shared')[key] ~= nil then -- Expose all `vim.shared` functions on the `vim` module. t[key] = require('vim.shared')[key] diff --git a/test/functional/lua/tree_sitter_spec.lua b/test/functional/lua/tree_sitter_spec.lua deleted file mode 100644 index e25eb47a60..0000000000 --- a/test/functional/lua/tree_sitter_spec.lua +++ /dev/null @@ -1,67 +0,0 @@ --- Test suite for testing interactions with API bindings -local helpers = require('test.functional.helpers')(after_each) - -local meths = helpers.meths -local clear = helpers.clear -local eq = helpers.eq -local insert = helpers.insert -local meth_pcall = helpers.meth_pcall -local exec_lua = helpers.exec_lua -local iswin = helpers.iswin - -before_each(clear) - -describe('tree-sitter API', function() - -- error tests not requiring a parser library - it('handles basic errors', function() - --eq({false, 'Error executing lua: vim.schedule: expected function'}, - -- meth_pcall(meths.execute_lua, "parser = vim.tree_sitter.create_parser(0, 'nosuchlang')", {})) - - - - end) - - local ts_path = os.getenv("TREE_SITTER_DIR") - - describe('with C parser', function() - if ts_path == nil then - it("works", function() pending("TREE_SITTER_PATH not set, skipping tree-sitter parser tests") end) - return - end - - before_each(function() - -- TODO the .so/.dylib/.dll thingie - local path = ts_path .. '/bin/c'..(iswin() and '.dll' or '.so') - exec_lua([[ - local path = ... - vim.tree_sitter.add_language(path,'c') - - ]], path) - end) - - it('parses buffer', function() - insert([[ - int main() { - int x = 3; - }]]) - - exec_lua([[ - parser = vim.tree_sitter.create_parser(0, "c") - tree = parser:parse_tree() - root = tree:root() - ]]) - - --eq("", exec_lua("return tostring(parser)")) - eq("", exec_lua("return tostring(tree)")) - eq("", exec_lua("return tostring(root)")) - eq({0,0,3,0}, exec_lua("return {root:range()}")) - - eq(1, exec_lua("return root:child_count()")) - exec_lua("child = root:child(0)") - eq("", exec_lua("return tostring(child)")) - eq({0,0,2,1}, exec_lua("return {child:range()}")) - end) - - end) -end) - diff --git a/test/functional/lua/treesitter_spec.lua b/test/functional/lua/treesitter_spec.lua new file mode 100644 index 0000000000..8916e59563 --- /dev/null +++ b/test/functional/lua/treesitter_spec.lua @@ -0,0 +1,67 @@ +-- Test suite for testing interactions with API bindings +local helpers = require('test.functional.helpers')(after_each) + +local meths = helpers.meths +local clear = helpers.clear +local eq = helpers.eq +local insert = helpers.insert +local meth_pcall = helpers.meth_pcall +local exec_lua = helpers.exec_lua +local iswin = helpers.iswin + +before_each(clear) + +describe('tree-sitter API', function() + -- error tests not requiring a parser library + it('handles basic errors', function() + --eq({false, 'Error executing lua: vim.schedule: expected function'}, + -- meth_pcall(meths.execute_lua, "parser = vim.treesitter.create_parser(0, 'nosuchlang')", {})) + + + + end) + + local ts_path = os.getenv("TREE_SITTER_DIR") + + describe('with C parser', function() + if ts_path == nil then + it("works", function() pending("TREE_SITTER_PATH not set, skipping tree-sitter parser tests") end) + return + end + + before_each(function() + -- TODO the .so/.dylib/.dll thingie + local path = ts_path .. '/bin/c'..(iswin() and '.dll' or '.so') + exec_lua([[ + local path = ... + vim.treesitter.add_language(path,'c') + + ]], path) + end) + + it('parses buffer', function() + insert([[ + int main() { + int x = 3; + }]]) + + exec_lua([[ + parser = vim.treesitter.create_parser(0, "c") + tree = parser:parse_tree() + root = tree:root() + ]]) + + --eq("", exec_lua("return tostring(parser)")) + eq("", exec_lua("return tostring(tree)")) + eq("", exec_lua("return tostring(root)")) + eq({0,0,3,0}, exec_lua("return {root:range()}")) + + eq(1, exec_lua("return root:child_count()")) + exec_lua("child = root:child(0)") + eq("", exec_lua("return tostring(child)")) + eq({0,0,2,1}, exec_lua("return {child:range()}")) + end) + + end) +end) + -- cgit