From 688b961d13bd54a14836f08c3ded3121d3fb15a0 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 19 Apr 2024 16:04:57 +0100 Subject: feat(treesitter): add support for wasm parsers Problem: Installing treesitter parser is hard (harder than climbing to heaven). Solution: Add optional support for wasm parsers with `wasmtime`. Notes: * Needs to be enabled by setting `ENABLE_WASMTIME` for tree-sitter and Neovim. Build with `make CMAKE_EXTRA_FLAGS=-DENABLE_WASMTIME=ON DEPS_CMAKE_FLAGS=-DENABLE_WASMTIME=ON` * Adds optional Rust (obviously) and C11 dependencies. * Wasmtime comes with a lot of features that can negatively affect Neovim performance due to library and symbol table size. Make sure to build with minimal features and full LTO. * To reduce re-compilation times, install `sccache` and build with `RUSTC_WRAPPER= make ...` --- runtime/lua/vim/treesitter/language.lua | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index d0a74daa6c..e31ce6cb59 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -109,7 +109,14 @@ function M.add(lang, opts) path = paths[1] end - vim._ts_add_language(path, lang, symbol_name) + if vim.endswith(path, '.wasm') then + if not vim._ts_add_language_from_wasm then + error(string.format("Unable to load wasm parser '%s': not built with ENABLE_WASMTIME ", path)) + end + vim._ts_add_language_from_wasm(path, lang) + else + vim._ts_add_language_from_object(path, lang, symbol_name) + end M.register(lang, filetype) end -- cgit From 041d98fe8d892a81ed659c32be5360d4f80e7d18 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Sat, 14 Sep 2024 13:27:44 +0200 Subject: feat(treesitter)!: add default fallback to `ft_to_lang` lookups Problem: Language names are only registered for filetype<->language lookups when parsers are actually loaded; this means users cannot rely on `vim.treesitter.language.get_lang()` or `get_filetypes()` to return the correct value when language and filetype coincide and always need to add explicit fallbacks. Solution: Always return the language name as valid filetype in `get_filetypes()`, and default to the filetype in `get_lang()`. Document this behavior. --- runtime/lua/vim/treesitter/language.lua | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index e31ce6cb59..e4e1c2fff0 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -7,11 +7,15 @@ local ft_to_lang = { help = 'vimdoc', } ---- Get the filetypes associated with the parser named {lang}. +--- Returns the filetypes for which a parser named {lang} is used. +--- +--- The list includes {lang} itself plus all filetypes registered via +--- |vim.treesitter.language.register()|. +--- --- @param lang string Name of parser --- @return string[] filetypes function M.get_filetypes(lang) - local r = {} ---@type string[] + local r = { lang } ---@type string[] for ft, p in pairs(ft_to_lang) do if p == lang then r[#r + 1] = ft @@ -20,6 +24,12 @@ function M.get_filetypes(lang) return r end +--- Returns the language name to be used when loading a parser for {filetype}. +--- +--- If no language has been explicitly registered via |vim.treesitter.language.register()|, +--- default to {filetype}. For composite filetypes like `html.glimmer`, only the main filetype is +--- returned. +--- --- @param filetype string --- @return string|nil function M.get_lang(filetype) @@ -29,9 +39,9 @@ function M.get_lang(filetype) if ft_to_lang[filetype] then return ft_to_lang[filetype] end - -- support subfiletypes like html.glimmer + -- for subfiletypes like html.glimmer use only "main" filetype filetype = vim.split(filetype, '.', { plain = true })[1] - return ft_to_lang[filetype] + return ft_to_lang[filetype] or filetype end ---@deprecated @@ -59,10 +69,6 @@ end ---@class vim.treesitter.language.add.Opts ---@inlinedoc --- ----Default filetype the parser should be associated with. ----(Default: {lang}) ----@field filetype? string|string[] ---- ---Optional path the parser is located at ---@field path? string --- @@ -78,21 +84,18 @@ end function M.add(lang, opts) opts = opts or {} local path = opts.path - local filetype = opts.filetype or lang local symbol_name = opts.symbol_name vim.validate({ lang = { lang, 'string' }, path = { path, 'string', true }, symbol_name = { symbol_name, 'string', true }, - filetype = { filetype, { 'string', 'table' }, true }, }) -- parser names are assumed to be lowercase (consistent behavior on case-insensitive file systems) lang = lang:lower() if vim._ts_has_language(lang) then - M.register(lang, filetype) return end @@ -117,7 +120,6 @@ function M.add(lang, opts) else vim._ts_add_language_from_object(path, lang, symbol_name) end - M.register(lang, filetype) end --- @param x string|string[] -- cgit From 99e0facf3a001608287ec6db69b01c77443c7b9d Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Sun, 15 Sep 2024 14:19:08 +0200 Subject: feat(treesitter)!: use return values in `language.add()` Problem: No clear way to check whether parsers are available for a given language. Solution: Make `language.add()` return `true` if a parser was successfully added and `nil` otherwise. Use explicit `assert` instead of relying on thrown errors. --- runtime/lua/vim/treesitter/language.lua | 45 +++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 14 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index e4e1c2fff0..9f7807e036 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -62,8 +62,22 @@ function M.require_language(lang, path, silent, symbol_name) return installed end - M.add(lang, opts) - return true + return M.add(lang, opts) +end + +--- Load wasm or native parser (wrapper) +--- todo(clason): move to C +--- +---@param path string Path of parser library +---@param lang string Language name +---@param symbol_name? string Internal symbol name for the language to load (default lang) +---@return boolean? True if parser is loaded +local function loadparser(path, lang, symbol_name) + if vim.endswith(path, '.wasm') then + return vim._ts_add_language_from_wasm and vim._ts_add_language_from_wasm(path, lang) + else + return vim._ts_add_language_from_object(path, lang, symbol_name) + end end ---@class vim.treesitter.language.add.Opts @@ -77,10 +91,18 @@ end --- Load parser with name {lang} --- ---- Parsers are searched in the `parser` runtime directory, or the provided {path} +--- Parsers are searched in the `parser` runtime directory, or the provided {path}. +--- Can be used to check for available parsers before enabling treesitter features, e.g., +--- ```lua +--- if vim.treesitter.language.add('markdown') then +--- vim.treesitter.start(bufnr, 'markdown') +--- end +--- ``` --- ---@param lang string Name of the parser (alphanumerical and `_` only) ---@param opts? vim.treesitter.language.add.Opts Options: +---@return boolean? True if parser is loaded +---@return string? Error if parser cannot be loaded function M.add(lang, opts) opts = opts or {} local path = opts.path @@ -96,30 +118,25 @@ function M.add(lang, opts) lang = lang:lower() if vim._ts_has_language(lang) then - return + return true end if path == nil then + -- allow only safe language names when looking for libraries to load if not (lang and lang:match('[%w_]+') == lang) then - error("'" .. lang .. "' is not a valid language name") + return nil, string.format('Invalid language name "%s"', lang) end local fname = 'parser/' .. lang .. '.*' local paths = api.nvim_get_runtime_file(fname, false) if #paths == 0 then - error("no parser for '" .. lang .. "' language, see :help treesitter-parsers") + return nil, string.format('No parser for language "%s"', lang) end path = paths[1] end - if vim.endswith(path, '.wasm') then - if not vim._ts_add_language_from_wasm then - error(string.format("Unable to load wasm parser '%s': not built with ENABLE_WASMTIME ", path)) - end - vim._ts_add_language_from_wasm(path, lang) - else - vim._ts_add_language_from_object(path, lang, symbol_name) - end + return loadparser(path, lang, symbol_name) or nil, + string.format('Cannot load parser %s for language "%s"', path, lang) end --- @param x string|string[] -- cgit