From c032e83b22994332dd8769ef34cb817906a63cac Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Thu, 26 Jan 2023 09:42:23 +0100 Subject: fix(treesitter): validate language name Problem: Some injections (like markdown) allow specifying arbitrary language names for code blocks, which may be lead to errors when looking for a corresponding parser in runtime path. Solution: Validate that the language name only contains alphanumeric characters and `_` (e.g., for `c_sharp`) and error otherwise. --- runtime/lua/vim/treesitter/language.lua | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index c92d63b8c4..8634e53b7b 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -6,7 +6,7 @@ local M = {} --- --- Parsers are searched in the `parser` runtime directory, or the provided {path} --- ----@param lang string Language the parser should parse +---@param lang string Language the parser should parse (alphanumerical and `_` only) ---@param path (string|nil) Optional path the parser is located at ---@param silent (boolean|nil) Don't throw an error if language not found ---@param symbol_name (string|nil) Internal symbol name for the language to load @@ -16,13 +16,19 @@ function M.require_language(lang, path, silent, symbol_name) return true end if path == nil then - local fname = 'parser/' .. vim.fn.fnameescape(lang) .. '.*' + if not (lang and lang:match('[%w_]+') == lang) then + if silent then + return false + end + error("'" .. lang .. "' is not a valid language name") + end + + local fname = 'parser/' .. lang .. '.*' local paths = a.nvim_get_runtime_file(fname, false) if #paths == 0 then if silent then return false end - error("no parser for '" .. lang .. "' language, see :help treesitter-parsers") end path = paths[1] -- cgit From 8714a4009c0f0be0bb27a6b3eb486eeb3d9f3049 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 21 Feb 2023 17:09:18 +0000 Subject: feat(treesitter): add filetype -> lang API Problem: vim.treesitter does not know how to map a specific filetype to a parser. This creates problems since in a few places (including in vim.treesitter itself), the filetype is incorrectly used in place of lang. Solution: Add an API to enable this: - Add vim.treesitter.language.add() as a replacement for vim.treesitter.language.require_language(). - Optional arguments are now passed via an opts table. - Also takes a filetype (or list of filetypes) so we can keep track of what filetypes are associated with which langs. - Deprecated vim.treesitter.language.require_language(). - Add vim.treesitter.language.get_lang() which returns the associated lang for a given filetype. - Add vim.treesitter.language.register() to associate filetypes to a lang without loading the parser. --- runtime/lua/vim/treesitter/language.lua | 84 +++++++++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 8 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 8634e53b7b..8637d7d544 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -2,19 +2,66 @@ local a = vim.api local M = {} +---@type table +local ft_to_lang = {} + +---@param filetype string +---@return string|nil +function M.get_lang(filetype) + return ft_to_lang[filetype] +end + +---@deprecated +function M.require_language(lang, path, silent, symbol_name) + return M.add(lang, { + silent = silent, + path = path, + symbol_name = symbol_name, + }) +end + +---@class treesitter.RequireLangOpts +---@field path? string +---@field silent? boolean +---@field filetype? string|string[] +---@field symbol_name? string + --- Asserts that a parser for the language {lang} is installed. --- --- Parsers are searched in the `parser` runtime directory, or the provided {path} --- ---@param lang string Language the parser should parse (alphanumerical and `_` only) ----@param path (string|nil) Optional path the parser is located at ----@param silent (boolean|nil) Don't throw an error if language not found ----@param symbol_name (string|nil) Internal symbol name for the language to load +---@param opts (table|nil) Options: +--- - filetype (string|string[]) Filetype(s) that lang can be parsed with. +--- Note this is not strictly the same as lang since a single lang can +--- parse multiple filetypes. +--- Defaults to lang. +--- - path (string|nil) Optional path the parser is located at +--- - symbol_name (string|nil) Internal symbol name for the language to load +--- - silent (boolean|nil) Don't throw an error if language not found ---@return boolean If the specified language is installed -function M.require_language(lang, path, silent, symbol_name) +function M.add(lang, opts) + ---@cast opts treesitter.RequireLangOpts + opts = opts or {} + local path = opts.path + local silent = opts.silent + local filetype = opts.filetype or lang + local symbol_name = opts.symbol_name + + vim.validate({ + lang = { lang, 'string' }, + path = { path, 'string', true }, + silent = { silent, 'boolean', true }, + symbol_name = { symbol_name, 'string', true }, + filetype = { filetype, { 'string', 'table' }, true }, + }) + + M.register(lang, filetype or lang) + if vim._ts_has_language(lang) then return true end + if path == nil then if not (lang and lang:match('[%w_]+') == lang) then if silent then @@ -35,9 +82,9 @@ function M.require_language(lang, path, silent, symbol_name) end if silent then - return pcall(function() - vim._ts_add_language(path, lang, symbol_name) - end) + if not pcall(vim._ts_add_language, path, lang, symbol_name) then + return false + end else vim._ts_add_language(path, lang, symbol_name) end @@ -45,6 +92,27 @@ function M.require_language(lang, path, silent, symbol_name) return true end +--- Register a lang to be used for a filetype (or list of filetypes). +---@param lang string Language to register +---@param filetype string|string[] Filetype(s) to associate with lang +function M.register(lang, filetype) + vim.validate({ + lang = { lang, 'string' }, + filetype = { filetype, { 'string', 'table' } }, + }) + + local filetypes ---@type string[] + if type(filetype) == 'string' then + filetypes = { filetype } + else + filetypes = filetype + end + + for _, f in ipairs(filetypes) do + ft_to_lang[f] = lang + end +end + --- Inspects the provided language. --- --- Inspecting provides some useful information on the language like node names, ... @@ -52,7 +120,7 @@ end ---@param lang string Language ---@return table function M.inspect_language(lang) - M.require_language(lang) + M.add(lang) return vim._ts_inspect_language(lang) end -- cgit From c57af5d41cd039194dbd9c6fb5b68b377d2a5b59 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 24 Feb 2023 09:50:59 +0000 Subject: feat(treesitter)!: remove silent option from language.add() Simply use `pcall` if you want to silence an error. --- runtime/lua/vim/treesitter/language.lua | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 8637d7d544..0796383bf5 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -13,11 +13,19 @@ end ---@deprecated function M.require_language(lang, path, silent, symbol_name) - return M.add(lang, { + local opts = { silent = silent, path = path, symbol_name = symbol_name, - }) + } + + if silent then + local installed = pcall(M.add, lang, opts) + return installed + end + + M.add(lang, opts) + return true end ---@class treesitter.RequireLangOpts @@ -38,20 +46,16 @@ end --- Defaults to lang. --- - path (string|nil) Optional path the parser is located at --- - symbol_name (string|nil) Internal symbol name for the language to load ---- - silent (boolean|nil) Don't throw an error if language not found ----@return boolean If the specified language is installed function M.add(lang, opts) ---@cast opts treesitter.RequireLangOpts opts = opts or {} local path = opts.path - local silent = opts.silent local filetype = opts.filetype or lang local symbol_name = opts.symbol_name vim.validate({ lang = { lang, 'string' }, path = { path, 'string', true }, - silent = { silent, 'boolean', true }, symbol_name = { symbol_name, 'string', true }, filetype = { filetype, { 'string', 'table' }, true }, }) @@ -64,32 +68,18 @@ function M.add(lang, opts) if path == nil then if not (lang and lang:match('[%w_]+') == lang) then - if silent then - return false - end error("'" .. lang .. "' is not a valid language name") end local fname = 'parser/' .. lang .. '.*' local paths = a.nvim_get_runtime_file(fname, false) if #paths == 0 then - if silent then - return false - end error("no parser for '" .. lang .. "' language, see :help treesitter-parsers") end path = paths[1] end - if silent then - if not pcall(vim._ts_add_language, path, lang, symbol_name) then - return false - end - else - vim._ts_add_language(path, lang, symbol_name) - end - - return true + vim._ts_add_language(path, lang, symbol_name) end --- Register a lang to be used for a filetype (or list of filetypes). -- cgit From f64098a2df774c79dd454f63ac491570cdcaf2b2 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 27 Feb 2023 15:33:18 +0000 Subject: fix(treesitter): fixup for health --- runtime/lua/vim/treesitter/language.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 0796383bf5..5bcc786e88 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -63,7 +63,7 @@ function M.add(lang, opts) M.register(lang, filetype or lang) if vim._ts_has_language(lang) then - return true + return end if path == nil then -- cgit From 6d4f48182131c36d57589eefd4cefe3c70256d04 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 3 Mar 2023 09:44:02 +0000 Subject: fix(treesitter): disallow empty filetypes Fixes #22473 --- runtime/lua/vim/treesitter/language.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 5bcc786e88..5f34d9cd56 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -60,6 +60,16 @@ function M.add(lang, opts) filetype = { filetype, { 'string', 'table' }, true }, }) + if filetype == '' then + error(string.format("'%s' is not a valid filetype", filetype)) + elseif type(filetype) == 'table' then + for _, f in ipairs(filetype) do + if f == '' then + error(string.format("'%s' is not a valid filetype", filetype)) + end + end + end + M.register(lang, filetype or lang) if vim._ts_has_language(lang) then -- cgit From adfa9de8ebc4bce96d212280eccddc0306d1b013 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 10 Mar 2023 10:12:57 +0000 Subject: fix(treesitter): do not error on empty filetype Ignore instead --- runtime/lua/vim/treesitter/language.lua | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 5f34d9cd56..47375fd5e6 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -60,16 +60,6 @@ function M.add(lang, opts) filetype = { filetype, { 'string', 'table' }, true }, }) - if filetype == '' then - error(string.format("'%s' is not a valid filetype", filetype)) - elseif type(filetype) == 'table' then - for _, f in ipairs(filetype) do - if f == '' then - error(string.format("'%s' is not a valid filetype", filetype)) - end - end - end - M.register(lang, filetype or lang) if vim._ts_has_language(lang) then @@ -109,7 +99,9 @@ function M.register(lang, filetype) end for _, f in ipairs(filetypes) do - ft_to_lang[f] = lang + if f ~= '' then + ft_to_lang[f] = lang + end end end -- cgit From cbbf8bd666c8419fdab80a0887948c8a36279c19 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 24 Mar 2023 14:43:14 +0000 Subject: feat(treesitter)!: deprecate top level indexes to modules (#22761) The following top level Treesitter functions have been moved: - vim.treesitter.inspect_language() -> vim.treesitter.language.inspect() - vim.treesitter.get_query_files() -> vim.treesitter.query.get_files() - vim.treesitter.set_query() -> vim.treesitter.query.set() - vim.treesitter.query.set_query() -> vim.treesitter.query.set() - vim.treesitter.get_query() -> vim.treesitter.query.get() - vim.treesitter.query.get_query() -> vim.treesitter.query.get() - vim.treesitter.parse_query() -> vim.treesitter.query.parse() - vim.treesitter.query.parse_query() -> vim.treesitter.query.parse() - vim.treesitter.add_predicate() -> vim.treesitter.query.add_predicate() - vim.treesitter.add_directive() -> vim.treesitter.query.add_directive() - vim.treesitter.list_predicates() -> vim.treesitter.query.list_predicates() - vim.treesitter.list_directives() -> vim.treesitter.query.list_directives() - vim.treesitter.query.get_range() -> vim.treesitter.get_range() - vim.treesitter.query.get_node_text() -> vim.treesitter.get_node_text() --- runtime/lua/vim/treesitter/language.lua | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 47375fd5e6..974d66ec05 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -1,5 +1,6 @@ local a = vim.api +---@class TSLanguageModule local M = {} ---@type table @@ -111,9 +112,19 @@ end --- ---@param lang string Language ---@return table -function M.inspect_language(lang) +function M.inspect(lang) M.add(lang) return vim._ts_inspect_language(lang) end +---@deprecated +function M.inspect_language(...) + vim.deprecate( + 'vim.treesitter.language.inspect_language()', + 'vim.treesitter.language.inspect()', + '0.10' + ) + return M.inspect(...) +end + return M -- cgit From 61e54f26361b2e7d08eabde9a4cbf42aaa41683b Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Thu, 30 Mar 2023 10:26:28 +0100 Subject: feat: add `vim.treesitter.language.get_filetypes()` (#22643) --- runtime/lua/vim/treesitter/language.lua | 57 +++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 20 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 974d66ec05..b1c788e6ba 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -6,9 +6,25 @@ local M = {} ---@type table local ft_to_lang = {} ----@param filetype string ----@return string|nil +--- Get the filetypes associated with the parser named {lang}. +--- @param lang string Name of parser +--- @return string[] filetypes +function M.get_filetypes(lang) + local r = {} ---@type string[] + for ft, p in pairs(ft_to_lang) do + if p == lang then + r[#r + 1] = ft + end + end + return r +end + +--- @param filetype string +--- @return string|nil function M.get_lang(filetype) + if filetype == '' then + return + end return ft_to_lang[filetype] end @@ -35,16 +51,14 @@ end ---@field filetype? string|string[] ---@field symbol_name? string ---- Asserts that a parser for the language {lang} is installed. +--- Load parser with name {lang} --- --- Parsers are searched in the `parser` runtime directory, or the provided {path} --- ----@param lang string Language the parser should parse (alphanumerical and `_` only) +---@param lang string Name of the parser (alphanumerical and `_` only) ---@param opts (table|nil) Options: ---- - filetype (string|string[]) Filetype(s) that lang can be parsed with. ---- Note this is not strictly the same as lang since a single lang can ---- parse multiple filetypes. ---- Defaults to lang. +--- - filetype (string|string[]) Default filetype the parser should be associated with. +--- Defaults to {lang}. --- - path (string|nil) Optional path the parser is located at --- - symbol_name (string|nil) Internal symbol name for the language to load function M.add(lang, opts) @@ -61,7 +75,7 @@ function M.add(lang, opts) filetype = { filetype, { 'string', 'table' }, true }, }) - M.register(lang, filetype or lang) + M.register(lang, filetype) if vim._ts_has_language(lang) then return @@ -83,23 +97,26 @@ function M.add(lang, opts) vim._ts_add_language(path, lang, symbol_name) end ---- Register a lang to be used for a filetype (or list of filetypes). ----@param lang string Language to register ----@param filetype string|string[] Filetype(s) to associate with lang +--- @private +--- @param x string|string[] +--- @return string[] +local function ensure_list(x) + if type(x) == 'table' then + return x + end + return { x } +end + +--- Register a parser named {lang} to be used for {filetype}(s). +--- @param lang string Name of parser +--- @param filetype string|string[] Filetype(s) to associate with lang function M.register(lang, filetype) vim.validate({ lang = { lang, 'string' }, filetype = { filetype, { 'string', 'table' } }, }) - local filetypes ---@type string[] - if type(filetype) == 'string' then - filetypes = { filetype } - else - filetypes = filetype - end - - for _, f in ipairs(filetypes) do + for _, f in ipairs(ensure_list(filetype)) do if f ~= '' then ft_to_lang[f] = lang end -- cgit From d7f7450017b9b05303698a6cda54303ef22c63b3 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Fri, 31 Mar 2023 17:09:00 +0200 Subject: refactor(treesitter)!: rename help parser to vimdoc --- runtime/lua/vim/treesitter/language.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index b1c788e6ba..5b74bb6200 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -4,7 +4,9 @@ local a = vim.api local M = {} ---@type table -local ft_to_lang = {} +local ft_to_lang = { + help = 'vimdoc', +} --- Get the filetypes associated with the parser named {lang}. --- @param lang string Name of parser -- cgit From 34ac75b32927328a0c691c5bda987c0fdb5ce9eb Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Wed, 5 Apr 2023 17:19:53 +0100 Subject: refactor: rename local API alias from a to api Problem: Codebase inconsistently binds vim.api onto a or api. Solution: Use api everywhere. a as an identifier is too short to have at the module level. --- runtime/lua/vim/treesitter/language.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 5b74bb6200..b616d4d70b 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -1,4 +1,4 @@ -local a = vim.api +local api = vim.api ---@class TSLanguageModule local M = {} @@ -89,7 +89,7 @@ function M.add(lang, opts) end local fname = 'parser/' .. lang .. '.*' - local paths = a.nvim_get_runtime_file(fname, false) + local paths = api.nvim_get_runtime_file(fname, false) if #paths == 0 then error("no parser for '" .. lang .. "' language, see :help treesitter-parsers") end -- cgit From 32dc484ec9ec2d86a5fc7127e37f1ef115b9be76 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Sat, 13 May 2023 13:29:11 +0200 Subject: fix(treesitter): support subfiletypes in get_lang (#23605) --- runtime/lua/vim/treesitter/language.lua | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index b616d4d70b..08c297c9ad 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -27,6 +27,11 @@ function M.get_lang(filetype) if filetype == '' then return end + if ft_to_lang[filetype] then + return ft_to_lang[filetype] + end + -- support subfiletypes like html.glimmer + filetype = vim.split(filetype, '.', { plain = true })[1] return ft_to_lang[filetype] end -- cgit From be74807eef13ff8c90d55cf8b22b01d6d33b1641 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 18 Jul 2023 15:42:30 +0100 Subject: docs(lua): more improvements (#24387) * docs(lua): teach lua2dox how to table * docs(lua): teach gen_vimdoc.py about local functions No more need to mark local functions with @private * docs(lua): mention @nodoc and @meta in dev-lua-doc * fixup! Co-authored-by: Justin M. Keyes --------- Co-authored-by: Justin M. Keyes --- runtime/lua/vim/treesitter/language.lua | 1 - 1 file changed, 1 deletion(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 08c297c9ad..9695e2c41c 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -104,7 +104,6 @@ function M.add(lang, opts) vim._ts_add_language(path, lang, symbol_name) end ---- @private --- @param x string|string[] --- @return string[] local function ensure_list(x) -- cgit From a4743487b71b54f05063465d5f8cde8014bcb73c Mon Sep 17 00:00:00 2001 From: L Lllvvuu Date: Thu, 14 Sep 2023 00:08:43 -0700 Subject: fix(treesitter): `language.add` - only register parser if it exists Fixes: #24531 --- runtime/lua/vim/treesitter/language.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime/lua/vim/treesitter/language.lua') diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua index 9695e2c41c..15bf666a1e 100644 --- a/runtime/lua/vim/treesitter/language.lua +++ b/runtime/lua/vim/treesitter/language.lua @@ -82,9 +82,8 @@ function M.add(lang, opts) filetype = { filetype, { 'string', 'table' }, true }, }) - M.register(lang, filetype) - if vim._ts_has_language(lang) then + M.register(lang, filetype) return end @@ -102,6 +101,7 @@ function M.add(lang, opts) end vim._ts_add_language(path, lang, symbol_name) + M.register(lang, filetype) end --- @param x string|string[] -- cgit