From 517f0cc634b985057da5b95cf4ad659ee456a77e Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Mon, 4 Dec 2023 12:38:31 -0800 Subject: build: enable lintlua for scripts/ dir #26391 Problem: We don't enable stylua for many Lua scripts. Automating code-style is an important tool for reducing time spent on accidental (non-essential) complexity. Solution: - Enable lintlua for `scripts/` directory. - Specify `call_parentheses = "Input"`, we should allow kwargs-style function invocations. --- scripts/lua2dox.lua | 55 +++++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 25 deletions(-) (limited to 'scripts/lua2dox.lua') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 1c8bc5a3cb..c4ad7fbb03 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -59,9 +59,12 @@ local TAGGED_TYPES = { 'TSNode', 'LanguageTree' } -- Document these as 'table' local ALIAS_TYPES = { - 'Range', 'Range4', 'Range6', 'TSMetadata', + 'Range', + 'Range4', + 'Range6', + 'TSMetadata', 'vim.filetype.add.filetypes', - 'vim.filetype.match.args' + 'vim.filetype.match.args', } local debug_outfile = nil --- @type string? @@ -103,7 +106,7 @@ function StreamRead.new(filename) -- syphon lines to our table local filecontents = {} --- @type string[] for line in io.lines(filename) do - filecontents[#filecontents+1] = line + filecontents[#filecontents + 1] = line end return setmetatable({ @@ -176,9 +179,15 @@ local function process_magic(line, generics) local magic_split = vim.split(magic, ' ', { plain = true }) local directive = magic_split[1] - if vim.list_contains({ - 'cast', 'diagnostic', 'overload', 'meta', 'type' - }, directive) then + if + vim.list_contains({ + 'cast', + 'diagnostic', + 'overload', + 'meta', + 'type', + }, directive) + then -- Ignore LSP directives return '// gg:"' .. line .. '"' end @@ -202,8 +211,7 @@ local function process_magic(line, generics) if directive == 'param' then for _, type in ipairs(TYPES) do magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2') - magic = - magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') + magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') end magic_split = vim.split(magic, ' ', { plain = true }) type_index = 3 @@ -225,7 +233,7 @@ local function process_magic(line, generics) -- fix optional parameters if magic_split[2]:find('%?$') then if not ty:find('nil') then - ty = ty .. '|nil' + ty = ty .. '|nil' end magic_split[2] = magic_split[2]:sub(1, -2) end @@ -240,18 +248,15 @@ local function process_magic(line, generics) end for _, type in ipairs(ALIAS_TYPES) do - ty = ty:gsub('^'..type..'$', 'table') --- @type string + ty = ty:gsub('^' .. type .. '$', 'table') --- @type string end -- surround some types by () for _, type in ipairs(TYPES) do - ty = ty - :gsub('^(' .. type .. '|nil):?$', '(%1)') - :gsub('^(' .. type .. '):?$', '(%1)') + ty = ty:gsub('^(' .. type .. '|nil):?$', '(%1)'):gsub('^(' .. type .. '):?$', '(%1)') end magic_split[type_index] = ty - end magic = table.concat(magic_split, ' ') @@ -281,7 +286,7 @@ local function process_block_comment(line, in_stream) -- easier to program in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2))) end - comment_parts[#comment_parts+1] = thisComment + comment_parts[#comment_parts + 1] = thisComment end local comment = table.concat(comment_parts) @@ -303,7 +308,7 @@ local function process_function_header(line) if fn:sub(1, 1) == '(' then -- it's an anonymous function - return '// ZZ: '..line + return '// ZZ: ' .. line end -- fn has a name, so is interesting @@ -330,10 +335,7 @@ local function process_function_header(line) comma = ', ' end - fn = fn:sub(1, paren_start) - .. 'self' - .. comma - .. fn:sub(paren_start + 1) + fn = fn:sub(1, paren_start) .. 'self' .. comma .. fn:sub(paren_start + 1) end if line:match('local') then @@ -357,7 +359,7 @@ local function process_line(line, in_stream, generics) return process_magic(line:sub(4), generics) end - if vim.startswith(line, '--'..'[[') then -- it's a long comment + if vim.startswith(line, '--' .. '[[') then -- it's a long comment return process_block_comment(line:sub(5), in_stream) end @@ -375,7 +377,7 @@ local function process_line(line, in_stream, generics) local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=') if v and v:match('%.') then -- Special: this lets gen_vimdoc.py handle tables. - return 'table '..v..'() {}' + return 'table ' .. v .. '() {}' end end @@ -418,7 +420,7 @@ local TApp = { timestamp = os.date('%c %Z', os.time()), name = 'Lua2DoX', version = '0.2 20130128', - copyright = 'Copyright (c) Simon Dales 2012-13' + copyright = 'Copyright (c) Simon Dales 2012-13', } setmetatable(TApp, { __index = TApp }) @@ -447,12 +449,15 @@ if arg[1] == '--help' then elseif arg[1] == '--version' then writeln(TApp:getVersion()) writeln(TApp.copyright) -else -- It's a filter. +else -- It's a filter. local filename = arg[1] if arg[2] == '--outdir' then local outdir = arg[3] - if type(outdir) ~= 'string' or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) then + if + type(outdir) ~= 'string' + or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) + then error(('invalid --outdir: "%s"'):format(tostring(outdir))) end vim.fn.mkdir(outdir, 'p') -- cgit From 2f9ee9b6cfc61a0504fc0bc22bdf481828e2ea91 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 9 Jan 2024 17:36:46 +0000 Subject: fix(doc): improve doc generation of types using lpeg Added a lpeg grammar for LuaCATS and use it in lua2dox.lua --- scripts/lua2dox.lua | 192 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 114 insertions(+), 78 deletions(-) (limited to 'scripts/lua2dox.lua') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index c4ad7fbb03..abc9e5b338 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -55,17 +55,7 @@ The effect is that you will get the function documented, but not with the parame local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' } -local TAGGED_TYPES = { 'TSNode', 'LanguageTree' } - --- Document these as 'table' -local ALIAS_TYPES = { - 'Range', - 'Range4', - 'Range6', - 'TSMetadata', - 'vim.filetype.add.filetypes', - 'vim.filetype.match.args', -} +local luacats_parser = require('src/nvim/generators/luacats_grammar') local debug_outfile = nil --- @type string? local debug_output = {} @@ -161,6 +151,91 @@ local function removeCommentFromLine(line) return line:sub(1, pos_comment - 1), line:sub(pos_comment) end +--- @param parsed luacats.Return +--- @return string +local function get_return_type(parsed) + local elems = {} --- @type string[] + for _, v in ipairs(parsed) do + local e = v.type --- @type string + if v.name then + e = e .. ' ' .. v.name --- @type string + end + elems[#elems + 1] = e + end + return '(' .. table.concat(elems, ', ') .. ')' +end + +--- @param name string +--- @return string +local function process_name(name, optional) + if optional then + name = name:sub(1, -2) --- @type string + end + return name +end + +--- @param ty string +--- @param generics table +--- @return string +local function process_type(ty, generics, optional) + -- replace generic types + for k, v in pairs(generics) do + ty = ty:gsub(k, v) --- @type string + end + + -- strip parens + ty = ty:gsub('^%((.*)%)$', '%1') + + if optional and not ty:find('nil') then + ty = ty .. '?' + end + + -- remove whitespace in unions + ty = ty:gsub('%s*|%s*', '|') + + -- replace '|nil' with '?' + ty = ty:gsub('|nil', '?') + ty = ty:gsub('nil|(.*)', '%1?') + + return '(`' .. ty .. '`)' +end + +--- @param parsed luacats.Param +--- @param generics table +--- @return string +local function process_param(parsed, generics) + local name, ty = parsed.name, parsed.type + local optional = vim.endswith(name, '?') + + return table.concat({ + '/// @param', + process_name(name, optional), + process_type(ty, generics, optional), + parsed.desc, + }, ' ') +end + +--- @param parsed luacats.Return +--- @param generics table +--- @return string +local function process_return(parsed, generics) + local ty, name --- @type string, string + if #parsed == 1 then + ty, name = parsed[1].type, parsed[1].name or '' + else + ty, name = get_return_type(parsed), '' + end + + local optional = vim.endswith(name, '?') + + return table.concat({ + '/// @return', + process_type(ty, generics, optional), + process_name(name, optional), + parsed.desc, + }, ' ') +end + --- Processes "@…" directives in a docstring line. --- --- @param line string @@ -175,93 +250,54 @@ local function process_magic(line, generics) return '/// ' .. line end - local magic = line:sub(2) - local magic_split = vim.split(magic, ' ', { plain = true }) + local magic_split = vim.split(line, ' ', { plain = true }) local directive = magic_split[1] if vim.list_contains({ - 'cast', - 'diagnostic', - 'overload', - 'meta', - 'type', + '@cast', + '@diagnostic', + '@overload', + '@meta', + '@type', }, directive) then -- Ignore LSP directives return '// gg:"' .. line .. '"' - end - - if directive == 'defgroup' or directive == 'addtogroup' then + elseif directive == '@defgroup' or directive == '@addtogroup' then -- Can't use '.' in defgroup, so convert to '--' - return '/// @' .. magic:gsub('%.', '-dot-') - end - - if directive == 'generic' then - local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)') - if generic_type == '' then - generic_type = 'any' - end - generics[generic_name] = generic_type - return + return '/// ' .. line:gsub('%.', '-dot-') end - local type_index = 2 - - if directive == 'param' then + -- preprocess line before parsing + if directive == '@param' or directive == '@return' then for _, type in ipairs(TYPES) do - magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2') - magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') - end - magic_split = vim.split(magic, ' ', { plain = true }) - type_index = 3 - elseif directive == 'return' then - for _, type in ipairs(TYPES) do - magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1') - magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1') - end - -- Remove first "#" comment char, if any. https://github.com/LuaLS/lua-language-server/wiki/Annotations#return - magic = magic:gsub('# ', '', 1) - -- handle the return of vim.spell.check - magic = magic:gsub('({.*}%[%])', '`%1`') - magic_split = vim.split(magic, ' ', { plain = true }) - end - - local ty = magic_split[type_index] - - if ty then - -- fix optional parameters - if magic_split[2]:find('%?$') then - if not ty:find('nil') then - ty = ty .. '|nil' - end - magic_split[2] = magic_split[2]:sub(1, -2) - end + line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') + line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') - -- replace generic types - for k, v in pairs(generics) do - ty = ty:gsub(k, v) --- @type string + line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1') + line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1') end + end - for _, type in ipairs(TAGGED_TYPES) do - ty = ty:gsub(type, '|%1|') - end + local parsed = luacats_parser:match(line) - for _, type in ipairs(ALIAS_TYPES) do - ty = ty:gsub('^' .. type .. '$', 'table') --- @type string - end + if not parsed then + return '/// ' .. line + end - -- surround some types by () - for _, type in ipairs(TYPES) do - ty = ty:gsub('^(' .. type .. '|nil):?$', '(%1)'):gsub('^(' .. type .. '):?$', '(%1)') - end + local kind = parsed.kind - magic_split[type_index] = ty + if kind == 'generic' then + generics[parsed.name] = parsed.type or 'any' + return + elseif kind == 'param' then + return process_param(parsed --[[@as luacats.Param]], generics) + elseif kind == 'return' then + return process_return(parsed --[[@as luacats.Return]], generics) end - magic = table.concat(magic_split, ' ') - - return '/// @' .. magic + error(string.format('unhandled parsed line %q: %s', line, parsed)) end --- @param line string -- cgit From a7df0415ab6ae9a89ca12c6765758bfd54fa69c9 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Sat, 13 Jan 2024 18:36:30 -0500 Subject: fix(lua2dox): filter out the entire `---@alias` block Problem: Any preceding luadocs block that define alias types with `@alias` magic would be prepended to the documentation of functions that follow, despite the "blank line" separator. For example: ``` --- @alias some.type.between.functions --- Blah blah long documentation for alias --- | "foo" # foo --- | "bar" # bar --- The documentation that should appear in vimdoc. function M.function_to_include_in_doc() ... end ``` then the vimdoc generated for `function_to_include_in_doc` would include the text from the alias block (e.g., "Blah blah ... for alias"). Solution: - refactor: Lua2DoxFilter should maintain its own internal state `generics`, rather than carrying it as a parameter to local helper functions. - Add another boolean state `boolean_state` which represents whether to ignore the current docstring block (magic lines). This flag will be reset as soon as the block is end. Note: As expected, there is no change at all in the current docs generated, because we have been working around and writing luadoc comments so that such erroneous docstring resulting from preceding `@alias` blocks won't appear. --- scripts/lua2dox.lua | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) (limited to 'scripts/lua2dox.lua') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index abc9e5b338..871720bd2a 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -136,9 +136,17 @@ end -- input filter --- @class Lua2DoxFilter -local Lua2DoxFilter = {} +local Lua2DoxFilter = { + generics = {}, --- @type table + block_ignore = false, --- @type boolean +} setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter }) +function Lua2DoxFilter:reset() + self.generics = {} + self.block_ignore = false +end + --- trim comment off end of string --- --- @param line string @@ -239,13 +247,16 @@ end --- Processes "@…" directives in a docstring line. --- --- @param line string ---- @param generics table --- @return string? -local function process_magic(line, generics) +function Lua2DoxFilter:process_magic(line) line = line:gsub('^%s+@', '@') line = line:gsub('@package', '@private') line = line:gsub('@nodoc', '@private') + if self.block_ignore then + return '// gg:" ' .. line .. '"' + end + if not vim.startswith(line, '@') then -- it's a magic comment return '/// ' .. line end @@ -269,6 +280,12 @@ local function process_magic(line, generics) return '/// ' .. line:gsub('%.', '-dot-') end + if directive == '@alias' then + -- this contiguous block should be all ignored. + self.block_ignore = true + return '// gg:"' .. line .. '"' + end + -- preprocess line before parsing if directive == '@param' or directive == '@return' then for _, type in ipairs(TYPES) do @@ -289,12 +306,12 @@ local function process_magic(line, generics) local kind = parsed.kind if kind == 'generic' then - generics[parsed.name] = parsed.type or 'any' + self.generics[parsed.name] = parsed.type or 'any' return elseif kind == 'param' then - return process_param(parsed --[[@as luacats.Param]], generics) + return process_param(parsed --[[@as luacats.Param]], self.generics) elseif kind == 'return' then - return process_return(parsed --[[@as luacats.Return]], generics) + return process_return(parsed --[[@as luacats.Return]], self.generics) end error(string.format('unhandled parsed line %q: %s', line, parsed)) @@ -303,7 +320,7 @@ end --- @param line string --- @param in_stream StreamRead --- @return string -local function process_block_comment(line, in_stream) +function Lua2DoxFilter:process_block_comment(line, in_stream) local comment_parts = {} --- @type string[] local done --- @type boolean? @@ -337,7 +354,7 @@ end --- @param line string --- @return string -local function process_function_header(line) +function Lua2DoxFilter:process_function_header(line) local pos_fn = assert(line:find('function')) -- we've got a function local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8))) @@ -385,18 +402,17 @@ end --- @param line string --- @param in_stream StreamRead ---- @param generics table> --- @return string? -local function process_line(line, in_stream, generics) +function Lua2DoxFilter:process_line(line, in_stream) local line_raw = line line = vim.trim(line) if vim.startswith(line, '---') then - return process_magic(line:sub(4), generics) + return Lua2DoxFilter:process_magic(line:sub(4)) end if vim.startswith(line, '--' .. '[[') then -- it's a long comment - return process_block_comment(line:sub(5), in_stream) + return Lua2DoxFilter:process_block_comment(line:sub(5), in_stream) end -- Hax... I'm sorry @@ -406,7 +422,7 @@ local function process_line(line, in_stream, generics) line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') if line:find('^function') or line:find('^local%s+function') then - return process_function_header(line) + return Lua2DoxFilter:process_function_header(line) end if not line:match('^local') then @@ -429,15 +445,13 @@ end function Lua2DoxFilter:filter(filename) local in_stream = StreamRead.new(filename) - local generics = {} --- @type table - while not in_stream:eof() do local line = in_stream:getLine() - local out_line = process_line(line, in_stream, generics) + local out_line = self:process_line(line, in_stream) if not vim.startswith(vim.trim(line), '---') then - generics = {} + self:reset() end if out_line then -- cgit From 50284d07b6f020c819aeb07bfb30d88453e63b6d Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 9 Jan 2024 12:47:57 +0000 Subject: fix(diagnostic): typing --- scripts/lua2dox.lua | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts/lua2dox.lua') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 871720bd2a..4f9973449e 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -291,9 +291,11 @@ function Lua2DoxFilter:process_magic(line) for _, type in ipairs(TYPES) do line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') + line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1') line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1') + line = line:gsub('^@return%s+.*%((' .. type .. '%?)%)', '@return %1') end end -- cgit From c4417ae70c03815c2fb64edb479017e79d223cf7 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 6 Feb 2024 15:08:17 +0000 Subject: fix(doc): prevent doxygen confusion --- scripts/lua2dox.lua | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'scripts/lua2dox.lua') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 4f9973449e..0b3daa59b2 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -447,6 +447,8 @@ end function Lua2DoxFilter:filter(filename) local in_stream = StreamRead.new(filename) + local last_was_magic = false + while not in_stream:eof() do local line = in_stream:getLine() @@ -457,6 +459,16 @@ function Lua2DoxFilter:filter(filename) end if out_line then + -- Ensure all magic blocks associate with some object to prevent doxygen + -- from getting confused. + if vim.startswith(out_line, '///') then + last_was_magic = true + else + if last_was_magic and out_line:match('^// zz: [^-]+') then + writeln('local_function _ignore() {}') + end + last_was_magic = false + end writeln(out_line) end end -- cgit From 9beb40a4db5613601fc1a4b828a44e5977eca046 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Thu, 15 Feb 2024 17:16:04 +0000 Subject: feat(docs): replace lua2dox.lua Problem: The documentation flow (`gen_vimdoc.py`) has several issues: - it's not very versatile - depends on doxygen - doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C. - The intermediate XML files and filters makes it too much like a rube goldberg machine. Solution: Re-implement the flow using Lua, LPEG and treesitter. - `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic. - `lua2dox.lua` is gone! - No more XML files. - Doxygen is now longer used and instead we now use: - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`). - LPEG for C parsing (see `scripts/cdoc_parser.lua`) - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`). - Treesitter for Markdown parsing (see `scripts/text_utils.lua`). - The generated `runtime/doc/*.mpack` files have been removed. - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly. - Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change). --- scripts/lua2dox.lua | 544 ---------------------------------------------------- 1 file changed, 544 deletions(-) delete mode 100644 scripts/lua2dox.lua (limited to 'scripts/lua2dox.lua') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua deleted file mode 100644 index 0b3daa59b2..0000000000 --- a/scripts/lua2dox.lua +++ /dev/null @@ -1,544 +0,0 @@ ------------------------------------------------------------------------------ --- Copyright (C) 2012 by Simon Dales -- --- simon@purrsoft.co.uk -- --- -- --- This program is free software; you can redistribute it and/or modify -- --- it under the terms of the GNU General Public License as published by -- --- the Free Software Foundation; either version 2 of the License, or -- --- (at your option) any later version. -- --- -- --- This program is distributed in the hope that it will be useful, -- --- but WITHOUT ANY WARRANTY; without even the implied warranty of -- --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- --- GNU General Public License for more details. -- --- -- --- You should have received a copy of the GNU General Public License -- --- along with this program; if not, write to the -- --- Free Software Foundation, Inc., -- --- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -- ------------------------------------------------------------------------------ - ---[[! -Lua-to-Doxygen converter - -Partially from lua2dox -http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm - -RUNNING -------- - -This script "lua2dox.lua" gets called by "gen_vimdoc.py". - -DEBUGGING/DEVELOPING ---------------------- - -1. To debug, run gen_vimdoc.py with --keep-tmpfiles: - python3 scripts/gen_vimdoc.py -t treesitter --keep-tmpfiles -2. The filtered result will be written to ./tmp-lua2dox-doc/….lua.c - -Doxygen must be on your system. You can experiment like so: - -- Run "doxygen -g" to create a default Doxyfile. -- Then alter it to let it recognise lua. Add the following line: - FILE_PATTERNS = *.lua -- Then run "doxygen". - -The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. -It only has to be good enough for doxygen to see it as legal. - -One limitation is that each line is treated separately (except for long comments). -The implication is that class and function declarations must be on the same line. - -There is hack that will insert the "missing" close paren. -The effect is that you will get the function documented, but not with the parameter list you might expect. -]] - -local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' } - -local luacats_parser = require('src/nvim/generators/luacats_grammar') - -local debug_outfile = nil --- @type string? -local debug_output = {} - ---- write to stdout ---- @param str? string -local function write(str) - if not str then - return - end - - io.write(str) - if debug_outfile then - table.insert(debug_output, str) - end -end - ---- write to stdout ---- @param str? string -local function writeln(str) - write(str) - write('\n') -end - ---- an input file buffer ---- @class StreamRead ---- @field currentLine string? ---- @field contentsLen integer ---- @field currentLineNo integer ---- @field filecontents string[] -local StreamRead = {} - ---- @return StreamRead ---- @param filename string -function StreamRead.new(filename) - assert(filename, ('invalid file: %s'):format(filename)) - -- get lines from file - -- syphon lines to our table - local filecontents = {} --- @type string[] - for line in io.lines(filename) do - filecontents[#filecontents + 1] = line - end - - return setmetatable({ - filecontents = filecontents, - contentsLen = #filecontents, - currentLineNo = 1, - }, { __index = StreamRead }) -end - --- get a line -function StreamRead:getLine() - if self.currentLine then - self.currentLine = nil - return self.currentLine - end - - -- get line - if self.currentLineNo <= self.contentsLen then - local line = self.filecontents[self.currentLineNo] - self.currentLineNo = self.currentLineNo + 1 - return line - end - - return '' -end - --- save line fragment ---- @param line_fragment string -function StreamRead:ungetLine(line_fragment) - self.currentLine = line_fragment -end - --- is it eof? -function StreamRead:eof() - return not self.currentLine and self.currentLineNo > self.contentsLen -end - --- input filter ---- @class Lua2DoxFilter -local Lua2DoxFilter = { - generics = {}, --- @type table - block_ignore = false, --- @type boolean -} -setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter }) - -function Lua2DoxFilter:reset() - self.generics = {} - self.block_ignore = false -end - ---- trim comment off end of string ---- ---- @param line string ---- @return string, string? -local function removeCommentFromLine(line) - local pos_comment = line:find('%-%-') - if not pos_comment then - return line - end - return line:sub(1, pos_comment - 1), line:sub(pos_comment) -end - ---- @param parsed luacats.Return ---- @return string -local function get_return_type(parsed) - local elems = {} --- @type string[] - for _, v in ipairs(parsed) do - local e = v.type --- @type string - if v.name then - e = e .. ' ' .. v.name --- @type string - end - elems[#elems + 1] = e - end - return '(' .. table.concat(elems, ', ') .. ')' -end - ---- @param name string ---- @return string -local function process_name(name, optional) - if optional then - name = name:sub(1, -2) --- @type string - end - return name -end - ---- @param ty string ---- @param generics table ---- @return string -local function process_type(ty, generics, optional) - -- replace generic types - for k, v in pairs(generics) do - ty = ty:gsub(k, v) --- @type string - end - - -- strip parens - ty = ty:gsub('^%((.*)%)$', '%1') - - if optional and not ty:find('nil') then - ty = ty .. '?' - end - - -- remove whitespace in unions - ty = ty:gsub('%s*|%s*', '|') - - -- replace '|nil' with '?' - ty = ty:gsub('|nil', '?') - ty = ty:gsub('nil|(.*)', '%1?') - - return '(`' .. ty .. '`)' -end - ---- @param parsed luacats.Param ---- @param generics table ---- @return string -local function process_param(parsed, generics) - local name, ty = parsed.name, parsed.type - local optional = vim.endswith(name, '?') - - return table.concat({ - '/// @param', - process_name(name, optional), - process_type(ty, generics, optional), - parsed.desc, - }, ' ') -end - ---- @param parsed luacats.Return ---- @param generics table ---- @return string -local function process_return(parsed, generics) - local ty, name --- @type string, string - if #parsed == 1 then - ty, name = parsed[1].type, parsed[1].name or '' - else - ty, name = get_return_type(parsed), '' - end - - local optional = vim.endswith(name, '?') - - return table.concat({ - '/// @return', - process_type(ty, generics, optional), - process_name(name, optional), - parsed.desc, - }, ' ') -end - ---- Processes "@…" directives in a docstring line. ---- ---- @param line string ---- @return string? -function Lua2DoxFilter:process_magic(line) - line = line:gsub('^%s+@', '@') - line = line:gsub('@package', '@private') - line = line:gsub('@nodoc', '@private') - - if self.block_ignore then - return '// gg:" ' .. line .. '"' - end - - if not vim.startswith(line, '@') then -- it's a magic comment - return '/// ' .. line - end - - local magic_split = vim.split(line, ' ', { plain = true }) - local directive = magic_split[1] - - if - vim.list_contains({ - '@cast', - '@diagnostic', - '@overload', - '@meta', - '@type', - }, directive) - then - -- Ignore LSP directives - return '// gg:"' .. line .. '"' - elseif directive == '@defgroup' or directive == '@addtogroup' then - -- Can't use '.' in defgroup, so convert to '--' - return '/// ' .. line:gsub('%.', '-dot-') - end - - if directive == '@alias' then - -- this contiguous block should be all ignored. - self.block_ignore = true - return '// gg:"' .. line .. '"' - end - - -- preprocess line before parsing - if directive == '@param' or directive == '@return' then - for _, type in ipairs(TYPES) do - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') - - line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1') - line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1') - line = line:gsub('^@return%s+.*%((' .. type .. '%?)%)', '@return %1') - end - end - - local parsed = luacats_parser:match(line) - - if not parsed then - return '/// ' .. line - end - - local kind = parsed.kind - - if kind == 'generic' then - self.generics[parsed.name] = parsed.type or 'any' - return - elseif kind == 'param' then - return process_param(parsed --[[@as luacats.Param]], self.generics) - elseif kind == 'return' then - return process_return(parsed --[[@as luacats.Return]], self.generics) - end - - error(string.format('unhandled parsed line %q: %s', line, parsed)) -end - ---- @param line string ---- @param in_stream StreamRead ---- @return string -function Lua2DoxFilter:process_block_comment(line, in_stream) - local comment_parts = {} --- @type string[] - local done --- @type boolean? - - while not done and not in_stream:eof() do - local thisComment --- @type string? - local closeSquare = line:find(']]') - if not closeSquare then -- need to look on another line - thisComment = line .. '\n' - line = in_stream:getLine() - else - thisComment = line:sub(1, closeSquare - 1) - done = true - - -- unget the tail of the line - -- in most cases it's empty. This may make us less efficient but - -- easier to program - in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2))) - end - comment_parts[#comment_parts + 1] = thisComment - end - - local comment = table.concat(comment_parts) - - if comment:sub(1, 1) == '@' then -- it's a long magic comment - return '/*' .. comment .. '*/ ' - end - - -- discard - return '/* zz:' .. comment .. '*/ ' -end - ---- @param line string ---- @return string -function Lua2DoxFilter:process_function_header(line) - local pos_fn = assert(line:find('function')) - -- we've got a function - local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8))) - - if fn:sub(1, 1) == '(' then - -- it's an anonymous function - return '// ZZ: ' .. line - end - -- fn has a name, so is interesting - - -- want to fix for iffy declarations - if fn:find('[%({]') then - -- we might have a missing close paren - if not fn:find('%)') then - fn = fn .. ' ___MissingCloseParenHere___)' - end - end - - -- Big hax - if fn:find(':') then - fn = fn:gsub(':', '.', 1) - - local paren_start = fn:find('(', 1, true) - local paren_finish = fn:find(')', 1, true) - - -- Nothing in between the parens - local comma --- @type string - if paren_finish == paren_start + 1 then - comma = '' - else - comma = ', ' - end - - fn = fn:sub(1, paren_start) .. 'self' .. comma .. fn:sub(paren_start + 1) - end - - if line:match('local') then - -- Special: tell gen_vimdoc.py this is a local function. - return 'local_function ' .. fn .. '{}' - end - - -- add vanilla function - return 'function ' .. fn .. '{}' -end - ---- @param line string ---- @param in_stream StreamRead ---- @return string? -function Lua2DoxFilter:process_line(line, in_stream) - local line_raw = line - line = vim.trim(line) - - if vim.startswith(line, '---') then - return Lua2DoxFilter:process_magic(line:sub(4)) - end - - if vim.startswith(line, '--' .. '[[') then -- it's a long comment - return Lua2DoxFilter:process_block_comment(line:sub(5), in_stream) - end - - -- Hax... I'm sorry - -- M.fun = vim.memoize(function(...) - -- -> - -- function M.fun(...) - line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') - - if line:find('^function') or line:find('^local%s+function') then - return Lua2DoxFilter:process_function_header(line) - end - - if not line:match('^local') then - local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=') - if v and v:match('%.') then - -- Special: this lets gen_vimdoc.py handle tables. - return 'table ' .. v .. '() {}' - end - end - - if #line > 0 then -- we don't know what this line means, so just comment it out - return '// zz: ' .. line - end - - return '' -end - --- Processes the file and writes filtered output to stdout. ----@param filename string -function Lua2DoxFilter:filter(filename) - local in_stream = StreamRead.new(filename) - - local last_was_magic = false - - while not in_stream:eof() do - local line = in_stream:getLine() - - local out_line = self:process_line(line, in_stream) - - if not vim.startswith(vim.trim(line), '---') then - self:reset() - end - - if out_line then - -- Ensure all magic blocks associate with some object to prevent doxygen - -- from getting confused. - if vim.startswith(out_line, '///') then - last_was_magic = true - else - if last_was_magic and out_line:match('^// zz: [^-]+') then - writeln('local_function _ignore() {}') - end - last_was_magic = false - end - writeln(out_line) - end - end -end - ---- @class TApp ---- @field timestamp string|osdate ---- @field name string ---- @field version string ---- @field copyright string ---- this application -local TApp = { - timestamp = os.date('%c %Z', os.time()), - name = 'Lua2DoX', - version = '0.2 20130128', - copyright = 'Copyright (c) Simon Dales 2012-13', -} - -setmetatable(TApp, { __index = TApp }) - -function TApp:getRunStamp() - return self.name .. ' (' .. self.version .. ') ' .. self.timestamp -end - -function TApp:getVersion() - return self.name .. ' (' .. self.version .. ') ' -end - ---main - -if arg[1] == '--help' then - writeln(TApp:getVersion()) - writeln(TApp.copyright) - writeln([[ - run as: - nvim -l scripts/lua2dox.lua - -------------- - Param: - : interprets filename - --version : show version/copyright info - --help : this help text]]) -elseif arg[1] == '--version' then - writeln(TApp:getVersion()) - writeln(TApp.copyright) -else -- It's a filter. - local filename = arg[1] - - if arg[2] == '--outdir' then - local outdir = arg[3] - if - type(outdir) ~= 'string' - or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) - then - error(('invalid --outdir: "%s"'):format(tostring(outdir))) - end - vim.fn.mkdir(outdir, 'p') - debug_outfile = string.format('%s/%s.c', outdir, vim.fs.basename(filename)) - end - - Lua2DoxFilter:filter(filename) - - -- output the tail - writeln('// #######################') - writeln('// app run:' .. TApp:getRunStamp()) - writeln('// #######################') - writeln() - - if debug_outfile then - local f = assert(io.open(debug_outfile, 'w')) - f:write(table.concat(debug_output)) - f:close() - end -end -- cgit