From 65de1a22c4d94cd8591f90255bcde72e6b385e60 Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Thu, 30 Nov 2023 07:31:22 +0800 Subject: ci(lintcommit): fix empty and period check with multiple colons (#26312) --- scripts/lintcommit.lua | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lintcommit.lua b/scripts/lintcommit.lua index d2c8601c25..a3ad4657e9 100644 --- a/scripts/lintcommit.lua +++ b/scripts/lintcommit.lua @@ -74,7 +74,11 @@ local function validate_commit(commit_message) if after_idx > vim.tbl_count(commit_split) then return [[Commit message does not include colons.]] end - local after_colon = commit_split[after_idx] + local after_colon = '' + while after_idx <= vim.tbl_count(commit_split) do + after_colon = after_colon .. commit_split[after_idx] + after_idx = after_idx + 1 + end -- Check if commit introduces a breaking change. if vim.endswith(before_colon, "!") then @@ -239,11 +243,14 @@ function M._test() ['refactor(): empty scope'] = false, ['ci( ): whitespace as scope'] = false, ['ci: period at end of sentence.'] = false, + ['ci: period: at end of sentence.'] = false, ['ci: Capitalized first word'] = false, ['ci: UPPER_CASE First Word'] = true, ['unknown: using unknown type'] = false, ['feat: foo:bar'] = true, + ['feat: :foo:bar'] = true, ['feat(something): foo:bar'] = true, + ['feat(something): :foo:bar'] = true, ['feat(:grep): read from pipe'] = true, ['feat(:grep/:make): read from pipe'] = true, ['feat(:grep): foo:bar'] = true, -- cgit From 517f0cc634b985057da5b95cf4ad659ee456a77e Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Mon, 4 Dec 2023 12:38:31 -0800 Subject: build: enable lintlua for scripts/ dir #26391 Problem: We don't enable stylua for many Lua scripts. Automating code-style is an important tool for reducing time spent on accidental (non-essential) complexity. Solution: - Enable lintlua for `scripts/` directory. - Specify `call_parentheses = "Input"`, we should allow kwargs-style function invocations. --- scripts/bump_deps.lua | 53 +++---- scripts/gen_eval_files.lua | 72 +++++----- scripts/gen_filetype.lua | 102 +++++++------- scripts/gen_help_html.lua | 341 ++++++++++++++++++++++++++++++--------------- scripts/lintcommit.lua | 67 ++++----- scripts/lua2dox.lua | 55 ++++---- scripts/vimpatch.lua | 10 +- 7 files changed, 412 insertions(+), 288 deletions(-) (limited to 'scripts') diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua index 076ad374cf..c5294893e0 100755 --- a/scripts/bump_deps.lua +++ b/scripts/bump_deps.lua @@ -138,9 +138,10 @@ local function get_archive_info(repo, ref) 'Failed to download archive from GitHub' ) - local shacmd = (vim.fn.executable('sha256sum') == 1 - and{ 'sha256sum', archive_path } - or { 'shasum', '-a', '256', archive_path }) + local shacmd = ( + vim.fn.executable('sha256sum') == 1 and { 'sha256sum', archive_path } + or { 'shasum', '-a', '256', archive_path } + ) local archive_sha = run(shacmd):gmatch('%w+')() return { url = archive_url, sha = archive_sha } end @@ -152,18 +153,7 @@ local function write_cmakelists_line(symbol, kind, value) 'sed', '-i', '-e', - 's/' - .. symbol - .. '_' - .. kind - .. '.*$' - .. '/' - .. symbol - .. '_' - .. kind - .. ' ' - .. value - .. '/', + 's/' .. symbol .. '_' .. kind .. '.*$' .. '/' .. symbol .. '_' .. kind .. ' ' .. value .. '/', deps_file, }, 'Failed to write ' .. deps_file) end @@ -203,16 +193,13 @@ local function update_cmakelists(dependency, archive, comment) p('Updating ' .. dependency.name .. ' to ' .. archive.url .. '\n') write_cmakelists_line(dependency.symbol, 'URL', archive.url:gsub('/', '\\/')) write_cmakelists_line(dependency.symbol, 'SHA256', archive.sha) - run_die( - { - 'git', - 'commit', - deps_file, - '-m', - commit_prefix .. 'bump ' .. dependency.name .. ' to ' .. comment, - }, - 'git failed to commit' - ) + run_die({ + 'git', + 'commit', + deps_file, + '-m', + commit_prefix .. 'bump ' .. dependency.name .. ' to ' .. comment, + }, 'git failed to commit') end local function verify_cmakelists_committed() @@ -318,9 +305,9 @@ function M.commit(dependency_name, commit) end function M.version(dependency_name, version) - vim.validate{ - dependency_name={dependency_name,'s'}, - version={version,'s'}, + vim.validate { + dependency_name = { dependency_name, 's' }, + version = { version, 's' }, } local dependency = assert(get_dependency(dependency_name)) verify_cmakelists_committed() @@ -384,7 +371,7 @@ function M.submit_pr() end local function usage() - local this_script = _G.arg[0]:match("[^/]*.lua$") + local this_script = _G.arg[0]:match('[^/]*.lua$') print(([=[ Bump Nvim dependencies @@ -421,13 +408,13 @@ local function parseargs() elseif _G.arg[i] == '--pr' then args.pr = true elseif _G.arg[i] == '--branch' then - args.branch = _G.arg[i+1] + args.branch = _G.arg[i + 1] elseif _G.arg[i] == '--dep' then - args.dep = _G.arg[i+1] + args.dep = _G.arg[i + 1] elseif _G.arg[i] == '--version' then - args.version = _G.arg[i+1] + args.version = _G.arg[i + 1] elseif _G.arg[i] == '--commit' then - args.commit = _G.arg[i+1] + args.commit = _G.arg[i + 1] elseif _G.arg[i] == '--head' then args.head = true end diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index e331dd996e..dddc7d000c 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -1,7 +1,7 @@ -- Generator for various vimdoc and Lua type files local DEP_API_METADATA = 'build/api_metadata.mpack' -local DEP_API_DOC = 'runtime/doc/api.mpack' +local DEP_API_DOC = 'runtime/doc/api.mpack' --- @class vim.api.metadata --- @field name string @@ -302,7 +302,7 @@ local function get_api_keysets_meta() for _, k in ipairs(keysets) do local params = {} for _, key in ipairs(k.keys) do - table.insert(params, {key..'?', api_type(k.types[key] or 'any')}) + table.insert(params, { key .. '?', api_type(k.types[key] or 'any') }) end ret[k.name] = { signature = 'NA', @@ -396,7 +396,7 @@ local function render_sig_and_tag(name, fun, write) local tag = table.concat(tags, ' ') local siglen = #fun.signature - local conceal_offset = 2*(#tags - 1) + local conceal_offset = 2 * (#tags - 1) local tag_pad_len = math.max(1, 80 - #tag + conceal_offset) if siglen + #tag > 80 then @@ -473,7 +473,7 @@ local function render_option_default(d, vimdoc) end end - if dt == "" or dt == nil or type(dt) == 'function' then + if dt == '' or dt == nil or type(dt) == 'function' then dt = d.meta end @@ -481,22 +481,22 @@ local function render_option_default(d, vimdoc) if not vimdoc then v = vim.inspect(dt) --[[@as string]] else - v = type(dt) == 'string' and '"'..dt..'"' or tostring(dt) + v = type(dt) == 'string' and '"' .. dt .. '"' or tostring(dt) end --- @type table local envvars = { TMPDIR = false, VIMRUNTIME = false, - XDG_CONFIG_HOME = vim.env.HOME..'/.local/config', - XDG_DATA_HOME = vim.env.HOME..'/.local/share', - XDG_STATE_HOME = vim.env.HOME..'/.local/state', + XDG_CONFIG_HOME = vim.env.HOME .. '/.local/config', + XDG_DATA_HOME = vim.env.HOME .. '/.local/share', + XDG_STATE_HOME = vim.env.HOME .. '/.local/state', } for name, default in pairs(envvars) do local value = vim.env[name] or default if value then - v = v:gsub(vim.pesc(value), '$'..name) + v = v:gsub(vim.pesc(value), '$' .. name) end end @@ -509,26 +509,26 @@ end local function render_option_meta(_f, opt, write) write('') for _, l in ipairs(split(norm_text(opt.desc))) do - write('--- '..l) + write('--- ' .. l) end - write('--- @type '..OPTION_TYPES[opt.type]) - write('vim.o.'..opt.full_name..' = '..render_option_default(opt.defaults)) + write('--- @type ' .. OPTION_TYPES[opt.type]) + write('vim.o.' .. opt.full_name .. ' = ' .. render_option_default(opt.defaults)) if opt.abbreviation then - write('vim.o.'..opt.abbreviation..' = vim.o.'..opt.full_name) + write('vim.o.' .. opt.abbreviation .. ' = vim.o.' .. opt.full_name) end for _, s in pairs { - {'wo', 'window'}, - {'bo', 'buffer'}, - {'go', 'global'}, + { 'wo', 'window' }, + { 'bo', 'buffer' }, + { 'go', 'global' }, } do local id, scope = s[1], s[2] if vim.list_contains(opt.scope, scope) or (id == 'go' and #opt.scope > 1) then - local pfx = 'vim.'..id..'.' - write(pfx..opt.full_name..' = vim.o.'..opt.full_name) + local pfx = 'vim.' .. id .. '.' + write(pfx .. opt.full_name .. ' = vim.o.' .. opt.full_name) if opt.abbreviation then - write(pfx..opt.abbreviation..' = '..pfx..opt.full_name) + write(pfx .. opt.abbreviation .. ' = ' .. pfx .. opt.full_name) end end end @@ -541,14 +541,14 @@ local function scope_to_doc(s) global = 'global', buffer = 'local to buffer', window = 'local to window', - tab = 'local to tab page' + tab = 'local to tab page', } if #s == 1 then return m[s[1]] end assert(s[1] == 'global') - return 'global or '..m[s[2]]..' |global-local|' + return 'global or ' .. m[s[2]] .. ' |global-local|' end -- @param o vim.option_meta @@ -602,23 +602,23 @@ local function build_option_tags(opt) --- @type string[] local tags = { opt.full_name } - tags[#tags+1] = opt.abbreviation + tags[#tags + 1] = opt.abbreviation if opt.type == 'bool' then for i = 1, #tags do - tags[#tags+1] = 'no'..tags[i] + tags[#tags + 1] = 'no' .. tags[i] end end for i, t in ipairs(tags) do - tags[i] = "'"..t.."'" + tags[i] = "'" .. t .. "'" end for _, t in ipairs(opt.tags or {}) do - tags[#tags+1] = t + tags[#tags + 1] = t end for i, t in ipairs(tags) do - tags[i] = "*"..t.."*" + tags[i] = '*' .. t .. '*' end return tags @@ -630,10 +630,10 @@ end local function render_option_doc(_f, opt, write) local tags = build_option_tags(opt) local tag_str = table.concat(tags, ' ') - local conceal_offset = 2*(#tags - 1) + local conceal_offset = 2 * (#tags - 1) local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) -- local pad = string.rep(' ', 80 - #tag_str + conceal_offset) - write(tag_pad..tag_str) + write(tag_pad .. tag_str) local name_str --- @type string if opt.abbreviation then @@ -649,19 +649,19 @@ local function render_option_doc(_f, opt, write) if opt.defaults.doc then local deflen = #string.format('%s%s%s (', name_str, pad, otype) --- @type string - v = v:gsub('\n', '\n'..string.rep(' ', deflen - 2)) + v = v:gsub('\n', '\n' .. string.rep(' ', deflen - 2)) end write(string.format('%s%s%s\t(default %s)', name_str, pad, otype, v)) else write(string.format('%s\t%s', name_str, otype)) end - write('\t\t\t'..scope_to_doc(opt.scope)..scope_more_doc(opt)) + write('\t\t\t' .. scope_to_doc(opt.scope) .. scope_more_doc(opt)) for _, l in ipairs(split(opt.desc)) do if l == '<' or l:match('^<%s') then write(l) else - write('\t'..l:gsub('\\<', '<')) + write('\t' .. l:gsub('\\<', '<')) end end end @@ -751,21 +751,21 @@ local CONFIG = { header = { '' }, from = 'A jump table for the options with a short description can be found at |Q_op|.', footer = { - ' vim:tw=78:ts=8:noet:ft=help:norl:' + ' vim:tw=78:ts=8:noet:ft=help:norl:', }, funcs = get_option_meta, render = render_option_doc, - } + }, } --- @param elem nvim.gen_eval_files.elem local function render(elem) - print('Rendering '..elem.path) - local from_lines = {} --- @type string[] + print('Rendering ' .. elem.path) + local from_lines = {} --- @type string[] local from = elem.from if from then for line in io.lines(elem.path) do - from_lines[#from_lines+1] = line + from_lines[#from_lines + 1] = line if line:match(from) then break end diff --git a/scripts/gen_filetype.lua b/scripts/gen_filetype.lua index 42478a1082..18b53f1ea4 100644 --- a/scripts/gen_filetype.lua +++ b/scripts/gen_filetype.lua @@ -8,18 +8,18 @@ if do_not_run then return end -local filetype_vim = "runtime/filetype.vim" -local filetype_lua = "runtime/lua/vim/filetype.lua" +local filetype_vim = 'runtime/filetype.vim' +local filetype_lua = 'runtime/lua/vim/filetype.lua' local keywords = { - ["for"] = true, - ["or"] = true, - ["and"] = true, - ["end"] = true, - ["do"] = true, - ["if"] = true, - ["while"] = true, - ["repeat"] = true, + ['for'] = true, + ['or'] = true, + ['and'] = true, + ['end'] = true, + ['do'] = true, + ['if'] = true, + ['while'] = true, + ['repeat'] = true, } local sections = { @@ -28,42 +28,42 @@ local sections = { pattern = { str = {}, func = {} }, } -local specialchars = "%*%?\\%$%[%]%{%}" +local specialchars = '%*%?\\%$%[%]%{%}' local function add_pattern(pat, ft) local ok = true -- Patterns that start or end with { or } confuse splitting on commas and make parsing harder, so just skip those - if not string.find(pat, "^%{") and not string.find(pat, "%}$") then - for part in string.gmatch(pat, "[^,]+") do - if not string.find(part, "[" .. specialchars .. "]") then - if type(ft) == "string" then + if not string.find(pat, '^%{') and not string.find(pat, '%}$') then + for part in string.gmatch(pat, '[^,]+') do + if not string.find(part, '[' .. specialchars .. ']') then + if type(ft) == 'string' then sections.filename.str[part] = ft else sections.filename.func[part] = ft end - elseif string.match(part, "^%*%.[^%./" .. specialchars .. "]+$") then - if type(ft) == "string" then + elseif string.match(part, '^%*%.[^%./' .. specialchars .. ']+$') then + if type(ft) == 'string' then sections.extension.str[part:sub(3)] = ft else sections.extension.func[part:sub(3)] = ft end else - if string.match(part, "^%*/[^" .. specialchars .. "]+$") then + if string.match(part, '^%*/[^' .. specialchars .. ']+$') then -- For patterns matching */some/pattern we want to easily match files -- with path /some/pattern, so include those in filename detection - if type(ft) == "string" then + if type(ft) == 'string' then sections.filename.str[part:sub(2)] = ft else sections.filename.func[part:sub(2)] = ft end end - if string.find(part, "^[%w-_.*?%[%]/]+$") then - local p = part:gsub("%.", "%%."):gsub("%*", ".*"):gsub("%?", ".") + if string.find(part, '^[%w-_.*?%[%]/]+$') then + local p = part:gsub('%.', '%%.'):gsub('%*', '.*'):gsub('%?', '.') -- Insert into array to maintain order rather than setting -- key-value directly - if type(ft) == "string" then + if type(ft) == 'string' then sections.pattern.str[p] = ft else sections.pattern.func[p] = ft @@ -80,14 +80,16 @@ end local function parse_line(line) local pat, ft - pat, ft = line:match("^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)") + pat, ft = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)') if pat then return add_pattern(pat, ft) else local func - pat, func = line:match("^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)") + pat, func = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)') if pat then - return add_pattern(pat, function() return func end) + return add_pattern(pat, function() + return func + end) end end end @@ -95,12 +97,12 @@ end local unparsed = {} local full_line for line in io.lines(filetype_vim) do - local cont = string.match(line, "^%s*\\%s*(.*)$") + local cont = string.match(line, '^%s*\\%s*(.*)$') if cont then - full_line = full_line .. " " .. cont + full_line = full_line .. ' ' .. cont else if full_line then - if not parse_line(full_line) and string.find(full_line, "^%s*au%a* Buf") then + if not parse_line(full_line) and string.find(full_line, '^%s*au%a* Buf') then table.insert(unparsed, full_line) end end @@ -109,40 +111,46 @@ for line in io.lines(filetype_vim) do end if #unparsed > 0 then - print("Failed to parse the following patterns:") + print('Failed to parse the following patterns:') for _, v in ipairs(unparsed) do print(v) end end local function add_item(indent, key, ft) - if type(ft) == "string" then - if string.find(key, "%A") or keywords[key] then - key = string.format("[\"%s\"]", key) + if type(ft) == 'string' then + if string.find(key, '%A') or keywords[key] then + key = string.format('["%s"]', key) end return string.format([[%s%s = "%s",]], indent, key, ft) - elseif type(ft) == "function" then + elseif type(ft) == 'function' then local func = ft() - if string.find(key, "%A") or keywords[key] then - key = string.format("[\"%s\"]", key) + if string.find(key, '%A') or keywords[key] then + key = string.format('["%s"]', key) end -- Right now only a single argument is supported, which covers -- everything in filetype.vim as of this writing - local arg = string.match(func, "%((.*)%)$") - func = string.gsub(func, "%(.*$", "") - if arg == "" then + local arg = string.match(func, '%((.*)%)$') + func = string.gsub(func, '%(.*$', '') + if arg == '' then -- Function with no arguments, call the function directly return string.format([[%s%s = function() vim.fn["%s"]() end,]], indent, key, func) elseif string.match(arg, [[^(["']).*%1$]]) then -- String argument - if func == "s:StarSetf" then + if func == 's:StarSetf' then return string.format([[%s%s = starsetf(%s),]], indent, key, arg) else return string.format([[%s%s = function() vim.fn["%s"](%s) end,]], indent, key, func, arg) end - elseif string.find(arg, "%(") then + elseif string.find(arg, '%(') then -- Function argument - return string.format([[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]], indent, key, func, arg) + return string.format( + [[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]], + indent, + key, + func, + arg + ) else assert(false, arg) end @@ -153,7 +161,7 @@ do local lines = {} local start = false for line in io.lines(filetype_lua) do - if line:match("^%s+-- END [A-Z]+$") then + if line:match('^%s+-- END [A-Z]+$') then start = false end @@ -161,14 +169,14 @@ do table.insert(lines, line) end - local indent, section = line:match("^(%s+)-- BEGIN ([A-Z]+)$") + local indent, section = line:match('^(%s+)-- BEGIN ([A-Z]+)$') if section then start = true local t = sections[string.lower(section)] local sorted = {} for k, v in pairs(t.str) do - table.insert(sorted, {[k] = v}) + table.insert(sorted, { [k] = v }) end table.sort(sorted, function(a, b) @@ -182,7 +190,7 @@ do sorted = {} for k, v in pairs(t.func) do - table.insert(sorted, {[k] = v}) + table.insert(sorted, { [k] = v }) end table.sort(sorted, function(a, b) @@ -195,7 +203,7 @@ do end end end - local f = io.open(filetype_lua, "w") - f:write(table.concat(lines, "\n") .. "\n") + local f = io.open(filetype_lua, 'w') + f:write(table.concat(lines, '\n') .. '\n') f:close() end diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 633207e018..9369711b0f 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -60,26 +60,26 @@ local new_layout = { -- TODO: These known invalid |links| require an update to the relevant docs. local exclude_invalid = { - ["'string'"] = "eval.txt", + ["'string'"] = 'eval.txt', Query = 'treesitter.txt', matchit = 'vim_diff.txt', - ["set!"] = "treesitter.txt", + ['set!'] = 'treesitter.txt', } -- False-positive "invalid URLs". local exclude_invalid_urls = { - ["http://"] = "usr_23.txt", - ["http://."] = "usr_23.txt", - ["http://aspell.net/man-html/Affix-Compression.html"] = "spell.txt", - ["http://aspell.net/man-html/Phonetic-Code.html"] = "spell.txt", - ["http://canna.sourceforge.jp/"] = "mbyte.txt", - ["http://gnuada.sourceforge.net"] = "ft_ada.txt", - ["http://lua-users.org/wiki/StringLibraryTutorial"] = "lua.txt", - ["http://michael.toren.net/code/"] = "pi_tar.txt", - ["http://papp.plan9.de"] = "syntax.txt", - ["http://wiki.services.openoffice.org/wiki/Dictionaries"] = "spell.txt", - ["http://www.adapower.com"] = "ft_ada.txt", - ["http://www.jclark.com/"] = "quickfix.txt", + ['http://'] = 'usr_23.txt', + ['http://.'] = 'usr_23.txt', + ['http://aspell.net/man-html/Affix-Compression.html'] = 'spell.txt', + ['http://aspell.net/man-html/Phonetic-Code.html'] = 'spell.txt', + ['http://canna.sourceforge.jp/'] = 'mbyte.txt', + ['http://gnuada.sourceforge.net'] = 'ft_ada.txt', + ['http://lua-users.org/wiki/StringLibraryTutorial'] = 'lua.txt', + ['http://michael.toren.net/code/'] = 'pi_tar.txt', + ['http://papp.plan9.de'] = 'syntax.txt', + ['http://wiki.services.openoffice.org/wiki/Dictionaries'] = 'spell.txt', + ['http://www.adapower.com'] = 'ft_ada.txt', + ['http://www.jclark.com/'] = 'quickfix.txt', } -- Deprecated, brain-damaged files that I don't care about. @@ -98,19 +98,18 @@ local function tofile(fname, text) end local function html_esc(s) - return s:gsub( - '&', '&'):gsub( - '<', '<'):gsub( - '>', '>') + return s:gsub('&', '&'):gsub('<', '<'):gsub('>', '>') end local function url_encode(s) -- Credit: tpope / vim-unimpaired -- NOTE: these chars intentionally *not* escaped: ' ( ) - return vim.fn.substitute(vim.fn.iconv(s, 'latin1', 'utf-8'), + return vim.fn.substitute( + vim.fn.iconv(s, 'latin1', 'utf-8'), [=[[^A-Za-z0-9()'_.~-]]=], [=[\="%".printf("%02X",char2nr(submatch(0)))]=], - 'g') + 'g' + ) end local function expandtabs(s) @@ -131,7 +130,7 @@ local function to_heading_tag(text) end local function basename_noext(f) - return vim.fs.basename(f:gsub('%.txt', '')) + return vim.fs.basename(f:gsub('%.txt', '')) end local function is_blank(s) @@ -151,7 +150,7 @@ local function fix_url(url) local removed_chars = '' local fixed_url = url -- Remove up to one of each char from end of the URL, in this order. - for _, c in ipairs({ '.', ')', }) do + for _, c in ipairs({ '.', ')' }) do if fixed_url:sub(-1) == c then removed_chars = c .. removed_chars fixed_url = fixed_url:sub(1, -2) @@ -162,7 +161,7 @@ end --- Checks if a given line is a "noise" line that doesn't look good in HTML form. local function is_noise(line, noise_lines) - if ( + if -- First line is always noise. (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0) or line:find('Type .*gO.* to see the table of contents') @@ -177,7 +176,7 @@ local function is_noise(line, noise_lines) or line:find('^%s*vim?%:.*ft=help') or line:find('^%s*vim?%:.*filetype=help') or line:find('[*>]local%-additions[*<]') - ) then + then -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) table.insert(noise_lines or {}, line) return true @@ -188,28 +187,32 @@ end --- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. local function get_bug_url_vimdoc(fname, to_fname, sample_text) local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ('https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' - ..vim.fs.basename(fname) - ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' - ..this_url - ..'%0D%0DContext%3A%0D%0D%60%60%60%0D' - ..url_encode(sample_text) - ..'%0D%60%60%60') + local bug_url = ( + 'https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + .. vim.fs.basename(fname) + .. '+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' + .. this_url + .. '%0D%0DContext%3A%0D%0D%60%60%60%0D' + .. url_encode(sample_text) + .. '%0D%60%60%60' + ) return bug_url end --- Creates a github issue URL at neovim/neovim with prefilled content. local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ('https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' - ..vim.fs.basename(fname) - ..'+&body=%60gen_help_html.lua%60+problem+at%3A+' - ..this_url - ..'%0D' - ..(token_name and '+unhandled+token%3A+%60'..token_name..'%60' or '') - ..'%0DContext%3A%0D%0D%60%60%60%0D' - ..url_encode(sample_text) - ..'%0D%60%60%60') + local bug_url = ( + 'https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' + .. vim.fs.basename(fname) + .. '+&body=%60gen_help_html.lua%60+problem+at%3A+' + .. this_url + .. '%0D' + .. (token_name and '+unhandled+token%3A+%60' .. token_name .. '%60' or '') + .. '%0DContext%3A%0D%0D%60%60%60%0D' + .. url_encode(sample_text) + .. '%0D%60%60%60' + ) return bug_url end @@ -274,9 +277,11 @@ end local function get_tagname(node, bufnr) local text = vim.treesitter.get_node_text(node, bufnr) - local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(text) or text - local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" - local helppage = get_helppage(helpfile) -- "api.html" + local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') + and ("'%s'"):format(text) + or text + local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" return helppage, tag end @@ -295,11 +300,9 @@ local function ignore_parse_error(fname, s) if ignore_errors[vim.fs.basename(fname)] then return true end - return ( - -- Ignore parse errors for unclosed tag. - -- This is common in vimdocs and is treated as plaintext by :help. - s:find("^[`'|*]") - ) + -- Ignore parse errors for unclosed tag. + -- This is common in vimdocs and is treated as plaintext by :help. + return s:find("^[`'|*]") end local function has_ancestor(node, ancestor_name) @@ -377,10 +380,11 @@ local function visit_validate(root, level, lang_tree, opt, stats) -- Flatten the sample text to a single, truncated line. sample_text = vim.trim(sample_text):gsub('[\t\n]', ' '):sub(1, 80) table.insert(stats.parse_errors, sample_text) - elseif (node_name == 'word' or node_name == 'uppercase_name') - and (not vim.tbl_contains({'codespan', 'taglink', 'tag'}, parent)) + elseif + (node_name == 'word' or node_name == 'uppercase_name') + and (not vim.tbl_contains({ 'codespan', 'taglink', 'tag' }, parent)) then - local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. + local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. if spell_dict[text_nopunct] then invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} invalid_spelling[text_nopunct][vim.fs.basename(opt.fname)] = node_text(root:parent()) @@ -399,7 +403,7 @@ local function fix_tab_after_conceal(text, next_node_text) -- Vim tabs take into account the two concealed characters even though they -- are invisible, so we need to add back in the two spaces if this is -- followed by a tab to make the tab alignment to match Vim's behavior. - if string.sub(next_node_text,1,1) == '\t' then + if string.sub(next_node_text, 1, 1) == '\t' then text = text .. ' ' end return text @@ -411,9 +415,15 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local node_name = (root.named and root:named()) and root:type() or nil -- Previous sibling kind (string). - local prev = root:prev_sibling() and (root:prev_sibling().named and root:prev_sibling():named()) and root:prev_sibling():type() or nil + local prev = root:prev_sibling() + and (root:prev_sibling().named and root:prev_sibling():named()) + and root:prev_sibling():type() + or nil -- Next sibling kind (string). - local next_ = root:next_sibling() and (root:next_sibling().named and root:next_sibling():named()) and root:next_sibling():type() or nil + local next_ = root:next_sibling() + and (root:next_sibling().named and root:next_sibling():named()) + and root:next_sibling():type() + or nil -- Parent kind (string). local parent = root:parent() and root:parent():type() or nil local text = '' @@ -450,7 +460,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) trimmed = trim(text) end - if node_name == 'help_file' then -- root node + if node_name == 'help_file' then -- root node return text elseif node_name == 'url' then local fixed_url, removed_chars = fix_url(trimmed) @@ -459,18 +469,22 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return text elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then if is_noise(text, stats.noise_lines) then - return '' -- Discard common "noise" lines. + return '' -- Discard common "noise" lines. end -- Remove "===" and tags from ToC text. local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) -- Use the first *tag* node as the heading anchor, if any. local tagnode = first(root, 'tag') -- Use the *tag* as the heading anchor id, if possible. - local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname) + local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) + or to_heading_tag(hname) if node_name == 'h1' or #headings == 0 then table.insert(headings, { name = hname, subheadings = {}, tag = tagname }) else - table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, tag = tagname }) + table.insert( + headings[#headings].subheadings, + { name = hname, subheadings = {}, tag = tagname } + ) end local el = node_name == 'h1' and 'h2' or 'h3' return ('<%s id="%s" class="help-heading">%s\n'):format(el, tagname, text, el) @@ -490,11 +504,16 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end return string.format('
\n%s\n
\n', text) elseif node_name == 'line' then - if (parent ~= 'codeblock' or parent ~= 'code') and (is_blank(text) or is_noise(text, stats.noise_lines)) then - return '' -- Discard common "noise" lines. + if + (parent ~= 'codeblock' or parent ~= 'code') + and (is_blank(text) or is_noise(text, stats.noise_lines)) + then + return '' -- Discard common "noise" lines. end -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. - local div = opt.old and root:child(0) and vim.list_contains({'column_heading', 'h1', 'h2', 'h3'}, root:child(0):type()) + local div = opt.old + and root:child(0) + and vim.list_contains({ 'column_heading', 'h1', 'h2', 'h3' }, root:child(0):type()) return string.format('%s%s', div and trim(text) or text, div and '' or '\n') elseif node_name == 'line_li' then local sib = root:prev_sibling() @@ -520,12 +539,17 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if ignored then return text end - local s = ('%s%s'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) + local s = ('%s%s'):format( + ws(), + helppage, + url_encode(tagname), + html_esc(tagname) + ) if opt.old and node_name == 'taglink' then s = fix_tab_after_conceal(s, node_text(root:next_sibling())) end return s - elseif vim.list_contains({'codespan', 'keycode'}, node_name) then + elseif vim.list_contains({ 'codespan', 'keycode' }, node_name) then if root:has_error() then return text end @@ -541,24 +565,28 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) elseif node_name == 'language' then language = node_text(root) return '' - elseif node_name == 'code' then -- Highlighted codeblock (child). + elseif node_name == 'code' then -- Highlighted codeblock (child). if is_blank(text) then return '' end local code if language then - code = ('
%s
'):format(language,trim(trim_indent(text), 2)) + code = ('
%s
'):format( + language, + trim(trim_indent(text), 2) + ) language = nil else code = ('
%s
'):format(trim(trim_indent(text), 2)) end return code - elseif node_name == 'tag' then -- anchor + elseif node_name == 'tag' then -- anchor if root:has_error() then return text end - local in_heading = vim.list_contains({'h1', 'h2', 'h3'}, parent) - local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' + local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent) + local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' + or 'help-tag' local tagname = node_text(root:child(1), false) if vim.tbl_count(stats.first_tags) < 2 then -- Force the first 2 tags in the doc to be anchored at the main heading. @@ -567,14 +595,29 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end local el = in_heading and 'span' or 'code' local encoded_tagname = url_encode(tagname) - local s = ('%s<%s id="%s" class="%s">%s'):format(ws(), el, encoded_tagname, cssclass, encoded_tagname, trimmed, el) + local s = ('%s<%s id="%s" class="%s">%s'):format( + ws(), + el, + encoded_tagname, + cssclass, + encoded_tagname, + trimmed, + el + ) if opt.old then - s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) end if in_heading and prev ~= 'tag' then -- Don't set "id", let the heading use the tag as its "id" (used by search engines). - s = ('%s<%s class="%s">%s'):format(ws(), el, cssclass, encoded_tagname, trimmed, el) + s = ('%s<%s class="%s">%s'):format( + ws(), + el, + cssclass, + encoded_tagname, + trimmed, + el + ) -- Start the container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. --

foo bartag1 tag2

@@ -593,11 +636,17 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) return ('%s'):format( - get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) - else -- Unknown token. + get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), + trimmed + ) + else -- Unknown token. local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' return ('%s'):format( - node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) + node_name, + get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), + trimmed + ), + ('unknown-token:"%s"'):format(node_name) end end @@ -605,9 +654,11 @@ local function get_helpfiles(include) local dir = './build/runtime/doc' local rv = {} for f, type in vim.fs.dir(dir) do - if (vim.endswith(f, '.txt') - and type == 'file' - and (not include or vim.list_contains(include, f))) then + if + vim.endswith(f, '.txt') + and type == 'file' + and (not include or vim.list_contains(include, f)) + then local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') table.insert(rv, fullpath) end @@ -633,7 +684,7 @@ end --- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system. local function ensure_runtimepath() if not vim.o.runtimepath:find('build/lib/nvim/') then - vim.cmd[[set runtimepath^=./build/lib/nvim/]] + vim.cmd [[set runtimepath^=./build/lib/nvim/]] end end @@ -645,11 +696,11 @@ end local function parse_buf(fname, parser_path) local buf if type(fname) == 'string' then - vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. + vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename. buf = vim.api.nvim_get_current_buf() else buf = fname - vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. + vim.cmd('sbuffer ' .. tostring(fname)) -- Buffer number. end if parser_path then vim.treesitter.language.add('vimdoc', { path = parser_path }) @@ -671,7 +722,7 @@ local function validate_one(fname, parser_path) } local lang_tree, buf = parse_buf(fname, parser_path) for _, tree in ipairs(lang_tree:trees()) do - visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname, }, stats) + visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats) end lang_tree:destroy() vim.cmd.close() @@ -690,10 +741,10 @@ local function gen_one(fname, to_fname, old, commit, parser_path) local stats = { noise_lines = {}, parse_errors = {}, - first_tags = {}, -- Track the first few tags in doc. + first_tags = {}, -- Track the first few tags in doc. } local lang_tree, buf = parse_buf(fname, parser_path) - local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. + local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. local title = to_titlecase(basename_noext(fname)) local html = ([[ @@ -777,9 +828,17 @@ local function gen_one(fname, to_fname, old, commit, parser_path) local main = '' for _, tree in ipairs(lang_tree:trees()) do - main = main .. (visit_node(tree:root(), 0, tree, headings, - { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1, }, - stats)) + main = main + .. ( + visit_node( + tree:root(), + 0, + tree, + headings, + { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1 }, + stats + ) + ) end main = ([[ @@ -809,7 +868,14 @@ local function gen_one(fname, to_fname, old, commit, parser_path)
%s - ]]):format(logo_svg, stats.first_tags[2] or '', stats.first_tags[1] or '', title, vim.fs.basename(fname), main) + ]]):format( + logo_svg, + stats.first_tags[2] or '', + stats.first_tags[1] or '', + title, + vim.fs.basename(fname), + main + ) local toc = [[
@@ -819,13 +885,16 @@ local function gen_one(fname, to_fname, old, commit, parser_path)
]] - local n = 0 -- Count of all headings + subheadings. - for _, h1 in ipairs(headings) do n = n + 1 + #h1.subheadings end + local n = 0 -- Count of all headings + subheadings. + for _, h1 in ipairs(headings) do + n = n + 1 + #h1.subheadings + end for _, h1 in ipairs(headings) do toc = toc .. ('
%s\n'):format(h1.tag, h1.name) - if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. + if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. for _, h2 in ipairs(h1.subheadings) do - toc = toc .. ('
%s
\n'):format(h2.tag, h2.name) + toc = toc + .. ('
%s
\n'):format(h2.tag, h2.name) end end toc = toc .. '
' @@ -859,11 +928,16 @@ local function gen_one(fname, to_fname, old, commit, parser_path) ]]):format( - os.date('%Y-%m-%d %H:%M'), commit, commit:sub(1, 7), #stats.parse_errors, bug_link, - html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines) + os.date('%Y-%m-%d %H:%M'), + commit, + commit:sub(1, 7), + #stats.parse_errors, + bug_link, + html_esc(table.concat(stats.noise_lines, '\n')), + #stats.noise_lines + ) - html = ('%s%s%s
\n%s\n\n'):format( - html, main, toc, footer) + html = ('%s%s%s\n%s\n\n'):format(html, main, toc, footer) vim.cmd('q!') lang_tree:destroy() return html, stats @@ -1038,9 +1112,15 @@ function M._test() helpfiles = get_helpfiles() local function ok(cond, expected, actual) - assert((not expected and not actual) or (expected and actual), 'if "expected" is given, "actual" is also required') + assert( + (not expected and not actual) or (expected and actual), + 'if "expected" is given, "actual" is also required' + ) if expected then - return assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual))) + return assert( + cond, + ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual)) + ) else return assert(cond) end @@ -1050,7 +1130,11 @@ function M._test() end ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) - ok(vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), tagmap['vim.diagnostic.set()'], 'diagnostic.txt') + ok( + vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), + tagmap['vim.diagnostic.set()'], + 'diagnostic.txt' + ) ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt') ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]])) ok(is_noise([[ NVIM REFERENCE MANUAL by Thiago de Arruda ]])) @@ -1060,7 +1144,10 @@ function M._test() eq(1, get_indent(' a')) eq(2, get_indent(' a\n b\n c\n')) eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) - eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + eq( + 'a\n \n b\n c\n d\n e\n', + trim_indent(' a\n \n b\n c\n d\n e\n') + ) local fixed_url, removed_chars = fix_url('https://example.com).') eq('https://example.com', fixed_url) @@ -1093,12 +1180,24 @@ end --- --- @returns info dict function M.gen(help_dir, to_dir, include, commit, parser_path) - vim.validate{ - help_dir={help_dir, function(d) return vim.fn.isdirectory(vim.fn.expand(d)) == 1 end, 'valid directory'}, - to_dir={to_dir, 's'}, - include={include, 't', true}, - commit={commit, 's', true}, - parser_path={parser_path, function(f) return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 end, 'valid vimdoc.{so,dll} filepath'}, + vim.validate { + help_dir = { + help_dir, + function(d) + return vim.fn.isdirectory(vim.fn.expand(d)) == 1 + end, + 'valid directory', + }, + to_dir = { to_dir, 's' }, + include = { include, 't', true }, + commit = { commit, 's', true }, + parser_path = { + parser_path, + function(f) + return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 + end, + 'valid vimdoc.{so,dll} filepath', + }, } local err_count = 0 @@ -1117,7 +1216,13 @@ function M.gen(help_dir, to_dir, include, commit, parser_path) local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?', parser_path) tofile(to_fname, html) - print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) + print( + ('generated (%-4s errors): %-15s => %s'):format( + #stats.parse_errors, + helpfile, + vim.fs.basename(to_fname) + ) + ) err_count = err_count + #stats.parse_errors end print(('generated %d html pages'):format(#helpfiles)) @@ -1139,10 +1244,22 @@ end -- -- @returns results dict function M.validate(help_dir, include, parser_path) - vim.validate{ - help_dir={help_dir, function(d) return vim.fn.isdirectory(vim.fn.expand(d)) == 1 end, 'valid directory'}, - include={include, 't', true}, - parser_path={parser_path, function(f) return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 end, 'valid vimdoc.{so,dll} filepath'}, + vim.validate { + help_dir = { + help_dir, + function(d) + return vim.fn.isdirectory(vim.fn.expand(d)) == 1 + end, + 'valid directory', + }, + include = { include, 't', true }, + parser_path = { + parser_path, + function(f) + return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 + end, + 'valid vimdoc.{so,dll} filepath', + }, } local err_count = 0 local files_to_errors = {} @@ -1157,7 +1274,9 @@ function M.validate(help_dir, include, parser_path) print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) if #rv.parse_errors > 0 then files_to_errors[helpfile] = rv.parse_errors - vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v) return s..'\n '..v end))) + vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v) + return s .. '\n ' .. v + end))) end err_count = err_count + #rv.parse_errors end diff --git a/scripts/lintcommit.lua b/scripts/lintcommit.lua index a3ad4657e9..96f6304247 100644 --- a/scripts/lintcommit.lua +++ b/scripts/lintcommit.lua @@ -16,7 +16,7 @@ local _trace = false -- Print message local function p(s) vim.cmd('set verbose=1') - vim.api.nvim_echo({{s, ''}}, false, {}) + vim.api.nvim_echo({ { s, '' } }, false, {}) vim.cmd('set verbose=0') end @@ -25,7 +25,7 @@ end -- Prints `cmd` if `trace` is enabled. local function run(cmd, or_die) if _trace then - p('run: '..vim.inspect(cmd)) + p('run: ' .. vim.inspect(cmd)) end local rv = vim.trim(vim.fn.system(cmd)) or '' if vim.v.shell_error ~= 0 then @@ -43,14 +43,14 @@ end local function validate_commit(commit_message) -- Return nil if the commit message starts with "fixup" as it signifies it's -- a work in progress and shouldn't be linted yet. - if vim.startswith(commit_message, "fixup") then + if vim.startswith(commit_message, 'fixup') then return nil end - local commit_split = vim.split(commit_message, ":", {plain = true}) + local commit_split = vim.split(commit_message, ':', { plain = true }) -- Return nil if the type is vim-patch since most of the normal rules don't -- apply. - if commit_split[1] == "vim-patch" then + if commit_split[1] == 'vim-patch' then return nil end @@ -81,32 +81,34 @@ local function validate_commit(commit_message) end -- Check if commit introduces a breaking change. - if vim.endswith(before_colon, "!") then + if vim.endswith(before_colon, '!') then before_colon = before_colon:sub(1, -2) end -- Check if type is correct - local type = vim.split(before_colon, "(", {plain = true})[1] - local allowed_types = {'build', 'ci', 'docs', 'feat', 'fix', 'perf', 'refactor', 'revert', 'test', 'vim-patch'} + local type = vim.split(before_colon, '(', { plain = true })[1] + local allowed_types = + { 'build', 'ci', 'docs', 'feat', 'fix', 'perf', 'refactor', 'revert', 'test', 'vim-patch' } if not vim.tbl_contains(allowed_types, type) then return string.format( [[Invalid commit type "%s". Allowed types are: %s. If none of these seem appropriate then use "fix"]], type, - vim.inspect(allowed_types)) + vim.inspect(allowed_types) + ) end -- Check if scope is appropriate - if before_colon:match("%(") then - local scope = vim.trim(commit_message:match("%((.-)%)")) + if before_colon:match('%(') then + local scope = vim.trim(commit_message:match('%((.-)%)')) if scope == '' then return [[Scope can't be empty]] end - if vim.startswith(scope, "nvim_") then - return [[Scope should be "api" instead of "nvim_..."]] + if vim.startswith(scope, 'nvim_') then + return [[Scope should be "api" instead of "nvim_..."]] end local alternative_scope = { @@ -123,17 +125,17 @@ local function validate_commit(commit_message) end -- Check that description doesn't end with a period - if vim.endswith(after_colon, ".") then + if vim.endswith(after_colon, '.') then return [[Description ends with a period (".").]] end -- Check that description starts with a whitespace. - if after_colon:sub(1,1) ~= " " then + if after_colon:sub(1, 1) ~= ' ' then return [[There should be a whitespace after the colon.]] end -- Check that description doesn't start with multiple whitespaces. - if after_colon:sub(1,2) == " " then + if after_colon:sub(1, 2) == ' ' then return [[There should only be one whitespace after the colon.]] end @@ -143,7 +145,7 @@ local function validate_commit(commit_message) end -- Check that description isn't just whitespaces - if vim.trim(after_colon) == "" then + if vim.trim(after_colon) == '' then return [[Description shouldn't be empty.]] end @@ -154,25 +156,25 @@ end function M.main(opt) _trace = not opt or not not opt.trace - local branch = run({'git', 'rev-parse', '--abbrev-ref', 'HEAD'}, true) + local branch = run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' }, true) -- TODO(justinmk): check $GITHUB_REF - local ancestor = run({'git', 'merge-base', 'origin/master', branch}) + local ancestor = run({ 'git', 'merge-base', 'origin/master', branch }) if not ancestor then - ancestor = run({'git', 'merge-base', 'upstream/master', branch}) + ancestor = run({ 'git', 'merge-base', 'upstream/master', branch }) end - local commits_str = run({'git', 'rev-list', ancestor..'..'..branch}, true) + local commits_str = run({ 'git', 'rev-list', ancestor .. '..' .. branch }, true) assert(commits_str) local commits = {} --- @type string[] - for substring in commits_str:gmatch("%S+") do - table.insert(commits, substring) + for substring in commits_str:gmatch('%S+') do + table.insert(commits, substring) end local failed = 0 for _, commit_id in ipairs(commits) do - local msg = run({'git', 'show', '-s', '--format=%s' , commit_id}) + local msg = run({ 'git', 'show', '-s', '--format=%s', commit_id }) if vim.v.shell_error ~= 0 then - p('Invalid commit-id: '..commit_id..'"') + p('Invalid commit-id: ' .. commit_id .. '"') else local invalid_msg = validate_commit(msg) if invalid_msg then @@ -183,20 +185,22 @@ function M.main(opt) p('\n') end - p(string.format([[ + p(string.format( + [[ Invalid commit message: "%s" Commit: %s %s ]], msg, commit_id, - invalid_msg)) + invalid_msg + )) end end end if failed > 0 then - p([[ + p([[ See also: https://github.com/neovim/neovim/blob/master/CONTRIBUTING.md#commit-messages @@ -259,7 +263,7 @@ function M._test() ['feat(:grep/:make)'] = false, ['feat(:grep'] = false, ['feat(:grep/:make'] = false, - ['ci: you\'re saying this commit message just goes on and on and on and on and on and on for way too long?'] = false, + ["ci: you're saying this commit message just goes on and on and on and on and on and on for way too long?"] = false, } local failed = 0 @@ -267,14 +271,15 @@ function M._test() local is_valid = (nil == validate_commit(message)) if is_valid ~= expected then failed = failed + 1 - p(string.format('[ FAIL ]: expected=%s, got=%s\n input: "%s"', expected, is_valid, message)) + p( + string.format('[ FAIL ]: expected=%s, got=%s\n input: "%s"', expected, is_valid, message) + ) end end if failed > 0 then os.exit(1) end - end --- @class LintcommitOptions diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 1c8bc5a3cb..c4ad7fbb03 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -59,9 +59,12 @@ local TAGGED_TYPES = { 'TSNode', 'LanguageTree' } -- Document these as 'table' local ALIAS_TYPES = { - 'Range', 'Range4', 'Range6', 'TSMetadata', + 'Range', + 'Range4', + 'Range6', + 'TSMetadata', 'vim.filetype.add.filetypes', - 'vim.filetype.match.args' + 'vim.filetype.match.args', } local debug_outfile = nil --- @type string? @@ -103,7 +106,7 @@ function StreamRead.new(filename) -- syphon lines to our table local filecontents = {} --- @type string[] for line in io.lines(filename) do - filecontents[#filecontents+1] = line + filecontents[#filecontents + 1] = line end return setmetatable({ @@ -176,9 +179,15 @@ local function process_magic(line, generics) local magic_split = vim.split(magic, ' ', { plain = true }) local directive = magic_split[1] - if vim.list_contains({ - 'cast', 'diagnostic', 'overload', 'meta', 'type' - }, directive) then + if + vim.list_contains({ + 'cast', + 'diagnostic', + 'overload', + 'meta', + 'type', + }, directive) + then -- Ignore LSP directives return '// gg:"' .. line .. '"' end @@ -202,8 +211,7 @@ local function process_magic(line, generics) if directive == 'param' then for _, type in ipairs(TYPES) do magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2') - magic = - magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') + magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') end magic_split = vim.split(magic, ' ', { plain = true }) type_index = 3 @@ -225,7 +233,7 @@ local function process_magic(line, generics) -- fix optional parameters if magic_split[2]:find('%?$') then if not ty:find('nil') then - ty = ty .. '|nil' + ty = ty .. '|nil' end magic_split[2] = magic_split[2]:sub(1, -2) end @@ -240,18 +248,15 @@ local function process_magic(line, generics) end for _, type in ipairs(ALIAS_TYPES) do - ty = ty:gsub('^'..type..'$', 'table') --- @type string + ty = ty:gsub('^' .. type .. '$', 'table') --- @type string end -- surround some types by () for _, type in ipairs(TYPES) do - ty = ty - :gsub('^(' .. type .. '|nil):?$', '(%1)') - :gsub('^(' .. type .. '):?$', '(%1)') + ty = ty:gsub('^(' .. type .. '|nil):?$', '(%1)'):gsub('^(' .. type .. '):?$', '(%1)') end magic_split[type_index] = ty - end magic = table.concat(magic_split, ' ') @@ -281,7 +286,7 @@ local function process_block_comment(line, in_stream) -- easier to program in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2))) end - comment_parts[#comment_parts+1] = thisComment + comment_parts[#comment_parts + 1] = thisComment end local comment = table.concat(comment_parts) @@ -303,7 +308,7 @@ local function process_function_header(line) if fn:sub(1, 1) == '(' then -- it's an anonymous function - return '// ZZ: '..line + return '// ZZ: ' .. line end -- fn has a name, so is interesting @@ -330,10 +335,7 @@ local function process_function_header(line) comma = ', ' end - fn = fn:sub(1, paren_start) - .. 'self' - .. comma - .. fn:sub(paren_start + 1) + fn = fn:sub(1, paren_start) .. 'self' .. comma .. fn:sub(paren_start + 1) end if line:match('local') then @@ -357,7 +359,7 @@ local function process_line(line, in_stream, generics) return process_magic(line:sub(4), generics) end - if vim.startswith(line, '--'..'[[') then -- it's a long comment + if vim.startswith(line, '--' .. '[[') then -- it's a long comment return process_block_comment(line:sub(5), in_stream) end @@ -375,7 +377,7 @@ local function process_line(line, in_stream, generics) local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=') if v and v:match('%.') then -- Special: this lets gen_vimdoc.py handle tables. - return 'table '..v..'() {}' + return 'table ' .. v .. '() {}' end end @@ -418,7 +420,7 @@ local TApp = { timestamp = os.date('%c %Z', os.time()), name = 'Lua2DoX', version = '0.2 20130128', - copyright = 'Copyright (c) Simon Dales 2012-13' + copyright = 'Copyright (c) Simon Dales 2012-13', } setmetatable(TApp, { __index = TApp }) @@ -447,12 +449,15 @@ if arg[1] == '--help' then elseif arg[1] == '--version' then writeln(TApp:getVersion()) writeln(TApp.copyright) -else -- It's a filter. +else -- It's a filter. local filename = arg[1] if arg[2] == '--outdir' then local outdir = arg[3] - if type(outdir) ~= 'string' or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) then + if + type(outdir) ~= 'string' + or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) + then error(('invalid --outdir: "%s"'):format(tostring(outdir))) end vim.fn.mkdir(outdir, 'p') diff --git a/scripts/vimpatch.lua b/scripts/vimpatch.lua index 836f672f6e..cbec50fc17 100755 --- a/scripts/vimpatch.lua +++ b/scripts/vimpatch.lua @@ -10,13 +10,13 @@ local function systemlist(...) local err = nvim.nvim_get_vvar('shell_error') local args_str = nvim.nvim_call_function('string', ...) if 0 ~= err then - error('command failed: '..args_str) + error('command failed: ' .. args_str) end return rv end local function vimpatch_sh_list_numbers() - return systemlist( { { 'bash', '-c', 'scripts/vim-patch.sh -M', } } ) + return systemlist({ { 'bash', '-c', 'scripts/vim-patch.sh -M' } }) end -- Generates the lines to be inserted into the src/version.c @@ -55,9 +55,9 @@ local function patch_version_c() nvim.nvim_command('silent normal! j0d/};\rk') -- Insert the lines. nvim.nvim_call_function('append', { - nvim.nvim_eval('line(".")'), - lines, - }) + nvim.nvim_eval('line(".")'), + lines, + }) nvim.nvim_command('silent write') end -- cgit From 3692fd4c873a2cd7ad69eb09765eed2993570c49 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Mon, 11 Dec 2023 04:10:00 -0500 Subject: feat(gen_lsp.lua): validate CLI args #26514 - Improve CLI argument parsing, rejects invalid argument and commands as early as possible. Also prints USAGE in the command line. - No longer allows `--`, use `--out ` instead. - Print a little bit of verbose messages to better know what's going on rather than remaining silent at all times. - Add type annotation `gen_lsp._opt` to avoid type warnings. --- scripts/gen_lsp.lua | 71 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 20 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua index 6ff8dcb3f4..9fbcc1c15e 100644 --- a/scripts/gen_lsp.lua +++ b/scripts/gen_lsp.lua @@ -1,11 +1,15 @@ ---[[ -Generates lua-ls annotations for lsp +-- Generates lua-ls annotations for lsp. + +local USAGE = [[ +Generates lua-ls annotations for lsp. + USAGE: -nvim -l scripts/gen_lsp.lua gen # this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua -nvim -l scripts/gen_lsp.lua gen --version 3.18 --build/new_lsp_types.lua +nvim -l scripts/gen_lsp.lua gen # by default, this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua nvim -l scripts/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua nvim -l scripts/gen_lsp.lua gen --version 3.18 --methods ---]] +]] + +local DEFAULT_LSP_VERSION = '3.18' local M = {} @@ -14,15 +18,18 @@ local function tofile(fname, text) if not f then error(('failed to write: %s'):format(f)) else + print(('Written to: %s'):format(fname)) f:write(text) f:close() end end +---@param opt gen_lsp._opt local function read_json(opt) local uri = 'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/' .. opt.version .. '/metaModel/metaModel.json' + print('Reading ' .. uri) local res = vim.system({ 'curl', '--no-progress-meter', uri, '-o', '-' }):wait() if res.code ~= 0 or (res.stdout or ''):len() < 999 then @@ -99,19 +106,30 @@ return protocol vim.cmd.write() end +---@class gen_lsp._opt +---@field output_file string +---@field version string +---@field methods boolean + +---@param opt gen_lsp._opt function M.gen(opt) - local protocol = read_json(opt) + local protocol = read_json(opt) --- @type table if opt.methods then gen_methods(protocol) end local output = { - '--[[', + '--' .. '[[', 'This file is autogenerated from scripts/gen_lsp.lua', 'Regenerate:', - [=[nvim -l scripts/gen_lsp.lua gen --version 3.18 --runtime/lua/vim/lsp/_meta/protocol.lua]=], - '--]]', + ([=[nvim -l scripts/gen_lsp.lua gen --version %s --out runtime/lua/vim/lsp/_meta/protocol.lua]=]):format( + DEFAULT_LSP_VERSION + ), + '--' .. ']]', + '', + '---@meta', + "error('Cannot require a meta file')", '', '---@alias lsp.null nil', '---@alias uinteger integer', @@ -265,26 +283,39 @@ end local opt = { output_file = 'runtime/lua/vim/lsp/_meta/protocol.lua', - version = nil, - methods = nil, + version = DEFAULT_LSP_VERSION, + methods = false, } -for i = 1, #_G.arg do +local command = nil +local i = 1 +while i <= #_G.arg do if _G.arg[i] == '--out' then - opt.output_file = _G.arg[i + 1] + opt.output_file = assert(_G.arg[i + 1], '--out needed') + i = i + 1 elseif _G.arg[i] == '--version' then - opt.version = _G.arg[i + 1] + opt.version = assert(_G.arg[i + 1], '--version needed') + i = i + 1 elseif _G.arg[i] == '--methods' then opt.methods = true - elseif vim.startswith(_G.arg[i], '--') then - opt.output_file = _G.arg[i]:sub(3) + elseif vim.startswith(_G.arg[i], '-') then + error('Unrecognized args: ' .. _G.arg[i]) + else + if command then + error('More than one command was given: ' .. _G.arg[i]) + else + command = _G.arg[i] + end end + i = i + 1 end -for _, a in ipairs(arg) do - if M[a] then - M[a](opt) - end +if not command then + print(USAGE) +elseif M[command] then + M[command](opt) -- see M.gen() +else + error('Unknown command: ' .. command) end return M -- cgit From ef58ee48f4d553d364c4284870b0860f8cc8427b Mon Sep 17 00:00:00 2001 From: dundargoc <33953936+dundargoc@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:31:39 +0100 Subject: docs: add wiki FAQ to the runtime documentation (#26539) Problem: Wiki contents are not discoverable and hard to maintain. Solution: Move FAQ to runtime docs. Co-authored-by: Christian Clason --- scripts/gen_help_html.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 9369711b0f..8226a0548f 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -80,6 +80,7 @@ local exclude_invalid_urls = { ['http://wiki.services.openoffice.org/wiki/Dictionaries'] = 'spell.txt', ['http://www.adapower.com'] = 'ft_ada.txt', ['http://www.jclark.com/'] = 'quickfix.txt', + ['http://oldblog.antirez.com/post/redis-and-scripting.html'] = 'faq.txt', } -- Deprecated, brain-damaged files that I don't care about. -- cgit From 7908dc0d1552e715cc4cc077e6b87d53d0e323c2 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Tue, 12 Dec 2023 22:48:48 +0100 Subject: docs: move vim-patch wiki page to runtime documentation --- scripts/vim-patch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 47c6d293bc..c7ccc3992d 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -24,7 +24,7 @@ CREATED_FILES=() usage() { echo "Port Vim patches to Neovim" - echo "https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim" + echo "https://neovim.io/doc/user/dev_vimpatch.html" echo echo "Usage: ${BASENAME} [-h | -l | -p vim-revision | -r pr-number]" echo @@ -427,7 +427,7 @@ stage_patch() { or "%s -s --draft" to create a draft pull request. See the wiki for more information: - * https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim + * https://neovim.io/doc/user/dev_vimpatch.html ' "${vim_version}" "${BASENAME}" "${BASENAME}" "${BASENAME}" return $ret } -- cgit From 3c2c022e5e299ecac4663c3813e2db5e2b099ffa Mon Sep 17 00:00:00 2001 From: Famiu Haque Date: Thu, 7 Dec 2023 01:34:29 +0600 Subject: refactor(options): remove option type macros Problem: We have `P_(BOOL|NUM|STRING)` macros to represent an option's type, which is redundant because `OptValType` can already do that. The current implementation of option type flags is also too limited to allow adding multitype options in the future. Solution: Remove `P_(BOOL|NUM|STRING)` and replace it with a new `type_flags` attribute in `vimoption_T`. Also do some groundwork for adding multitype options in the future. Side-effects: Attempting to set an invalid keycode option (e.g. `set t_foo=123`) no longer gives an error. --- scripts/gen_eval_files.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index dddc7d000c..2b0ad4431a 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -59,7 +59,7 @@ local LUA_KEYWORDS = { } local OPTION_TYPES = { - bool = 'boolean', + boolean = 'boolean', number = 'integer', string = 'string', } @@ -603,7 +603,7 @@ local function build_option_tags(opt) local tags = { opt.full_name } tags[#tags + 1] = opt.abbreviation - if opt.type == 'bool' then + if opt.type == 'boolean' then for i = 1, #tags do tags[#tags + 1] = 'no' .. tags[i] end @@ -642,7 +642,7 @@ local function render_option_doc(_f, opt, write) name_str = string.format("'%s'", opt.full_name) end - local otype = opt.type == 'bool' and 'boolean' or opt.type + local otype = opt.type == 'boolean' and 'boolean' or opt.type if opt.defaults.doc or opt.defaults.if_true ~= nil or opt.defaults.meta ~= nil then local v = render_option_default(opt.defaults, true) local pad = string.rep('\t', math.max(1, math.ceil((24 - #name_str) / 8))) -- cgit From e8d3c4cccb9d362952a09216a5a114ee6d024c14 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Wed, 20 Dec 2023 16:34:17 +0000 Subject: feat: generate types and docs for v variables --- scripts/gen_eval_files.lua | 118 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) (limited to 'scripts') diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index 2b0ad4431a..e2ca62c2ee 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -48,6 +48,16 @@ local LUA_OPTION_META_HEADER = { 'vim.wo = vim.wo', } +local LUA_VVAR_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", + '', + '--- @class vim.v', + 'vim.v = ...', +} + local LUA_KEYWORDS = { ['and'] = true, ['end'] = true, @@ -56,6 +66,8 @@ local LUA_KEYWORDS = { ['if'] = true, ['while'] = true, ['repeat'] = true, + ['true'] = true, + ['false'] = true, } local OPTION_TYPES = { @@ -534,6 +546,30 @@ local function render_option_meta(_f, opt, write) end end +--- @param _f string +--- @param opt vim.option_meta +--- @param write fun(line: string) +local function render_vvar_meta(_f, opt, write) + write('') + + local desc = split(norm_text(opt.desc)) + while desc[#desc]:match('^%s*$') do + desc[#desc] = nil + end + + for _, l in ipairs(desc) do + write('--- ' .. l) + end + + write('--- @type ' .. (opt.type or 'any')) + + if LUA_KEYWORDS[opt.full_name] then + write("vim.v['" .. opt.full_name .. "'] = ...") + else + write('vim.v.' .. opt.full_name .. ' = ...') + end +end + --- @param s string[] --- @return string local function scope_to_doc(s) @@ -574,6 +610,21 @@ local function scope_more_doc(o) return '' end +--- @param x string +--- @return string +local function dedent(x) + local xs = split(x) + local leading_ws = xs[1]:match('^%s*') --[[@as string]] + local leading_ws_pat = '^' .. leading_ws + + for i in ipairs(xs) do + local strip_pat = xs[i]:match(leading_ws_pat) and leading_ws_pat or '^%s*' + xs[i] = xs[i]:gsub(strip_pat, '') + end + + return table.concat(xs, '\n') +end + --- @return table local function get_option_meta() local opts = require('src/nvim/options').options @@ -596,6 +647,18 @@ local function get_option_meta() return ret end +--- @return table +local function get_vvar_meta() + local info = require('src/nvim/vvars').vars + local ret = {} --- @type table + for name, o in pairs(info) do + o.desc = dedent(o.desc) + o.full_name = name + ret[name] = o + end + return ret +end + --- @param opt vim.option_meta --- @return string[] local function build_option_tags(opt) @@ -666,6 +729,45 @@ local function render_option_doc(_f, opt, write) end end +--- @param _f string +--- @param vvar vim.option_meta +--- @param write fun(line: string) +local function render_vvar_doc(_f, vvar, write) + local name = vvar.full_name + + local tags = { 'v:' .. name, name .. '-variable' } + if vvar.tags then + vim.list_extend(tags, vvar.tags) + end + + for i, t in ipairs(tags) do + tags[i] = '*' .. t .. '*' + end + + local tag_str = table.concat(tags, ' ') + local conceal_offset = 2 * (#tags - 1) + + local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) + write(tag_pad .. tag_str) + + local desc = split(vvar.desc) + + if (#desc == 1 or #desc == 2 and desc[2]:match('^%s*$')) and #name < 10 then + -- single line + write('v:' .. name .. '\t' .. desc[1]:gsub('^%s*', '')) + write('') + else + write('v:' .. name) + for _, l in ipairs(desc) do + if l == '<' or l:match('^<%s') then + write(l) + else + write('\t\t' .. l:gsub('\\<', '<')) + end + end + end +end + --- @class nvim.gen_eval_files.elem --- @field path string --- @field from? string Skip lines in path until this pattern is reached. @@ -756,6 +858,22 @@ local CONFIG = { funcs = get_option_meta, render = render_option_doc, }, + { + path = 'runtime/lua/vim/_meta/vvars.lua', + header = LUA_VVAR_META_HEADER, + funcs = get_vvar_meta, + render = render_vvar_meta, + }, + { + path = 'runtime/doc/vvars.txt', + header = { '' }, + from = 'Type |gO| to see the table of contents.', + footer = { + ' vim:tw=78:ts=8:noet:ft=help:norl:', + }, + funcs = get_vvar_meta, + render = render_vvar_doc, + }, } --- @param elem nvim.gen_eval_files.elem -- cgit From 2f43af6423193eb52e9a6635034f8c79f40f1706 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Sun, 10 Dec 2023 23:50:54 -0500 Subject: refactor(gen_lsp.lua): add typing for the LSP protocol JSON data model Enhance readability and intellisense by incorporating type annotations. Types are not very strict and may not encompass th entire LSP Protocol metamodel; the scope is up to what's relevant for generating type annotations for LSP (`_meta/protocol.lua`). Based on the model schema: https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/3.18/metaModel/metaModel.schema.json No behavioral changes (and hence no diff on _meta/protocol.lua) should exist in this commit. --- scripts/gen_lsp.lua | 126 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua index 9fbcc1c15e..943e88896b 100644 --- a/scripts/gen_lsp.lua +++ b/scripts/gen_lsp.lua @@ -24,7 +24,17 @@ local function tofile(fname, text) end end ----@param opt gen_lsp._opt +--- The LSP protocol JSON data (it's partial, non-exhaustive). +--- https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/3.18/metaModel/metaModel.schema.json +--- @class vim._gen_lsp.Protocol +--- @field requests vim._gen_lsp.Request[] +--- @field notifications vim._gen_lsp.Notification[] +--- @field structures vim._gen_lsp.Structure[] +--- @field enumerations vim._gen_lsp.Enumeration[] +--- @field typeAliases vim._gen_lsp.TypeAlias[] + +---@param opt vim._gen_lsp.opt +---@return vim._gen_lsp.Protocol local function read_json(opt) local uri = 'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/' .. opt.version @@ -46,6 +56,7 @@ local function name(s) return s:gsub('^%$', 'dollar'):gsub('/', '_') end +---@param protocol vim._gen_lsp.Protocol local function gen_methods(protocol) local output = { '-- Generated by gen_lsp.lua, keep at end of file.', @@ -56,6 +67,32 @@ local function gen_methods(protocol) } local indent = (' '):rep(2) + --- @class vim._gen_lsp.Request + --- @field deprecated? string + --- @field documentation? string + --- @field messageDirection string + --- @field method string + --- @field params? any + --- @field proposed? boolean + --- @field registrationMethod? string + --- @field registrationOptions? any + --- @field since? string + + --- @class vim._gen_lsp.Notification + --- @field deprecated? string + --- @field documentation? string + --- @field errorData? any + --- @field messageDirection string + --- @field method string + --- @field params? any[] + --- @field partialResult? any + --- @field proposed? boolean + --- @field registrationMethod? string + --- @field registrationOptions? any + --- @field result any + --- @field since? string + + ---@type (vim._gen_lsp.Request|vim._gen_lsp.Notification)[] local all = vim.list_extend(protocol.requests, protocol.notifications) table.sort(all, function(a, b) return name(a.method) < name(b.method) @@ -106,14 +143,15 @@ return protocol vim.cmd.write() end ----@class gen_lsp._opt +---@class vim._gen_lsp.opt ---@field output_file string ---@field version string ---@field methods boolean ----@param opt gen_lsp._opt +---@param opt vim._gen_lsp.opt function M.gen(opt) - local protocol = read_json(opt) --- @type table + --- @type vim._gen_lsp.Protocol + local protocol = read_json(opt) if opt.methods then gen_methods(protocol) @@ -144,6 +182,7 @@ function M.gen(opt) local anonymous_num = 0 + ---@type string[] local anonym_classes = {} local simple_types = { @@ -154,32 +193,65 @@ function M.gen(opt) 'decimal', } + --- @class vim._gen_lsp.Type + --- @field kind string a common field for all Types. + --- @field name? string for ReferenceType, BaseType + --- @field element? any for ArrayType + --- @field items? vim._gen_lsp.Type[] for OrType, AndType + --- @field key? vim._gen_lsp.Type for MapType + --- @field value? string|vim._gen_lsp.Type for StringLiteralType, MapType, StructureLiteralType + + ---@param type vim._gen_lsp.Type + ---@return string local function parse_type(type) + -- ReferenceType | BaseType if type.kind == 'reference' or type.kind == 'base' then if vim.tbl_contains(simple_types, type.name) then return type.name end return 'lsp.' .. type.name + + -- ArrayType elseif type.kind == 'array' then return parse_type(type.element) .. '[]' + + -- OrType elseif type.kind == 'or' then local val = '' for _, item in ipairs(type.items) do - val = val .. parse_type(item) .. '|' + val = val .. parse_type(item) .. '|' --[[ @as string ]] end val = val:sub(0, -2) return val + + -- StringLiteralType elseif type.kind == 'stringLiteral' then return '"' .. type.value .. '"' + + -- MapType elseif type.kind == 'map' then - return 'table<' .. parse_type(type.key) .. ', ' .. parse_type(type.value) .. '>' + local key = assert(type.key) + local value = type.value --[[ @as vim._gen_lsp.Type ]] + return 'table<' .. parse_type(key) .. ', ' .. parse_type(value) .. '>' + + -- StructureLiteralType elseif type.kind == 'literal' then -- can I use ---@param disabled? {reason: string} -- use | to continue the inline class to be able to add docs -- https://github.com/LuaLS/lua-language-server/issues/2128 anonymous_num = anonymous_num + 1 local anonym = { '---@class anonym' .. anonymous_num } - for _, field in ipairs(type.value.properties) do + + --- @class vim._gen_lsp.StructureLiteral translated to anonymous @class. + --- @field deprecated? string + --- @field description? string + --- @field properties vim._gen_lsp.Property[] + --- @field proposed? boolean + --- @field since? string + + ---@type vim._gen_lsp.StructureLiteral + local structural_literal = assert(type.value) --[[ @as vim._gen_lsp.StructureLiteral ]] + for _, field in ipairs(structural_literal.properties) do if field.documentation then field.documentation = field.documentation:gsub('\n', '\n---') anonym[#anonym + 1] = '---' .. field.documentation @@ -195,6 +267,8 @@ function M.gen(opt) anonym_classes[#anonym_classes + 1] = line end return 'anonym' .. anonymous_num + + -- TupleType elseif type.kind == 'tuple' then local tuple = '{ ' for i, value in ipairs(type.items) do @@ -204,10 +278,20 @@ function M.gen(opt) tuple = tuple:sub(0, -3) return tuple .. ' }' end - vim.print(type) + + vim.print('WARNING: Unknown type ', type) return '' end + --- @class vim._gen_lsp.Structure translated to @class + --- @field deprecated? string + --- @field documentation? string + --- @field extends? { kind: string, name: string }[] + --- @field mixins? { kind: string, name: string }[] + --- @field name string + --- @field properties? vim._gen_lsp.Property[] members, translated to @field + --- @field proposed? boolean + --- @field since? string for _, structure in ipairs(protocol.structures) do if structure.documentation then structure.documentation = structure.documentation:gsub('\n', '\n---') @@ -225,6 +309,15 @@ function M.gen(opt) else output[#output + 1] = '---@class lsp.' .. structure.name end + + --- @class vim._gen_lsp.Property translated to @field + --- @field deprecated? string + --- @field documentation? string + --- @field name string + --- @field optional? boolean + --- @field proposed? boolean + --- @field since? string + --- @field type { kind: string, name: string } for _, field in ipairs(structure.properties or {}) do if field.documentation then field.documentation = field.documentation:gsub('\n', '\n---') @@ -239,6 +332,14 @@ function M.gen(opt) output[#output + 1] = '' end + --- @class vim._gen_lsp.Enumeration translated to @enum + --- @field deprecated string? + --- @field documentation string? + --- @field name string? + --- @field proposed boolean? + --- @field since string? + --- @field suportsCustomValues boolean? + --- @field values { name: string, value: string, documentation?: string, since?: string }[] for _, enum in ipairs(protocol.enumerations) do if enum.documentation then enum.documentation = enum.documentation:gsub('\n', '\n---') @@ -256,6 +357,13 @@ function M.gen(opt) output[#output + 1] = '' end + --- @class vim._gen_lsp.TypeAlias translated to @alias + --- @field deprecated? string? + --- @field documentation? string + --- @field name string + --- @field proposed? boolean + --- @field since? string + --- @field type vim._gen_lsp.Type for _, alias in ipairs(protocol.typeAliases) do if alias.documentation then alias.documentation = alias.documentation:gsub('\n', '\n---') @@ -274,6 +382,7 @@ function M.gen(opt) output[#output + 1] = '' end + -- anonymous classes for _, line in ipairs(anonym_classes) do output[#output + 1] = line end @@ -281,6 +390,7 @@ function M.gen(opt) tofile(opt.output_file, table.concat(output, '\n')) end +---@type vim._gen_lsp.opt local opt = { output_file = 'runtime/lua/vim/lsp/_meta/protocol.lua', version = DEFAULT_LSP_VERSION, -- cgit From 6c35fb421e888d0cbdfac07a5ff4579c9be7f0ec Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Mon, 11 Dec 2023 02:25:17 -0500 Subject: fix(gen_lsp.lua): improve type name, and fix wrong type inheritance Style improvements: 1. Anonymous classes derived from `StructureLiteralType` should have a better name. The class name can be also nested. Examples: ```diff ----@field serverInfo? anonym1 +---@field serverInfo? lsp._anonym1.serverInfo ``` ```diff ----@field insertTextModeSupport? anonym26 +---@field insertTextModeSupport? lsp._anonym26.completionItem.insertTextModeSupport ``` 2. Add one separate empty line before each `@field` definition. Without these, empty lines the doc can look confusing because descriptions also may contain empty lines. See `lsp.CompletionItem` for example: ```lua ---The kind of this completion item. Based of the kind ---an icon is chosen by the editor. ---@field kind? lsp.CompletionItemKind ---Tags for this completion item. --- ---@since 3.15.0 ---@field tags? lsp.CompletionItemTag[] ``` It might feel like "Tags for this completion item" belongs to `kind`, not `tags` due to the lack of separator blank lines. The following (after this commit) should look much better: ```diff ---The kind of this completion item. Based of the kind ---an icon is chosen by the editor. ---@field kind? lsp.CompletionItemKind +--- ---Tags for this completion item. --- ---@since 3.15.0 ---@field tags? lsp.CompletionItemTag[] ``` 3. Escape some LSP-specific annotations that can't be recognized by lua-ls. It'd be better to make them visible in LSP hover doc windows. Example: `@sample ...`. Fixes: 1. A type may extend from more than one base types (as well as mixin types). Previously only the first base class was being considered, resulting incomplete base classes for `@class` definitions. Example: `InlayHintOptions` (should have both of `resolveProvider` and `workDoneProgress`, the latter is from `WorkDoneProgressOptions`) ```diff ----@class lsp.InlayHintOptions +---@class lsp.InlayHintOptions: lsp.WorkDoneProgressOptions ``` 2. Remove `<200b>` (zero-width space) unicode characters. 3. Add the missing newline at EOF. --- scripts/gen_lsp.lua | 98 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 38 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua index 943e88896b..0e7eb38cca 100644 --- a/scripts/gen_lsp.lua +++ b/scripts/gen_lsp.lua @@ -51,7 +51,7 @@ local function read_json(opt) end -- Gets the Lua symbol for a given fully-qualified LSP method name. -local function name(s) +local function to_luaname(s) -- "$/" prefix is special: https://microsoft.github.io/language-server-protocol/specification/#dollarRequests return s:gsub('^%$', 'dollar'):gsub('/', '_') end @@ -95,7 +95,7 @@ local function gen_methods(protocol) ---@type (vim._gen_lsp.Request|vim._gen_lsp.Notification)[] local all = vim.list_extend(protocol.requests, protocol.notifications) table.sort(all, function(a, b) - return name(a.method) < name(b.method) + return to_luaname(a.method) < to_luaname(b.method) end) for _, item in ipairs(all) do if item.method then @@ -105,7 +105,7 @@ local function gen_methods(protocol) output[#output + 1] = indent .. '--- ' .. docstring end end - output[#output + 1] = ("%s%s = '%s',"):format(indent, name(item.method), item.method) + output[#output + 1] = ("%s%s = '%s',"):format(indent, to_luaname(item.method), item.method) end end output[#output + 1] = '}' @@ -193,6 +193,16 @@ function M.gen(opt) 'decimal', } + ---@param documentation string + local _process_documentation = function(documentation) + documentation = documentation:gsub('\n', '\n---') + -- Remove <200b> (zero-width space) unicode characters: e.g., `**/<200b>*` + documentation = documentation:gsub('\226\128\139', '') + -- Escape annotations that are not recognized by lua-ls + documentation = documentation:gsub('%^---@sample', '---\\@sample') + return '---' .. documentation + end + --- @class vim._gen_lsp.Type --- @field kind string a common field for all Types. --- @field name? string for ReferenceType, BaseType @@ -202,8 +212,10 @@ function M.gen(opt) --- @field value? string|vim._gen_lsp.Type for StringLiteralType, MapType, StructureLiteralType ---@param type vim._gen_lsp.Type + ---@param prefix? string Optional prefix associated with the this type, made of (nested) field name. + --- Used to generate class name for structure literal types. ---@return string - local function parse_type(type) + local function parse_type(type, prefix) -- ReferenceType | BaseType if type.kind == 'reference' or type.kind == 'base' then if vim.tbl_contains(simple_types, type.name) then @@ -213,13 +225,13 @@ function M.gen(opt) -- ArrayType elseif type.kind == 'array' then - return parse_type(type.element) .. '[]' + return parse_type(type.element, prefix) .. '[]' -- OrType elseif type.kind == 'or' then local val = '' for _, item in ipairs(type.items) do - val = val .. parse_type(item) .. '|' --[[ @as string ]] + val = val .. parse_type(item, prefix) .. '|' --[[ @as string ]] end val = val:sub(0, -2) return val @@ -232,7 +244,7 @@ function M.gen(opt) elseif type.kind == 'map' then local key = assert(type.key) local value = type.value --[[ @as vim._gen_lsp.Type ]] - return 'table<' .. parse_type(key) .. ', ' .. parse_type(value) .. '>' + return 'table<' .. parse_type(key, prefix) .. ', ' .. parse_type(value, prefix) .. '>' -- StructureLiteralType elseif type.kind == 'literal' then @@ -240,7 +252,14 @@ function M.gen(opt) -- use | to continue the inline class to be able to add docs -- https://github.com/LuaLS/lua-language-server/issues/2128 anonymous_num = anonymous_num + 1 - local anonym = { '---@class anonym' .. anonymous_num } + local anonymous_classname = 'lsp._anonym' .. anonymous_num + if prefix then + anonymous_classname = anonymous_classname .. '.' .. prefix + end + local anonym = vim.tbl_flatten { -- remove nil + anonymous_num > 1 and '' or nil, + '---@class ' .. anonymous_classname, + } --- @class vim._gen_lsp.StructureLiteral translated to anonymous @class. --- @field deprecated? string @@ -252,27 +271,29 @@ function M.gen(opt) ---@type vim._gen_lsp.StructureLiteral local structural_literal = assert(type.value) --[[ @as vim._gen_lsp.StructureLiteral ]] for _, field in ipairs(structural_literal.properties) do + anonym[#anonym + 1] = '---' if field.documentation then - field.documentation = field.documentation:gsub('\n', '\n---') - anonym[#anonym + 1] = '---' .. field.documentation + anonym[#anonym + 1] = _process_documentation(field.documentation) end anonym[#anonym + 1] = '---@field ' .. field.name .. (field.optional and '?' or '') .. ' ' - .. parse_type(field.type) + .. parse_type(field.type, prefix .. '.' .. field.name) end - anonym[#anonym + 1] = '' + -- anonym[#anonym + 1] = '' for _, line in ipairs(anonym) do - anonym_classes[#anonym_classes + 1] = line + if line then + anonym_classes[#anonym_classes + 1] = line + end end - return 'anonym' .. anonymous_num + return anonymous_classname -- TupleType elseif type.kind == 'tuple' then local tuple = '{ ' for i, value in ipairs(type.items) do - tuple = tuple .. '[' .. i .. ']: ' .. parse_type(value) .. ', ' + tuple = tuple .. '[' .. i .. ']: ' .. parse_type(value, prefix) .. ', ' end -- remove , at the end tuple = tuple:sub(0, -3) @@ -293,22 +314,22 @@ function M.gen(opt) --- @field proposed? boolean --- @field since? string for _, structure in ipairs(protocol.structures) do + -- output[#output + 1] = '' if structure.documentation then - structure.documentation = structure.documentation:gsub('\n', '\n---') - output[#output + 1] = '---' .. structure.documentation + output[#output + 1] = _process_documentation(structure.documentation) end - if structure.extends then - local class_string = '---@class lsp.' - .. structure.name - .. ': ' - .. parse_type(structure.extends[1]) - for _, mixin in ipairs(structure.mixins or {}) do - class_string = class_string .. ', ' .. parse_type(mixin) - end - output[#output + 1] = class_string - else - output[#output + 1] = '---@class lsp.' .. structure.name + local class_string = ('---@class lsp.%s'):format(structure.name) + if structure.extends or structure.mixins then + local inherits_from = table.concat( + vim.list_extend( + vim.tbl_map(parse_type, structure.extends or {}), + vim.tbl_map(parse_type, structure.mixins or {}) + ), + ', ' + ) + class_string = class_string .. ': ' .. inherits_from end + output[#output + 1] = class_string --- @class vim._gen_lsp.Property translated to @field --- @field deprecated? string @@ -319,15 +340,15 @@ function M.gen(opt) --- @field since? string --- @field type { kind: string, name: string } for _, field in ipairs(structure.properties or {}) do + output[#output + 1] = '---' -- Insert a single newline between @fields (and after @class) if field.documentation then - field.documentation = field.documentation:gsub('\n', '\n---') - output[#output + 1] = '---' .. field.documentation + output[#output + 1] = _process_documentation(field.documentation) end output[#output + 1] = '---@field ' .. field.name .. (field.optional and '?' or '') .. ' ' - .. parse_type(field.type) + .. parse_type(field.type, field.name) end output[#output + 1] = '' end @@ -342,8 +363,7 @@ function M.gen(opt) --- @field values { name: string, value: string, documentation?: string, since?: string }[] for _, enum in ipairs(protocol.enumerations) do if enum.documentation then - enum.documentation = enum.documentation:gsub('\n', '\n---') - output[#output + 1] = '---' .. enum.documentation + output[#output + 1] = _process_documentation(enum.documentation) end local enum_type = '---@alias lsp.' .. enum.name for _, value in ipairs(enum.values) do @@ -366,18 +386,20 @@ function M.gen(opt) --- @field type vim._gen_lsp.Type for _, alias in ipairs(protocol.typeAliases) do if alias.documentation then - alias.documentation = alias.documentation:gsub('\n', '\n---') - output[#output + 1] = '---' .. alias.documentation + output[#output + 1] = _process_documentation(alias.documentation) end if alias.type.kind == 'or' then local alias_type = '---@alias lsp.' .. alias.name .. ' ' for _, item in ipairs(alias.type.items) do - alias_type = alias_type .. parse_type(item) .. '|' + alias_type = alias_type .. parse_type(item, alias.name) .. '|' end alias_type = alias_type:sub(0, -2) output[#output + 1] = alias_type else - output[#output + 1] = '---@alias lsp.' .. alias.name .. ' ' .. parse_type(alias.type) + output[#output + 1] = '---@alias lsp.' + .. alias.name + .. ' ' + .. parse_type(alias.type, alias.name) end output[#output + 1] = '' end @@ -387,7 +409,7 @@ function M.gen(opt) output[#output + 1] = line end - tofile(opt.output_file, table.concat(output, '\n')) + tofile(opt.output_file, table.concat(output, '\n') .. '\n') end ---@type vim._gen_lsp.opt -- cgit From 31d7007bf745d3f03902b27c2124d473ec2f8906 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Wed, 27 Dec 2023 13:43:48 +0100 Subject: docs: convert BACKERS.md to backers.txt There is no reason for this file to be in project root, which is crowded as is. This also fits nicely part of the ongoing work towards gathering as much of the documentation as possible into one place. --- scripts/gen_help_html.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 8226a0548f..49417e72bb 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -37,6 +37,9 @@ local spell_dict = { vimL = 'Vimscript', viml = 'Vimscript', } +local spell_ignore_files = { + ['backers.txt'] = 'true', +} local language = nil local M = {} @@ -86,6 +89,7 @@ local exclude_invalid_urls = { -- Deprecated, brain-damaged files that I don't care about. local ignore_errors = { ['pi_netrw.txt'] = true, + ['backers.txt'] = true, } local function tofile(fname, text) @@ -386,9 +390,10 @@ local function visit_validate(root, level, lang_tree, opt, stats) and (not vim.tbl_contains({ 'codespan', 'taglink', 'tag' }, parent)) then local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. - if spell_dict[text_nopunct] then + local fname_basename = assert(vim.fs.basename(opt.fname)) + if spell_dict[text_nopunct] and not spell_ignore_files[fname_basename] then invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} - invalid_spelling[text_nopunct][vim.fs.basename(opt.fname)] = node_text(root:parent()) + invalid_spelling[text_nopunct][fname_basename] = node_text(root:parent()) end elseif node_name == 'url' then local fixed_url, _ = fix_url(trim(text)) -- cgit From 5dc0bdfe98b59bb03226167ed541d17cc5af30b1 Mon Sep 17 00:00:00 2001 From: Mathias Fußenegger Date: Tue, 2 Jan 2024 14:32:43 +0100 Subject: docs(glob): add glob module (#26853) --- scripts/gen_vimdoc.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 8ed88cb8f5..b51bd5fbf5 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -164,6 +164,7 @@ CONFIG = { 'filetype.lua', 'keymap.lua', 'fs.lua', + 'glob.lua', 'secure.lua', 'version.lua', 'iter.lua', @@ -187,6 +188,7 @@ CONFIG = { 'runtime/lua/vim/_inspector.lua', 'runtime/lua/vim/snippet.lua', 'runtime/lua/vim/text.lua', + 'runtime/lua/vim/glob.lua', 'runtime/lua/vim/_meta/builtin.lua', 'runtime/lua/vim/_meta/diff.lua', 'runtime/lua/vim/_meta/mpack.lua', @@ -251,6 +253,7 @@ CONFIG = { 'spell': 'vim.spell', 'snippet': 'vim.snippet', 'text': 'vim.text', + 'glob': 'vim.glob', }, 'append_only': [ 'shared.lua', -- cgit From 5e2d4b3c4dbd56342cabb6993d354f690e0a1575 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 28 Dec 2023 13:04:34 -0500 Subject: refactor(gen_vimdoc): use stronger typing for CONFIG, avoid dict --- scripts/gen_vimdoc.py | 279 +++++++++++++++++++++++++++++--------------------- 1 file changed, 165 insertions(+), 114 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index b51bd5fbf5..f22d11ed6b 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. + +r"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. Also generates *.mpack files. To inspect the *.mpack structure: :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B'))) @@ -32,24 +33,29 @@ The generated :help text for each function is formatted as follows: parameter is marked as [out]. - Each function documentation is separated by a single line. """ + +from __future__ import annotations + import argparse +import collections +import dataclasses +import logging import os import re -import sys import shutil -import textwrap import subprocess -import collections -import msgpack -import logging -from typing import Tuple +import sys +import textwrap from pathlib import Path - +from typing import Any, Callable, Dict, List, Literal, Tuple from xml.dom import minidom + +import msgpack + Element = minidom.Element Document = minidom.Document -MIN_PYTHON_VERSION = (3, 6) +MIN_PYTHON_VERSION = (3, 7) MIN_DOXYGEN_VERSION = (1, 9, 0) if sys.version_info < MIN_PYTHON_VERSION: @@ -68,7 +74,7 @@ if doxygen_version < MIN_DOXYGEN_VERSION: # Need a `nvim` that supports `-l`, try the local build nvim_path = Path(__file__).parent / "../build/bin/nvim" if nvim_path.exists(): - nvim = str(nvim_path) + nvim = nvim_path.resolve() else: # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works. nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True) @@ -103,12 +109,59 @@ filter_cmd = '%s %s' % (sys.executable, script_path) msgs = [] # Messages to show on exit. lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') -CONFIG = { - 'api': { - 'mode': 'c', - 'filename': 'api.txt', + +SectionName = str + +@dataclasses.dataclass +class Config: + """Config for documentation.""" + + mode: Literal['c', 'lua'] + + filename: str + """Generated documentation target, e.g. api.txt""" + + section_order: List[str] + """Section ordering.""" + + files: List[str] + """List of files/directories for doxygen to read, relative to `base_dir`.""" + + file_patterns: str + """file patterns used by doxygen.""" + + section_name: Dict[str, SectionName] + """Section name overrides. Key: filename (e.g., vim.c)""" + + section_fmt: Callable[[SectionName], str] + """For generated section names.""" + + helptag_fmt: Callable[[SectionName], str] + """Section helptag.""" + + fn_helptag_fmt: Callable[[str, str, bool], str] + """Per-function helptag.""" + + module_override: Dict[str, str] + """Module name overrides (for Lua).""" + + append_only: List[str] + """Append the docs for these modules, do not start a new section.""" + + fn_name_prefix: str + """Only function with this prefix are considered""" + + fn_name_fmt: Callable[[str, str], str] | None = None + + include_tables: bool = True + + +CONFIG: Dict[str, Config] = { + 'api': Config( + mode = 'c', + filename = 'api.txt', # Section ordering. - 'section_order': [ + section_order=[ 'vim.c', 'vimscript.c', 'command.c', @@ -121,31 +174,22 @@ CONFIG = { 'autocmd.c', 'ui.c', ], - # List of files/directories for doxygen to read, relative to `base_dir` - 'files': ['src/nvim/api'], - # file patterns used by doxygen - 'file_patterns': '*.h *.c', - # Only function with this prefix are considered - 'fn_name_prefix': 'nvim_', - # Section name overrides. - 'section_name': { + files=['src/nvim/api'], + file_patterns = '*.h *.c', + fn_name_prefix = 'nvim_', + section_name={ 'vim.c': 'Global', }, - # For generated section names. - 'section_fmt': lambda name: f'{name} Functions', - # Section helptag. - 'helptag_fmt': lambda name: f'*api-{name.lower()}*', - # Per-function helptag. - 'fn_helptag_fmt': lambda fstem, name, istbl: f'*{name}()*', - # Module name overrides (for Lua). - 'module_override': {}, - # Append the docs for these modules, do not start a new section. - 'append_only': [], - }, - 'lua': { - 'mode': 'lua', - 'filename': 'lua.txt', - 'section_order': [ + section_fmt=lambda name: f'{name} Functions', + helptag_fmt=lambda name: f'*api-{name.lower()}*', + fn_helptag_fmt=lambda fstem, name, istbl: f'*{name}()*', + module_override={}, + append_only=[], + ), + 'lua': Config( + mode='lua', + filename='lua.txt', + section_order=[ 'highlight.lua', 'regex.lua', 'diff.lua', @@ -171,7 +215,7 @@ CONFIG = { 'snippet.lua', 'text.lua', ], - 'files': [ + files=[ 'runtime/lua/vim/iter.lua', 'runtime/lua/vim/_editor.lua', 'runtime/lua/vim/_options.lua', @@ -197,30 +241,30 @@ CONFIG = { 'runtime/lua/vim/_meta/regex.lua', 'runtime/lua/vim/_meta/spell.lua', ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'fn_name_fmt': lambda fstem, name: ( + file_patterns='*.lua', + fn_name_prefix='', + fn_name_fmt=lambda fstem, name: ( name if fstem in [ 'vim.iter' ] else f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else f'vim.{name}' if fstem == '_options' and not name[0].isupper() else f'{fstem}.{name}' if fstem.startswith('vim') else name ), - 'section_name': { + section_name={ 'lsp.lua': 'core', '_inspector.lua': 'inspector', }, - 'section_fmt': lambda name: ( + section_fmt=lambda name: ( 'Lua module: vim' if name.lower() == '_editor' else 'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else f'VIM.{name.upper()}' if name.lower() in [ 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', 'regex' ] else 'VIM' if name.lower() == 'builtin' else f'Lua module: vim.{name.lower()}'), - 'helptag_fmt': lambda name: ( + helptag_fmt=lambda name: ( '*lua-vim*' if name.lower() == '_editor' else '*lua-vimscript*' if name.lower() == '_options' else f'*vim.{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( + fn_helptag_fmt=lambda fstem, name, istbl: ( f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else # Exclude fstem for methods f'*{name}()*' if ':' in name else @@ -230,7 +274,7 @@ CONFIG = { f'*{fstem}()*' if fstem.endswith('.' + name) else f'*{fstem}.{name}{"" if istbl else "()"}*' ), - 'module_override': { + module_override={ # `shared` functions are exposed on the `vim` module. 'shared': 'vim', '_inspector': 'vim', @@ -255,14 +299,14 @@ CONFIG = { 'text': 'vim.text', 'glob': 'vim.glob', }, - 'append_only': [ + append_only=[ 'shared.lua', ], - }, - 'lsp': { - 'mode': 'lua', - 'filename': 'lsp.txt', - 'section_order': [ + ), + 'lsp': Config( + mode='lua', + filename='lsp.txt', + section_order=[ 'lsp.lua', 'buf.lua', 'diagnostic.lua', @@ -276,50 +320,50 @@ CONFIG = { 'rpc.lua', 'protocol.lua', ], - 'files': [ + files=[ 'runtime/lua/vim/lsp', 'runtime/lua/vim/lsp.lua', ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'section_name': {'lsp.lua': 'lsp'}, - 'section_fmt': lambda name: ( + file_patterns='*.lua', + fn_name_prefix='', + section_name={'lsp.lua': 'lsp'}, + section_fmt=lambda name: ( 'Lua module: vim.lsp' if name.lower() == 'lsp' else f'Lua module: vim.lsp.{name.lower()}'), - 'helptag_fmt': lambda name: ( + helptag_fmt=lambda name: ( '*lsp-core*' if name.lower() == 'lsp' else f'*lsp-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( + fn_helptag_fmt=lambda fstem, name, istbl: ( f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else # HACK. TODO(justinmk): class/structure support in lua2dox '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'), - 'module_override': {}, - 'append_only': [], - }, - 'diagnostic': { - 'mode': 'lua', - 'filename': 'diagnostic.txt', - 'section_order': [ + module_override={}, + append_only=[], + ), + 'diagnostic': Config( + mode='lua', + filename='diagnostic.txt', + section_order=[ 'diagnostic.lua', ], - 'files': ['runtime/lua/vim/diagnostic.lua'], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'include_tables': False, - 'section_name': {'diagnostic.lua': 'diagnostic'}, - 'section_fmt': lambda _: 'Lua module: vim.diagnostic', - 'helptag_fmt': lambda _: '*diagnostic-api*', - 'fn_helptag_fmt': lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*', - 'module_override': {}, - 'append_only': [], - }, - 'treesitter': { - 'mode': 'lua', - 'filename': 'treesitter.txt', - 'section_order': [ + files=['runtime/lua/vim/diagnostic.lua'], + file_patterns='*.lua', + fn_name_prefix='', + include_tables=False, + section_name={'diagnostic.lua': 'diagnostic'}, + section_fmt=lambda _: 'Lua module: vim.diagnostic', + helptag_fmt=lambda _: '*diagnostic-api*', + fn_helptag_fmt=lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*', + module_override={}, + append_only=[], + ), + 'treesitter': Config( + mode='lua', + filename='treesitter.txt', + section_order=[ 'treesitter.lua', 'language.lua', 'query.lua', @@ -327,30 +371,30 @@ CONFIG = { 'languagetree.lua', 'dev.lua', ], - 'files': [ + files=[ 'runtime/lua/vim/treesitter.lua', 'runtime/lua/vim/treesitter/', ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'section_name': {}, - 'section_fmt': lambda name: ( + file_patterns='*.lua', + fn_name_prefix='', + section_name={}, + section_fmt=lambda name: ( 'Lua module: vim.treesitter' if name.lower() == 'treesitter' else f'Lua module: vim.treesitter.{name.lower()}'), - 'helptag_fmt': lambda name: ( + helptag_fmt=lambda name: ( '*lua-treesitter-core*' if name.lower() == 'treesitter' else f'*lua-treesitter-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( + fn_helptag_fmt=lambda fstem, name, istbl: ( f'*vim.{fstem}.{name}()*' if fstem == 'treesitter' else f'*{name}()*' if name[0].isupper() else f'*vim.treesitter.{fstem}.{name}()*'), - 'module_override': {}, - 'append_only': [], - } + module_override={}, + append_only=[], + ), } param_exclude = ( @@ -814,6 +858,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F return chunks, xrefs + def is_program_listing(para): """ Return True if `para` contains a "programlisting" (i.e. a Markdown code @@ -835,6 +880,7 @@ def is_program_listing(para): return len(children) == 1 and children[0].nodeName == 'programlisting' + def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='', fmt_vimhelp=False): """Renders (nested) Doxygen nodes as Vim :help text. @@ -910,6 +956,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): The `fmt_vimhelp` variable controls some special cases for use by fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) """ + config: Config = CONFIG[target] + fns = {} # Map of func_name:docstring. deprecated_fns = {} # Map of func_name:docstring. @@ -934,7 +982,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): continue istbl = return_type.startswith('table') # Special from lua2dox.lua. - if istbl and not CONFIG[target].get('include_tables', True): + if istbl and not config.include_tables: continue if return_type.startswith(('ArrayOf', 'DictionaryOf')): @@ -962,7 +1010,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): declname = get_child(param, 'declname') if declname: param_name = get_text(declname).strip() - elif CONFIG[target]['mode'] == 'lua': + elif config.mode == 'lua': # XXX: this is what lua2dox gives us... param_name = param_type param_type = '' @@ -998,11 +1046,11 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): fstem = '?' if '.' in compoundname: fstem = compoundname.split('.')[0] - fstem = CONFIG[target]['module_override'].get(fstem, fstem) - vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name, istbl) + fstem = config.module_override.get(fstem, fstem) + vimtag = config.fn_helptag_fmt(fstem, name, istbl) - if 'fn_name_fmt' in CONFIG[target]: - name = CONFIG[target]['fn_name_fmt'](fstem, name) + if config.fn_name_fmt: + name = config.fn_name_fmt(fstem, name) if istbl: aopen, aclose = '', '' @@ -1085,7 +1133,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): if 'Deprecated' in str(xrefs_all): deprecated_fns[name] = fn - elif name.startswith(CONFIG[target]['fn_name_prefix']): + elif name.startswith(config.fn_name_prefix): fns[name] = fn fns = collections.OrderedDict(sorted( @@ -1102,6 +1150,8 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): 1. Vim help text for functions found in `filename`. 2. Vim help text for deprecated functions. """ + config: Config = CONFIG[target] + fns_txt = {} # Map of func_name:vim-help-text. deprecated_fns_txt = {} # Map of func_name:vim-help-text. fns, _ = extract_from_xml(filename, target, text_width, True) @@ -1164,7 +1214,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): func_doc = "\n".join(map(align_tags, split_lines)) - if (name.startswith(CONFIG[target]['fn_name_prefix']) + if (name.startswith(config.fn_name_prefix) and name != "nvim_error_event"): fns_txt[name] = func_doc @@ -1237,9 +1287,12 @@ def main(doxygen_config, args): for target in CONFIG: if args.target is not None and target != args.target: continue + + config: Config = CONFIG[target] + mpack_file = os.path.join( base_dir, 'runtime', 'doc', - CONFIG[target]['filename'].replace('.txt', '.mpack')) + config.filename.replace('.txt', '.mpack')) if os.path.exists(mpack_file): os.remove(mpack_file) @@ -1255,11 +1308,10 @@ def main(doxygen_config, args): stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL)) p.communicate( doxygen_config.format( - input=' '.join( - [f'"{file}"' for file in CONFIG[target]['files']]), + input=' '.join([f'"{file}"' for file in config.files]), output=output_dir, filter=filter_cmd, - file_patterns=CONFIG[target]['file_patterns']) + file_patterns=config.file_patterns) .encode('utf8') ) if p.returncode: @@ -1294,9 +1346,9 @@ def main(doxygen_config, args): filename = os.path.basename(filename) name = os.path.splitext(filename)[0].lower() sectname = name.upper() if name == 'ui' else name.title() - sectname = CONFIG[target]['section_name'].get(filename, sectname) - title = CONFIG[target]['section_fmt'](sectname) - section_tag = CONFIG[target]['helptag_fmt'](sectname) + sectname = config.section_name.get(filename, sectname) + title = config.section_fmt(sectname) + section_tag = config.helptag_fmt(sectname) # Module/Section id matched against @defgroup. # "*api-buffer*" => "api-buffer" section_id = section_tag.strip('*') @@ -1319,22 +1371,22 @@ def main(doxygen_config, args): if len(sections) == 0: fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') - if len(sections) > len(CONFIG[target]['section_order']): + if len(sections) > len(config.section_order): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( - set(sections).difference(CONFIG[target]['section_order']))) - first_section_tag = sections[CONFIG[target]['section_order'][0]][1] + set(sections).difference(config.section_order))) + first_section_tag = sections[config.section_order[0]][1] docs = '' - for filename in CONFIG[target]['section_order']: + for filename in config.section_order: try: title, section_tag, section_doc = sections.pop(filename) except KeyError: msg(f'warning: empty docs, skipping (target={target}): {filename}') msg(f' existing docs: {sections.keys()}') continue - if filename not in CONFIG[target]['append_only']: + if filename not in config.append_only: docs += sep docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title))) docs += section_doc @@ -1343,8 +1395,7 @@ def main(doxygen_config, args): docs = docs.rstrip() + '\n\n' docs += f' vim:tw=78:ts=8:sw={indentation}:sts={indentation}:et:ft=help:norl:\n' - doc_file = os.path.join(base_dir, 'runtime', 'doc', - CONFIG[target]['filename']) + doc_file = os.path.join(base_dir, 'runtime', 'doc', config.filename) if os.path.exists(doc_file): delete_lines_below(doc_file, first_section_tag) -- cgit From 1a31d4cf2b88e9f677d7a2f23b10e4dacf803a9d Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 28 Dec 2023 14:11:13 -0500 Subject: refactor(gen_vimdoc): use typing for function API vimdoc generation --- scripts/gen_vimdoc.py | 143 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 41 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index f22d11ed6b..40ece01ba3 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -112,6 +112,11 @@ lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') SectionName = str +Docstring = str # Represents (formatted) vimdoc string + +FunctionName = str + + @dataclasses.dataclass class Config: """Config for documentation.""" @@ -881,6 +886,44 @@ def is_program_listing(para): return len(children) == 1 and children[0].nodeName == 'programlisting' +FunctionParam = Tuple[ + str, # type + str, # parameter name +] + +@dataclasses.dataclass +class FunctionDoc: + """Data structure for function documentation. Also exported as msgpack.""" + + annotations: List[str] + """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map""" + + signature: str + """Function signature with *tags*.""" + + parameters: List[FunctionParam] + """Parameters: (type, name)""" + + parameters_doc: Dict[str, Docstring] + """Parameters documentation. Key is parameter name, value is doc.""" + + doc: List[Docstring] + """Main description for the function. Separated by paragraph.""" + + return_: List[Docstring] + """Return:, or Return (multiple): (@return strings)""" + + seealso: List[Docstring] + """See also: (@see strings)""" + + # for fmt_node_as_vimhelp + desc_node: Element | None = None + brief_desc_node: Element | None = None + + # for INCLUDE_C_DECL + c_decl: str | None = None + + def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='', fmt_vimhelp=False): """Renders (nested) Doxygen nodes as Vim :help text. @@ -946,7 +989,10 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent= return clean_lines('\n'.join(rendered_blocks).strip()) -def extract_from_xml(filename, target, width, fmt_vimhelp): +def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ + Dict[FunctionName, FunctionDoc], + Dict[FunctionName, FunctionDoc], +]: """Extracts Doxygen info as maps without formatting the text. Returns two maps: @@ -958,8 +1004,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): """ config: Config = CONFIG[target] - fns = {} # Map of func_name:docstring. - deprecated_fns = {} # Map of func_name:docstring. + fns: Dict[FunctionName, FunctionDoc] = {} + deprecated_fns: Dict[FunctionName, FunctionDoc] = {} dom = minidom.parse(filename) compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) @@ -1084,7 +1130,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): # Tracks `xrefsect` titles. As of this writing, used only for separating # deprecated functions. xrefs_all = set() - paras = [] + paras: List[Dict[str, Any]] = [] brief_desc = find_first(member, 'briefdescription') if brief_desc: for child in brief_desc.childNodes: @@ -1103,47 +1149,48 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): desc.toprettyxml(indent=' ', newl='\n')), ' ' * indentation)) - fn = { - 'annotations': list(annotations), - 'signature': signature, - 'parameters': params, - 'parameters_doc': collections.OrderedDict(), - 'doc': [], - 'return': [], - 'seealso': [], - } + fn = FunctionDoc( + annotations=list(annotations), + signature=signature, + parameters=params, + parameters_doc=collections.OrderedDict(), + doc=[], + return_=[], + seealso=[], + ) if fmt_vimhelp: - fn['desc_node'] = desc - fn['brief_desc_node'] = brief_desc + fn.desc_node = desc + fn.brief_desc_node = brief_desc for m in paras: - if 'text' in m: - if not m['text'] == '': - fn['doc'].append(m['text']) + if m.get('text', ''): + fn.doc.append(m['text']) if 'params' in m: # Merge OrderedDicts. - fn['parameters_doc'].update(m['params']) + fn.parameters_doc.update(m['params']) if 'return' in m and len(m['return']) > 0: - fn['return'] += m['return'] + fn.return_ += m['return'] if 'seealso' in m and len(m['seealso']) > 0: - fn['seealso'] += m['seealso'] + fn.seealso += m['seealso'] if INCLUDE_C_DECL: - fn['c_decl'] = c_decl + fn.c_decl = c_decl if 'Deprecated' in str(xrefs_all): deprecated_fns[name] = fn elif name.startswith(config.fn_name_prefix): fns[name] = fn + # sort functions by name (lexicographically) fns = collections.OrderedDict(sorted( fns.items(), - key=lambda key_item_tuple: key_item_tuple[0].lower())) + key=lambda key_item_tuple: key_item_tuple[0].lower(), + )) deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items())) return fns, deprecated_fns -def fmt_doxygen_xml_as_vimhelp(filename, target): +def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: """Entrypoint for generating Vim :help from from Doxygen XML. Returns 2 items: @@ -1154,20 +1201,26 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): fns_txt = {} # Map of func_name:vim-help-text. deprecated_fns_txt = {} # Map of func_name:vim-help-text. + + fns: Dict[FunctionName, FunctionDoc] fns, _ = extract_from_xml(filename, target, text_width, True) - for name, fn in fns.items(): + for fn_name, fn in fns.items(): # Generate Vim :help for parameters. - if fn['desc_node']: - doc = fmt_node_as_vimhelp(fn['desc_node'], fmt_vimhelp=True) - if not doc and fn['brief_desc_node']: - doc = fmt_node_as_vimhelp(fn['brief_desc_node']) - if not doc and name.startswith("nvim__"): + + # Generate body. + doc = '' + if fn.desc_node: + doc = fmt_node_as_vimhelp(fn.desc_node, fmt_vimhelp=True) + if not doc and fn.brief_desc_node: + doc = fmt_node_as_vimhelp(fn.brief_desc_node) + if not doc and fn_name.startswith("nvim__"): continue if not doc: doc = 'TODO: Documentation' - annotations = '\n'.join(fn['annotations']) + # Annotations: put before Parameters + annotations: str = '\n'.join(fn.annotations) if annotations: annotations = ('\n\nAttributes: ~\n' + textwrap.indent(annotations, ' ')) @@ -1177,18 +1230,22 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): else: doc = doc[:i] + annotations + '\n\n' + doc[i:] + # C Declaration: (debug only) if INCLUDE_C_DECL: doc += '\n\nC Declaration: ~\n>\n' - doc += fn['c_decl'] + assert fn.c_decl is not None + doc += fn.c_decl doc += '\n<' - func_doc = fn['signature'] + '\n' + # Start of function documentations. e.g., + # nvim_cmd({*cmd}, {*opts}) *nvim_cmd()* + func_doc = fn.signature + '\n' func_doc += textwrap.indent(clean_lines(doc), ' ' * indentation) # Verbatim handling. func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) - split_lines = func_doc.split('\n') + split_lines: List[str] = func_doc.split('\n') start = 0 while True: try: @@ -1214,12 +1271,14 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): func_doc = "\n".join(map(align_tags, split_lines)) - if (name.startswith(config.fn_name_prefix) - and name != "nvim_error_event"): - fns_txt[name] = func_doc + if (fn_name.startswith(config.fn_name_prefix) + and fn_name != "nvim_error_event"): + fns_txt[fn_name] = func_doc - return ('\n\n'.join(list(fns_txt.values())), - '\n\n'.join(list(deprecated_fns_txt.values()))) + return ( + '\n\n'.join(list(fns_txt.values())), + '\n\n'.join(list(deprecated_fns_txt.values())), + ) def delete_lines_below(filename, tokenstr): @@ -1402,9 +1461,11 @@ def main(doxygen_config, args): with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) - fn_map_full = collections.OrderedDict(sorted(fn_map_full.items())) + fn_map_full = collections.OrderedDict(sorted( + (name, fn_doc.__dict__) for (name, fn_doc) in fn_map_full.items() + )) with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) + fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) # type: ignore if not args.keep_tmpfiles: shutil.rmtree(output_dir) -- cgit From 4e9298ecdf945b4d16c2c6e6e4ed82b97880917c Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 28 Dec 2023 17:50:05 -0500 Subject: refactor(gen_vimdoc): generate function doc from metadata, not from xml Problem: For function definitions to be included in the vimdoc (formatted) and to be exported as mpack data (unformatted), we had two internal representations of the same function/API metadata in duplicate; one is FunctionDoc (which was previously a dict), and the other is doxygen XML DOM from which vimdoc (functions sections) was generated. Solution: We should have a single path and unified data representation (i.e. FunctionDoc) that contains all the metadata and information about function APIs, from which both of mpack export and vimdoc are generated. I.e., vimdocs are no longer generated directly from doxygen XML nodes, but generated via: (XML DOM Nodes) ------------> FunctionDoc ------> mpack (unformatted) Recursive Internal | Formatting Metadata +---> vimdoc (formatted) This refactoring eliminates the hacky and ugly use of `fmt_vimhelp` in `fmt_node_as_vimhelp()` and all other helper functions! This way, `fmt_node_as_vimhelp()` can simplified as it no longer needs to handle generating of function docs, which needs to be done only in the topmost level of recursion. --- scripts/gen_vimdoc.py | 220 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 151 insertions(+), 69 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 40ece01ba3..eede5b76a2 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -751,7 +751,10 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, return text -def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=False): +def para_as_map(parent: Element, + indent: str = '', + width: int = (text_width - indentation), + ): """Extracts a Doxygen XML node to a map. Keys: @@ -787,7 +790,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F kind = '' if is_inline(parent): # Flatten inline text from a tree of non-block nodes. - text = doc_wrap(render_node(parent, "", fmt_vimhelp=fmt_vimhelp), + text = doc_wrap(render_node(parent, ""), indent=indent, width=width) else: prev = None # Previous node @@ -805,8 +808,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F elif kind == 'see': groups['seealso'].append(child) elif kind == 'warning': - text += render_node(child, text, indent=indent, - width=width, fmt_vimhelp=fmt_vimhelp) + text += render_node(child, text, indent=indent, width=width) elif kind == 'since': since_match = re.match(r'^(\d+)', get_text(child)) since = int(since_match.group(1)) if since_match else 0 @@ -827,8 +829,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F and ' ' != text[-1]): text += ' ' - text += render_node(child, text, indent=indent, width=width, - fmt_vimhelp=fmt_vimhelp) + text += render_node(child, text, indent=indent, width=width) prev = child chunks['text'] += text @@ -839,17 +840,17 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F update_params_map(child, ret_map=chunks['params'], width=width) for child in groups['note']: chunks['note'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp).rstrip()) + child, '', indent=indent, width=width).rstrip()) for child in groups['return']: chunks['return'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp)) + child, '', indent=indent, width=width)) for child in groups['seealso']: # Example: # # |autocommand| # chunks['seealso'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp)) + child, '', indent=indent, width=width)) xrefs = set() for child in groups['xrefs']: @@ -898,6 +899,9 @@ class FunctionDoc: annotations: List[str] """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map""" + notes: List[Docstring] + """Notes: (@note strings)""" + signature: str """Function signature with *tags*.""" @@ -916,41 +920,122 @@ class FunctionDoc: seealso: List[Docstring] """See also: (@see strings)""" - # for fmt_node_as_vimhelp - desc_node: Element | None = None - brief_desc_node: Element | None = None + xrefs: List[Docstring] + """XRefs. Currently only used to track Deprecated functions.""" # for INCLUDE_C_DECL c_decl: str | None = None + prerelease: bool = False + + def export_mpack(self) -> Dict[str, Any]: + """Convert a dict to be exported as mpack data.""" + exported = self.__dict__.copy() + del exported['notes'] + del exported['c_decl'] + del exported['prerelease'] + del exported['xrefs'] + exported['return'] = exported.pop('return_') + return exported + + def doc_concatenated(self) -> Docstring: + """Concatenate all the paragraphs in `doc` into a single string, but + remove blank lines before 'programlisting' blocks. #25127 + + BEFORE (without programlisting processing): + ```vimdoc + Example: + + >vim + :echo nvim_get_color_by_name("Pink") + < + ``` + + AFTER: + ```vimdoc + Example: >vim + :echo nvim_get_color_by_name("Pink") + < + ``` + """ + def is_program_listing(paragraph: str) -> bool: + lines = paragraph.strip().split('\n') + return lines[0].startswith('>') and lines[-1] == '<' + + rendered = [] + for paragraph in self.doc: + if is_program_listing(paragraph): + rendered.append(' ') # Example: >vim + elif rendered: + rendered.append('\n\n') + rendered.append(paragraph) + return ''.join(rendered) + + def render(self) -> Docstring: + """Renders function documentation as Vim :help text.""" + rendered_blocks: List[Docstring] = [] + + def fmt_param_doc(m): + """Renders a params map as Vim :help text.""" + max_name_len = max_name(m.keys()) + 4 + out = '' + for name, desc in m.items(): + if name == 'self': + continue + name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + out += '{}{}\n'.format(name, desc) + return out.rstrip() + + # Generate text from the gathered items. + chunks: List[Docstring] = [self.doc_concatenated()] + + notes = [] + if self.prerelease: + notes = [" This API is pre-release (unstable)."] + notes += self.notes + if len(notes) > 0: + chunks.append('\nNote: ~') + for s in notes: + chunks.append(' ' + s) + + if self.parameters_doc: + chunks.append('\nParameters: ~') + chunks.append(fmt_param_doc(self.parameters_doc)) + + if self.return_: + chunks.append('\nReturn (multiple): ~' if len(self.return_) > 1 + else '\nReturn: ~') + for s in self.return_: + chunks.append(' ' + s) + + if self.seealso: + chunks.append('\nSee also: ~') + for s in self.seealso: + chunks.append(' ' + s) + + # Note: xrefs are currently only used to remark "Deprecated: " + # for deprecated functions; visible when INCLUDE_DEPRECATED is set + for s in self.xrefs: + chunks.append('\n' + s) + + rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) + rendered_blocks.append('') -def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='', - fmt_vimhelp=False): + return clean_lines('\n'.join(rendered_blocks).strip()) + + +def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=''): """Renders (nested) Doxygen nodes as Vim :help text. + Only handles "text" nodes. Used for individual elements (see render_node()) + and in extract_defgroups(). + NB: Blank lines in a docstring manifest as tags. """ rendered_blocks = [] - def fmt_param_doc(m): - """Renders a params map as Vim :help text.""" - max_name_len = max_name(m.keys()) + 4 - out = '' - for name, desc in m.items(): - if name == 'self': - continue - name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) - out += '{}{}\n'.format(name, desc) - return out.rstrip() - - def has_nonexcluded_params(m): - """Returns true if any of the given params has at least - one non-excluded item.""" - if fmt_param_doc(m) != '': - return True - for child in parent.childNodes: - para, _ = para_as_map(child, indent, width, fmt_vimhelp) + para, _ = para_as_map(child, indent, width) # 'programlisting' blocks are Markdown code blocks. Do not include # these as a separate paragraph, but append to the last non-empty line @@ -963,25 +1048,6 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent= # Generate text from the gathered items. chunks = [para['text']] - notes = [" This API is pre-release (unstable)."] if para['prerelease'] else [] - notes += para['note'] - if len(notes) > 0: - chunks.append('\nNote: ~') - for s in notes: - chunks.append(s) - if len(para['params']) > 0 and has_nonexcluded_params(para['params']): - chunks.append('\nParameters: ~') - chunks.append(fmt_param_doc(para['params'])) - if len(para['return']) > 0: - chunks.append('\nReturn (multiple): ~' if len(para['return']) > 1 else '\nReturn: ~') - for s in para['return']: - chunks.append(s) - if len(para['seealso']) > 0: - chunks.append('\nSee also: ~') - for s in para['seealso']: - chunks.append(s) - for s in para['xrefs']: - chunks.append(s) rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) rendered_blocks.append('') @@ -989,7 +1055,8 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent= return clean_lines('\n'.join(rendered_blocks).strip()) -def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ +def extract_from_xml(filename, target, *, + width: int, fmt_vimhelp: bool) -> Tuple[ Dict[FunctionName, FunctionDoc], Dict[FunctionName, FunctionDoc], ]: @@ -1130,18 +1197,20 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ # Tracks `xrefsect` titles. As of this writing, used only for separating # deprecated functions. xrefs_all = set() - paras: List[Dict[str, Any]] = [] + paras: List[Dict[str, Any]] = [] # paras means paragraphs! brief_desc = find_first(member, 'briefdescription') if brief_desc: for child in brief_desc.childNodes: para, xrefs = para_as_map(child) + paras.append(para) xrefs_all.update(xrefs) desc = find_first(member, 'detaileddescription') if desc: + paras_detail = [] # override briefdescription for child in desc.childNodes: para, xrefs = para_as_map(child) - paras.append(para) + paras_detail.append(para) xrefs_all.update(xrefs) log.debug( textwrap.indent( @@ -1149,18 +1218,25 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ desc.toprettyxml(indent=' ', newl='\n')), ' ' * indentation)) + # override briefdescription, if detaileddescription is not empty + # (note: briefdescription can contain some erroneous luadoc + # comments from preceding comments, this is a bug of lua2dox) + if any((para['text'] or para['note'] or para['params'] or + para['return'] or para['seealso'] + ) for para in paras_detail): + paras = paras_detail + fn = FunctionDoc( annotations=list(annotations), + notes=[], signature=signature, parameters=params, parameters_doc=collections.OrderedDict(), doc=[], return_=[], seealso=[], + xrefs=[], ) - if fmt_vimhelp: - fn.desc_node = desc - fn.brief_desc_node = brief_desc for m in paras: if m.get('text', ''): @@ -1172,6 +1248,12 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ fn.return_ += m['return'] if 'seealso' in m and len(m['seealso']) > 0: fn.seealso += m['seealso'] + if m.get('prerelease', False): + fn.prerelease = True + if 'note' in m: + fn.notes += m['note'] + if 'xrefs' in m: + fn.xrefs += m['xrefs'] if INCLUDE_C_DECL: fn.c_decl = c_decl @@ -1203,17 +1285,14 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: deprecated_fns_txt = {} # Map of func_name:vim-help-text. fns: Dict[FunctionName, FunctionDoc] - fns, _ = extract_from_xml(filename, target, text_width, True) + fns, _ = extract_from_xml(filename, target, + width=text_width, fmt_vimhelp=True) for fn_name, fn in fns.items(): # Generate Vim :help for parameters. - # Generate body. - doc = '' - if fn.desc_node: - doc = fmt_node_as_vimhelp(fn.desc_node, fmt_vimhelp=True) - if not doc and fn.brief_desc_node: - doc = fmt_node_as_vimhelp(fn.brief_desc_node) + # Generate body from FunctionDoc, not XML nodes + doc = fn.render() if not doc and fn_name.startswith("nvim__"): continue if not doc: @@ -1393,11 +1472,14 @@ def main(doxygen_config, args): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) + # Extract unformatted (*.mpack). - fn_map, _ = extract_from_xml(xmlfile, target, 9999, False) + fn_map, _ = extract_from_xml( + xmlfile, target, width=9999, fmt_vimhelp=False) + # Extract formatted (:help). functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( - os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))), target) + xmlfile, target) if not functions_text and not deprecated_text: continue @@ -1461,11 +1543,11 @@ def main(doxygen_config, args): with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) - fn_map_full = collections.OrderedDict(sorted( - (name, fn_doc.__dict__) for (name, fn_doc) in fn_map_full.items() + fn_map_full_exported = collections.OrderedDict(sorted( + (name, fn_doc.export_mpack()) for (name, fn_doc) in fn_map_full.items() )) with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) # type: ignore + fp.write(msgpack.packb(fn_map_full_exported, use_bin_type=True)) # type: ignore if not args.keep_tmpfiles: shutil.rmtree(output_dir) -- cgit From f74f52a1a5ff668187bf12c397319972764a9704 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Fri, 29 Dec 2023 05:52:31 -0500 Subject: refactor(gen_vimdoc): refactor section and defgroup doc generation Problem: main() has too much logic implemented there, too difficult to read. Solution: Do more OOP, introduce `Section` dataclass that stores information about a "section", with documentation and concrete examples about what each field and variable would mean. Extract all the lines for rendering a section into `section.render()` pulled out of `main()`. --- scripts/gen_vimdoc.py | 185 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 138 insertions(+), 47 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index eede5b76a2..e5f2e61dfd 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -102,6 +102,8 @@ LOG_LEVELS = { text_width = 78 indentation = 4 +SECTION_SEP = '=' * text_width + script_path = os.path.abspath(__file__) base_dir = os.path.dirname(os.path.dirname(script_path)) out_dir = os.path.join(base_dir, 'tmp-{target}-doc') @@ -1378,7 +1380,7 @@ def delete_lines_below(filename, tokenstr): fp.writelines(lines[0:i]) -def extract_defgroups(base: str, dom: Document): +def extract_defgroups(base: str, dom: Document) -> Dict[SectionName, Docstring]: '''Generate module-level (section) docs (@defgroup).''' section_docs = {} @@ -1414,6 +1416,101 @@ def extract_defgroups(base: str, dom: Document): return section_docs +@dataclasses.dataclass +class Section: + """Represents a section. Includes section heading (defgroup) + and all the FunctionDoc that belongs to this section.""" + + name: str + '''Name of the section. Usually derived from basename of lua/c src file. + Example: "Autocmd".''' + + title: str + '''Formatted section config. see config.section_fmt(). + Example: "Autocmd Functions". ''' + + helptag: str + '''see config.helptag_fmt(). Example: *api-autocmd*''' + + @property + def id(self) -> str: + '''section id: Module/Section id matched against @defgroup. + e.g., "*api-autocmd*" => "api-autocmd" + ''' + return self.helptag.strip('*') + + doc: str = "" + '''Section heading docs extracted from @defgroup.''' + + # TODO: Do not carry rendered text, but handle FunctionDoc for better OOP + functions_text: Docstring | None = None + '''(Rendered) doc of all the functions that belong to this section.''' + + deprecated_functions_text: Docstring | None = None + '''(Rendered) doc of all the deprecated functions that belong to this + section.''' + + def __repr__(self): + return f"Section(title='{self.title}', helptag='{self.helptag}')" + + @classmethod + def make_from(cls, filename: str, config: Config, + section_docs: Dict[SectionName, str], + *, + functions_text: Docstring, + deprecated_functions_text: Docstring, + ): + # filename: e.g., 'autocmd.c' + # name: e.g. 'autocmd' + name = os.path.splitext(filename)[0].lower() + + # section name: e.g. "Autocmd" + sectname: SectionName + sectname = name.upper() if name == 'ui' else name.title() + sectname = config.section_name.get(filename, sectname) + + # Formatted (this is what's going to be written in the vimdoc) + # e.g., "Autocmd Functions" + title: str = config.section_fmt(sectname) + + # section tag: e.g., "*api-autocmd*" + section_tag: str = config.helptag_fmt(sectname) + + section = cls(name=sectname, title=title, helptag=section_tag, + functions_text=functions_text, + deprecated_functions_text=deprecated_functions_text, + ) + section.doc = section_docs.get(section.id) or '' + return section + + def render(self, add_header=True) -> str: + """Render as vimdoc.""" + doc = '' + + if add_header: + doc += SECTION_SEP + doc += '\n{}{}'.format( + self.title, + self.helptag.rjust(text_width - len(self.title)) + ) + + if self.doc: + doc += '\n\n' + self.doc + + if self.functions_text: + doc += '\n\n' + self.functions_text + + if INCLUDE_DEPRECATED and self.deprecated_functions_text: + doc += f'\n\n\nDeprecated {self.name} Functions: ~\n\n' + doc += self.deprecated_functions_text + + return doc + + def __bool__(self) -> bool: + """Whether this section has contents. Used for skipping empty ones.""" + return bool(self.doc or self.functions_text) + + def main(doxygen_config, args): """Generates: @@ -1455,14 +1552,16 @@ def main(doxygen_config, args): if p.returncode: sys.exit(p.returncode) - fn_map_full = {} # Collects all functions as each module is processed. - sections = {} - sep = '=' * text_width + # Collects all functions as each module is processed. + fn_map_full: Dict[FunctionName, FunctionDoc] = {} + # key: filename (e.g. autocmd.c) + sections: Dict[str, Section] = {} base = os.path.join(output_dir, 'xml') dom = minidom.parse(os.path.join(base, 'index.xml')) - section_docs = extract_defgroups(base, dom) + # Collect all @defgroups (section headings after the '===...' separator + section_docs: Dict[SectionName, Docstring] = extract_defgroups(base, dom) # Generate docs for all functions in the current module. for compound in dom.getElementsByTagName('compound'): @@ -1470,45 +1569,38 @@ def main(doxygen_config, args): continue filename = get_text(find_first(compound, 'name')) - if filename.endswith('.c') or filename.endswith('.lua'): - xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) + if not ( + filename.endswith('.c') or + filename.endswith('.lua') + ): + continue - # Extract unformatted (*.mpack). - fn_map, _ = extract_from_xml( - xmlfile, target, width=9999, fmt_vimhelp=False) + xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) - # Extract formatted (:help). - functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( - xmlfile, target) + # Extract unformatted (*.mpack). + fn_map, _ = extract_from_xml( + xmlfile, target, width=9999, fmt_vimhelp=False) - if not functions_text and not deprecated_text: - continue - else: - filename = os.path.basename(filename) - name = os.path.splitext(filename)[0].lower() - sectname = name.upper() if name == 'ui' else name.title() - sectname = config.section_name.get(filename, sectname) - title = config.section_fmt(sectname) - section_tag = config.helptag_fmt(sectname) - # Module/Section id matched against @defgroup. - # "*api-buffer*" => "api-buffer" - section_id = section_tag.strip('*') - - doc = '' - section_doc = section_docs.get(section_id) - if section_doc: - doc += '\n\n' + section_doc - - if functions_text: - doc += '\n\n' + functions_text - - if INCLUDE_DEPRECATED and deprecated_text: - doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n' - doc += deprecated_text - - if doc: - sections[filename] = (title, section_tag, doc) - fn_map_full.update(fn_map) + # Extract formatted (:help). + functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( + xmlfile, target) + + if not functions_text and not deprecated_text: + continue + + filename = os.path.basename(filename) + + section: Section = Section.make_from( + filename, config, section_docs, + functions_text=functions_text, + deprecated_functions_text=deprecated_text, + ) + + if section: # if not empty + sections[filename] = section + fn_map_full.update(fn_map) + else: + log.debug("Skipping empty section: %s", section) if len(sections) == 0: fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') @@ -1516,21 +1608,20 @@ def main(doxygen_config, args): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( set(sections).difference(config.section_order))) - first_section_tag = sections[config.section_order[0]][1] + first_section_tag = sections[config.section_order[0]].helptag docs = '' for filename in config.section_order: try: - title, section_tag, section_doc = sections.pop(filename) + section: Section = sections.pop(filename) except KeyError: msg(f'warning: empty docs, skipping (target={target}): {filename}') msg(f' existing docs: {sections.keys()}') continue - if filename not in config.append_only: - docs += sep - docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title))) - docs += section_doc + + add_sep_and_header = filename not in config.append_only + docs += section.render(add_header=add_sep_and_header) docs += '\n\n\n' docs = docs.rstrip() + '\n\n' -- cgit From 765729a145d3d8204ff68b1da0b5bb45c70262e2 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Fri, 29 Dec 2023 06:32:37 -0500 Subject: fix(gen_vimdoc): INCLUDE_DEPRECATED not generating docs for deprecateds Since some point INCLUDE_DEPRECATED stopped working as it is usually turned off when generating an actual vimdoc. This commit fixes this hidden feature back again (used for devel purposes only). --- scripts/gen_vimdoc.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index e5f2e61dfd..804f0fd198 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -89,8 +89,8 @@ else: # DEBUG = ('DEBUG' in os.environ) -INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) -INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) +INCLUDE_C_DECL = os.environ.get('INCLUDE_C_DECL', '0') != '0' +INCLUDE_DEPRECATED = os.environ.get('INCLUDE_DEPRECATED', '0') != '0' log = logging.getLogger(__name__) @@ -168,7 +168,7 @@ CONFIG: Dict[str, Config] = { mode = 'c', filename = 'api.txt', # Section ordering. - section_order=[ + section_order=[x for x in [ 'vim.c', 'vimscript.c', 'command.c', @@ -180,7 +180,8 @@ CONFIG: Dict[str, Config] = { 'tabpage.c', 'autocmd.c', 'ui.c', - ], + 'deprecated.c' if INCLUDE_DEPRECATED else '' + ] if x], files=['src/nvim/api'], file_patterns = '*.h *.c', fn_name_prefix = 'nvim_', @@ -1287,18 +1288,21 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: deprecated_fns_txt = {} # Map of func_name:vim-help-text. fns: Dict[FunctionName, FunctionDoc] - fns, _ = extract_from_xml(filename, target, - width=text_width, fmt_vimhelp=True) + deprecated_fns: Dict[FunctionName, FunctionDoc] + fns, deprecated_fns = extract_from_xml( + filename, target, width=text_width, fmt_vimhelp=True) - for fn_name, fn in fns.items(): + def _handle_fn(fn_name: FunctionName, fn: FunctionDoc, + fns_txt: Dict[FunctionName, Docstring], deprecated=False): # Generate Vim :help for parameters. # Generate body from FunctionDoc, not XML nodes doc = fn.render() if not doc and fn_name.startswith("nvim__"): - continue + return if not doc: - doc = 'TODO: Documentation' + doc = ('TODO: Documentation' if not deprecated + else 'Deprecated.') # Annotations: put before Parameters annotations: str = '\n'.join(fn.annotations) @@ -1356,6 +1360,11 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: and fn_name != "nvim_error_event"): fns_txt[fn_name] = func_doc + for fn_name, fn in fns.items(): + _handle_fn(fn_name, fn, fns_txt) + for fn_name, fn in deprecated_fns.items(): + _handle_fn(fn_name, fn, deprecated_fns_txt, deprecated=True) + return ( '\n\n'.join(list(fns_txt.values())), '\n\n'.join(list(deprecated_fns_txt.values())), @@ -1508,7 +1517,8 @@ class Section: def __bool__(self) -> bool: """Whether this section has contents. Used for skipping empty ones.""" - return bool(self.doc or self.functions_text) + return bool(self.doc or self.functions_text or + (INCLUDE_DEPRECATED and self.deprecated_functions_text)) def main(doxygen_config, args): @@ -1606,8 +1616,11 @@ def main(doxygen_config, args): fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') if len(sections) > len(config.section_order): raise RuntimeError( - 'found new modules "{}"; update the "section_order" map'.format( - set(sections).difference(config.section_order))) + '{}: found new modules {}; ' + 'update the "section_order" map'.format( + target, + set(sections).difference(config.section_order)) + ) first_section_tag = sections[config.section_order[0]].helptag docs = '' -- cgit From f40df63bdca33d343cada6ceaafbc8b765ed7cc6 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 4 Jan 2024 11:09:13 -0500 Subject: fix(docs): make lines not overflow in vim docs Problem: Some lines in the generated vim doc are overflowing, not correctly wrapped at 78 characters. This happens when docs body contains several consecutive 'inline' elements generated by doxygen. Solution: Take into account the current column offset of the last line, and prepend some padding before doc_wrap(). --- scripts/gen_vimdoc.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 804f0fd198..698336cf3e 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -34,7 +34,7 @@ The generated :help text for each function is formatted as follows: - Each function documentation is separated by a single line. """ -from __future__ import annotations +from __future__ import annotations # PEP-563, python 3.7+ import argparse import collections @@ -47,9 +47,12 @@ import subprocess import sys import textwrap from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Tuple +from typing import Any, Callable, Dict, List, Tuple from xml.dom import minidom +if sys.version_info >= (3, 8): + from typing import Literal + import msgpack Element = minidom.Element @@ -165,7 +168,7 @@ class Config: CONFIG: Dict[str, Config] = { 'api': Config( - mode = 'c', + mode='c', filename = 'api.txt', # Section ordering. section_order=[x for x in [ @@ -576,7 +579,7 @@ def is_inline(n): return True -def doc_wrap(text, prefix='', width=70, func=False, indent=None): +def doc_wrap(text, prefix='', width=70, func=False, indent=None) -> str: """Wraps text to `width`. First line is prefixed with `prefix`, subsequent lines are aligned. @@ -651,13 +654,19 @@ def update_params_map(parent, ret_map, width=text_width - indentation): return ret_map -def render_node(n, text, prefix='', indent='', width=text_width - indentation, - fmt_vimhelp=False): +def render_node(n: Element, text: str, prefix='', *, + indent: str = '', + width: int = (text_width - indentation), + fmt_vimhelp: bool = False): """Renders a node as Vim help text, recursively traversing all descendants.""" def ind(s): return s if fmt_vimhelp else '' + # Get the current column offset from the last line of `text` + # (needed to appropriately wrap multiple and contiguous inline elements) + col_offset: int = len_lastline(text) + text = '' # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) # text += (int(not space_preceding) * ' ') @@ -682,7 +691,14 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4)) elif is_inline(n): - text = doc_wrap(get_text(n), prefix=prefix, indent=indent, width=width) + o = get_text(n).strip() + if o: + DEL = chr(127) # a dummy character to pad for proper line wrap + assert len(DEL) == 1 + dummy_padding = DEL * max(0, col_offset - len(prefix)) + text += doc_wrap(dummy_padding + o, + prefix=prefix, indent=indent, width=width + ).replace(DEL, "") elif n.nodeName == 'verbatim': # TODO: currently we don't use this. The "[verbatim]" hint is there as # a reminder that we must decide how to format this if we do use it. -- cgit From 2f9ee9b6cfc61a0504fc0bc22bdf481828e2ea91 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 9 Jan 2024 17:36:46 +0000 Subject: fix(doc): improve doc generation of types using lpeg Added a lpeg grammar for LuaCATS and use it in lua2dox.lua --- scripts/gen_vimdoc.py | 14 ++-- scripts/lua2dox.lua | 192 ++++++++++++++++++++++++++++++-------------------- 2 files changed, 123 insertions(+), 83 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 698336cf3e..01532cc3d3 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -717,12 +717,14 @@ def render_node(n: Element, text: str, prefix='', *, elif n.nodeName in ('para', 'heading'): did_prefix = False for c in n.childNodes: + c_text = render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width) if (is_inline(c) - and '' != get_text(c).strip() + and '' != c_text.strip() and text - and ' ' != text[-1]): + and text[-1] not in (' ', '(', '|') + and not c_text.startswith(')')): text += ' ' - text += render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width) + text += c_text did_prefix = True elif n.nodeName == 'itemizedlist': for c in n.childNodes: @@ -840,15 +842,17 @@ def para_as_map(parent: Element, raise RuntimeError('unhandled simplesect: {}\n{}'.format( child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) else: + child_text = render_node(child, text, indent=indent, width=width) if (prev is not None and is_inline(self_or_child(prev)) and is_inline(self_or_child(child)) and '' != get_text(self_or_child(child)).strip() and text - and ' ' != text[-1]): + and text[-1] not in (' ', '(', '|') + and not child_text.startswith(')')): text += ' ' - text += render_node(child, text, indent=indent, width=width) + text += child_text prev = child chunks['text'] += text diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index c4ad7fbb03..abc9e5b338 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -55,17 +55,7 @@ The effect is that you will get the function documented, but not with the parame local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' } -local TAGGED_TYPES = { 'TSNode', 'LanguageTree' } - --- Document these as 'table' -local ALIAS_TYPES = { - 'Range', - 'Range4', - 'Range6', - 'TSMetadata', - 'vim.filetype.add.filetypes', - 'vim.filetype.match.args', -} +local luacats_parser = require('src/nvim/generators/luacats_grammar') local debug_outfile = nil --- @type string? local debug_output = {} @@ -161,6 +151,91 @@ local function removeCommentFromLine(line) return line:sub(1, pos_comment - 1), line:sub(pos_comment) end +--- @param parsed luacats.Return +--- @return string +local function get_return_type(parsed) + local elems = {} --- @type string[] + for _, v in ipairs(parsed) do + local e = v.type --- @type string + if v.name then + e = e .. ' ' .. v.name --- @type string + end + elems[#elems + 1] = e + end + return '(' .. table.concat(elems, ', ') .. ')' +end + +--- @param name string +--- @return string +local function process_name(name, optional) + if optional then + name = name:sub(1, -2) --- @type string + end + return name +end + +--- @param ty string +--- @param generics table +--- @return string +local function process_type(ty, generics, optional) + -- replace generic types + for k, v in pairs(generics) do + ty = ty:gsub(k, v) --- @type string + end + + -- strip parens + ty = ty:gsub('^%((.*)%)$', '%1') + + if optional and not ty:find('nil') then + ty = ty .. '?' + end + + -- remove whitespace in unions + ty = ty:gsub('%s*|%s*', '|') + + -- replace '|nil' with '?' + ty = ty:gsub('|nil', '?') + ty = ty:gsub('nil|(.*)', '%1?') + + return '(`' .. ty .. '`)' +end + +--- @param parsed luacats.Param +--- @param generics table +--- @return string +local function process_param(parsed, generics) + local name, ty = parsed.name, parsed.type + local optional = vim.endswith(name, '?') + + return table.concat({ + '/// @param', + process_name(name, optional), + process_type(ty, generics, optional), + parsed.desc, + }, ' ') +end + +--- @param parsed luacats.Return +--- @param generics table +--- @return string +local function process_return(parsed, generics) + local ty, name --- @type string, string + if #parsed == 1 then + ty, name = parsed[1].type, parsed[1].name or '' + else + ty, name = get_return_type(parsed), '' + end + + local optional = vim.endswith(name, '?') + + return table.concat({ + '/// @return', + process_type(ty, generics, optional), + process_name(name, optional), + parsed.desc, + }, ' ') +end + --- Processes "@…" directives in a docstring line. --- --- @param line string @@ -175,93 +250,54 @@ local function process_magic(line, generics) return '/// ' .. line end - local magic = line:sub(2) - local magic_split = vim.split(magic, ' ', { plain = true }) + local magic_split = vim.split(line, ' ', { plain = true }) local directive = magic_split[1] if vim.list_contains({ - 'cast', - 'diagnostic', - 'overload', - 'meta', - 'type', + '@cast', + '@diagnostic', + '@overload', + '@meta', + '@type', }, directive) then -- Ignore LSP directives return '// gg:"' .. line .. '"' - end - - if directive == 'defgroup' or directive == 'addtogroup' then + elseif directive == '@defgroup' or directive == '@addtogroup' then -- Can't use '.' in defgroup, so convert to '--' - return '/// @' .. magic:gsub('%.', '-dot-') - end - - if directive == 'generic' then - local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)') - if generic_type == '' then - generic_type = 'any' - end - generics[generic_name] = generic_type - return + return '/// ' .. line:gsub('%.', '-dot-') end - local type_index = 2 - - if directive == 'param' then + -- preprocess line before parsing + if directive == '@param' or directive == '@return' then for _, type in ipairs(TYPES) do - magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2') - magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') - end - magic_split = vim.split(magic, ' ', { plain = true }) - type_index = 3 - elseif directive == 'return' then - for _, type in ipairs(TYPES) do - magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1') - magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1') - end - -- Remove first "#" comment char, if any. https://github.com/LuaLS/lua-language-server/wiki/Annotations#return - magic = magic:gsub('# ', '', 1) - -- handle the return of vim.spell.check - magic = magic:gsub('({.*}%[%])', '`%1`') - magic_split = vim.split(magic, ' ', { plain = true }) - end - - local ty = magic_split[type_index] - - if ty then - -- fix optional parameters - if magic_split[2]:find('%?$') then - if not ty:find('nil') then - ty = ty .. '|nil' - end - magic_split[2] = magic_split[2]:sub(1, -2) - end + line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') + line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') - -- replace generic types - for k, v in pairs(generics) do - ty = ty:gsub(k, v) --- @type string + line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1') + line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1') end + end - for _, type in ipairs(TAGGED_TYPES) do - ty = ty:gsub(type, '|%1|') - end + local parsed = luacats_parser:match(line) - for _, type in ipairs(ALIAS_TYPES) do - ty = ty:gsub('^' .. type .. '$', 'table') --- @type string - end + if not parsed then + return '/// ' .. line + end - -- surround some types by () - for _, type in ipairs(TYPES) do - ty = ty:gsub('^(' .. type .. '|nil):?$', '(%1)'):gsub('^(' .. type .. '):?$', '(%1)') - end + local kind = parsed.kind - magic_split[type_index] = ty + if kind == 'generic' then + generics[parsed.name] = parsed.type or 'any' + return + elseif kind == 'param' then + return process_param(parsed --[[@as luacats.Param]], generics) + elseif kind == 'return' then + return process_return(parsed --[[@as luacats.Return]], generics) end - magic = table.concat(magic_split, ' ') - - return '/// @' .. magic + error(string.format('unhandled parsed line %q: %s', line, parsed)) end --- @param line string -- cgit From 2cdea852e8934beb89012f2127f333e4dd8aada8 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 11 Jan 2024 12:24:44 -0500 Subject: docs: auto-generate docs for `vim.lpeg` and `vim.re` - Add section `VIM.LPEG` and `VIM.RE` to docs/lua.txt. - Add `_meta/re.lua` which adds luadoc and type annotations, for the vendored `vim.re` package. - Fix minor style issues on `_meta/lpeg.lua` luadoc for better vimdocs generation. - Fix a bug on `gen_vimdoc` where non-helptags in verbatim code blocks were parsed as helptags, affecting code examples on `vim.lpeg.Cf`, etc. - Also move the `vim.regex` section below so that it can be located closer to `vim.lpeg` and `vim.re`. --- scripts/gen_vimdoc.py | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 01532cc3d3..4cb90a4588 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -202,7 +202,6 @@ CONFIG: Dict[str, Config] = { filename='lua.txt', section_order=[ 'highlight.lua', - 'regex.lua', 'diff.lua', 'mpack.lua', 'json.lua', @@ -220,6 +219,9 @@ CONFIG: Dict[str, Config] = { 'keymap.lua', 'fs.lua', 'glob.lua', + 'lpeg.lua', + 're.lua', + 'regex.lua', 'secure.lua', 'version.lua', 'iter.lua', @@ -250,6 +252,8 @@ CONFIG: Dict[str, Config] = { 'runtime/lua/vim/_meta/json.lua', 'runtime/lua/vim/_meta/base64.lua', 'runtime/lua/vim/_meta/regex.lua', + 'runtime/lua/vim/_meta/lpeg.lua', + 'runtime/lua/vim/_meta/re.lua', 'runtime/lua/vim/_meta/spell.lua', ], file_patterns='*.lua', @@ -268,7 +272,10 @@ CONFIG: Dict[str, Config] = { section_fmt=lambda name: ( 'Lua module: vim' if name.lower() == '_editor' else 'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else - f'VIM.{name.upper()}' if name.lower() in [ 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', 'regex' ] else + f'VIM.{name.upper()}' if name.lower() in [ + 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', + 'regex', 'lpeg', 're', + ] else 'VIM' if name.lower() == 'builtin' else f'Lua module: vim.{name.lower()}'), helptag_fmt=lambda name: ( @@ -305,6 +312,8 @@ CONFIG: Dict[str, Config] = { 'json': 'vim.json', 'base64': 'vim.base64', 'regex': 'vim.regex', + 'lpeg': 'vim.lpeg', + 're': 'vim.re', 'spell': 'vim.spell', 'snippet': 'vim.snippet', 'text': 'vim.text', @@ -1350,31 +1359,20 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: # Verbatim handling. func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) - split_lines: List[str] = func_doc.split('\n') - start = 0 - while True: - try: - start = split_lines.index('>', start) - except ValueError: - break - - try: - end = split_lines.index('<', start) - except ValueError: - break - - split_lines[start + 1:end] = [ - (' ' + x).rstrip() - for x in textwrap.dedent( - "\n".join( - split_lines[start+1:end] - ) - ).split("\n") - ] - - start = end - - func_doc = "\n".join(map(align_tags, split_lines)) + def process_helptags(func_doc: str) -> str: + lines: List[str] = func_doc.split('\n') + # skip ">lang ... <" regions + is_verbatim: bool = False + for i in range(len(lines)): + if re.search(' >([a-z])*$', lines[i]): + is_verbatim = True + elif is_verbatim and lines[i].strip() == '<': + is_verbatim = False + if not is_verbatim: + lines[i] = align_tags(lines[i]) + return "\n".join(lines) + + func_doc = process_helptags(func_doc) if (fn_name.startswith(config.fn_name_prefix) and fn_name != "nvim_error_event"): -- cgit From a7df0415ab6ae9a89ca12c6765758bfd54fa69c9 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Sat, 13 Jan 2024 18:36:30 -0500 Subject: fix(lua2dox): filter out the entire `---@alias` block Problem: Any preceding luadocs block that define alias types with `@alias` magic would be prepended to the documentation of functions that follow, despite the "blank line" separator. For example: ``` --- @alias some.type.between.functions --- Blah blah long documentation for alias --- | "foo" # foo --- | "bar" # bar --- The documentation that should appear in vimdoc. function M.function_to_include_in_doc() ... end ``` then the vimdoc generated for `function_to_include_in_doc` would include the text from the alias block (e.g., "Blah blah ... for alias"). Solution: - refactor: Lua2DoxFilter should maintain its own internal state `generics`, rather than carrying it as a parameter to local helper functions. - Add another boolean state `boolean_state` which represents whether to ignore the current docstring block (magic lines). This flag will be reset as soon as the block is end. Note: As expected, there is no change at all in the current docs generated, because we have been working around and writing luadoc comments so that such erroneous docstring resulting from preceding `@alias` blocks won't appear. --- scripts/lua2dox.lua | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index abc9e5b338..871720bd2a 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -136,9 +136,17 @@ end -- input filter --- @class Lua2DoxFilter -local Lua2DoxFilter = {} +local Lua2DoxFilter = { + generics = {}, --- @type table + block_ignore = false, --- @type boolean +} setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter }) +function Lua2DoxFilter:reset() + self.generics = {} + self.block_ignore = false +end + --- trim comment off end of string --- --- @param line string @@ -239,13 +247,16 @@ end --- Processes "@…" directives in a docstring line. --- --- @param line string ---- @param generics table --- @return string? -local function process_magic(line, generics) +function Lua2DoxFilter:process_magic(line) line = line:gsub('^%s+@', '@') line = line:gsub('@package', '@private') line = line:gsub('@nodoc', '@private') + if self.block_ignore then + return '// gg:" ' .. line .. '"' + end + if not vim.startswith(line, '@') then -- it's a magic comment return '/// ' .. line end @@ -269,6 +280,12 @@ local function process_magic(line, generics) return '/// ' .. line:gsub('%.', '-dot-') end + if directive == '@alias' then + -- this contiguous block should be all ignored. + self.block_ignore = true + return '// gg:"' .. line .. '"' + end + -- preprocess line before parsing if directive == '@param' or directive == '@return' then for _, type in ipairs(TYPES) do @@ -289,12 +306,12 @@ local function process_magic(line, generics) local kind = parsed.kind if kind == 'generic' then - generics[parsed.name] = parsed.type or 'any' + self.generics[parsed.name] = parsed.type or 'any' return elseif kind == 'param' then - return process_param(parsed --[[@as luacats.Param]], generics) + return process_param(parsed --[[@as luacats.Param]], self.generics) elseif kind == 'return' then - return process_return(parsed --[[@as luacats.Return]], generics) + return process_return(parsed --[[@as luacats.Return]], self.generics) end error(string.format('unhandled parsed line %q: %s', line, parsed)) @@ -303,7 +320,7 @@ end --- @param line string --- @param in_stream StreamRead --- @return string -local function process_block_comment(line, in_stream) +function Lua2DoxFilter:process_block_comment(line, in_stream) local comment_parts = {} --- @type string[] local done --- @type boolean? @@ -337,7 +354,7 @@ end --- @param line string --- @return string -local function process_function_header(line) +function Lua2DoxFilter:process_function_header(line) local pos_fn = assert(line:find('function')) -- we've got a function local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8))) @@ -385,18 +402,17 @@ end --- @param line string --- @param in_stream StreamRead ---- @param generics table> --- @return string? -local function process_line(line, in_stream, generics) +function Lua2DoxFilter:process_line(line, in_stream) local line_raw = line line = vim.trim(line) if vim.startswith(line, '---') then - return process_magic(line:sub(4), generics) + return Lua2DoxFilter:process_magic(line:sub(4)) end if vim.startswith(line, '--' .. '[[') then -- it's a long comment - return process_block_comment(line:sub(5), in_stream) + return Lua2DoxFilter:process_block_comment(line:sub(5), in_stream) end -- Hax... I'm sorry @@ -406,7 +422,7 @@ local function process_line(line, in_stream, generics) line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') if line:find('^function') or line:find('^local%s+function') then - return process_function_header(line) + return Lua2DoxFilter:process_function_header(line) end if not line:match('^local') then @@ -429,15 +445,13 @@ end function Lua2DoxFilter:filter(filename) local in_stream = StreamRead.new(filename) - local generics = {} --- @type table - while not in_stream:eof() do local line = in_stream:getLine() - local out_line = process_line(line, in_stream, generics) + local out_line = self:process_line(line, in_stream) if not vim.startswith(vim.trim(line), '---') then - generics = {} + self:reset() end if out_line then -- cgit From e5ff71fbbfea0431826a82a6de24656d93651ec3 Mon Sep 17 00:00:00 2001 From: altermo Date: Sun, 14 Jan 2024 12:10:31 +0100 Subject: docs(builtin): overload functions with union return types --- scripts/gen_eval_files.lua | 52 ++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 27 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index e2ca62c2ee..c49f598304 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -354,41 +354,35 @@ local function render_eval_meta(f, fun, write) local params = process_params(fun.params) - if fun.signature then - write('') - if fun.deprecated then - write('--- @deprecated') - end - - local desc = fun.desc - - if desc then - --- @type string - desc = desc:gsub('\n%s*\n%s*$', '\n') - for _, l in ipairs(split(desc)) do - l = l:gsub('^ ', ''):gsub('\t', ' '):gsub('@', '\\@') - write('--- ' .. l) - end - end + write('') + if fun.deprecated then + write('--- @deprecated') + end - local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 + local desc = fun.desc - for i, param in ipairs(params) do - local pname, ptype = param[1], param[2] - local optional = (pname ~= '...' and i > req_args) and '?' or '' - write(string.format('--- @param %s%s %s', pname, optional, ptype)) + if desc then + --- @type string + desc = desc:gsub('\n%s*\n%s*$', '\n') + for _, l in ipairs(split(desc)) do + l = l:gsub('^ ', ''):gsub('\t', ' '):gsub('@', '\\@') + write('--- ' .. l) end + end - if fun.returns ~= false then - write('--- @return ' .. (fun.returns or 'any')) - end + local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 - write(render_fun_sig(funname, params)) + for i, param in ipairs(params) do + local pname, ptype = param[1], param[2] + local optional = (pname ~= '...' and i > req_args) and '?' or '' + write(string.format('--- @param %s%s %s', pname, optional, ptype)) + end - return + if fun.returns ~= false then + write('--- @return ' .. (fun.returns or 'any')) end - print('no doc for', funname) + write(render_fun_sig(funname, params)) end --- @type table @@ -398,6 +392,10 @@ local rendered_tags = {} --- @param fun vim.EvalFn --- @param write fun(line: string) local function render_sig_and_tag(name, fun, write) + if not fun.signature then + return + end + local tags = { '*' .. name .. '()*' } if fun.tags then -- cgit From 50284d07b6f020c819aeb07bfb30d88453e63b6d Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 9 Jan 2024 12:47:57 +0000 Subject: fix(diagnostic): typing --- scripts/lua2dox.lua | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 871720bd2a..4f9973449e 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -291,9 +291,11 @@ function Lua2DoxFilter:process_magic(line) for _, type in ipairs(TYPES) do line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') + line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1') line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1') + line = line:gsub('^@return%s+.*%((' .. type .. '%?)%)', '@return %1') end end -- cgit From 5aa14e1231b7eccfbc54cba8f20d54105212847d Mon Sep 17 00:00:00 2001 From: altermo <107814000+altermo@users.noreply.github.com> Date: Wed, 17 Jan 2024 20:34:25 +0100 Subject: fix(lua): return after assert returns assert message (#27064) --- scripts/gen_help_html.lua | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 49417e72bb..5986a399d9 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -1123,10 +1123,8 @@ function M._test() 'if "expected" is given, "actual" is also required' ) if expected then - return assert( - cond, - ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual)) - ) + assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual))) + return cond else return assert(cond) end -- cgit From fa9a85ae468b9df30ae9e5c05a08c0f124e267df Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Tue, 16 Jan 2024 19:19:21 -0500 Subject: fix(lsp): clean up duplicate and unused meta type annotations --- scripts/gen_lsp.lua | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua index 0e7eb38cca..b332a17fdc 100644 --- a/scripts/gen_lsp.lua +++ b/scripts/gen_lsp.lua @@ -159,11 +159,13 @@ function M.gen(opt) local output = { '--' .. '[[', - 'This file is autogenerated from scripts/gen_lsp.lua', + 'THIS FILE IS GENERATED by scripts/gen_lsp.lua', + 'DO NOT EDIT MANUALLY', + '', + 'Based on LSP protocol ' .. opt.version, + '', 'Regenerate:', - ([=[nvim -l scripts/gen_lsp.lua gen --version %s --out runtime/lua/vim/lsp/_meta/protocol.lua]=]):format( - DEFAULT_LSP_VERSION - ), + ([=[nvim -l scripts/gen_lsp.lua gen --version %s]=]):format(DEFAULT_LSP_VERSION), '--' .. ']]', '', '---@meta', @@ -171,12 +173,9 @@ function M.gen(opt) '', '---@alias lsp.null nil', '---@alias uinteger integer', - '---@alias lsp.decimal number', + '---@alias decimal number', '---@alias lsp.DocumentUri string', '---@alias lsp.URI string', - '---@alias lsp.LSPObject table', - '---@alias lsp.LSPArray lsp.LSPAny[]', - '---@alias lsp.LSPAny lsp.LSPObject|lsp.LSPArray|string|number|boolean|nil', '', } -- cgit From d66ed4ea468d411668713c3777ad3658f18badf3 Mon Sep 17 00:00:00 2001 From: bfredl Date: Mon, 22 Jan 2024 08:49:45 +0100 Subject: refactor(api): give "hl_group" more accurate _meta type These can either be number or string in lua, so we can specify this directly as "number|string". --- scripts/gen_eval_files.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index c49f598304..499c7dc9a7 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -88,6 +88,7 @@ local API_TYPES = { LuaRef = 'function', Dictionary = 'table', Float = 'number', + HLGroupID = 'number|string', void = '', } -- cgit From b4da4783f98837efb7192a3e5a40dd491aeb1491 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Sat, 20 Jan 2024 18:24:45 +0100 Subject: build: make genappimage.sh work with existing CMAKE_INSTALL_PREFIX Using CMAKE_INSTALL_PREFIX is unreliable as it's a cache variable, meaning the users previous value will be used if not supplied. Instead, use the `--prefix` flag which is guaranteed to always work. --- scripts/genappimage.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh index b0bf186f85..0ad5e803e4 100755 --- a/scripts/genappimage.sh +++ b/scripts/genappimage.sh @@ -26,8 +26,8 @@ APP_DIR="$APP.AppDir" ######################################################################## # Build and install nvim into the AppImage -make CMAKE_BUILD_TYPE="${NVIM_BUILD_TYPE}" CMAKE_EXTRA_FLAGS="-DCMAKE_INSTALL_PREFIX=${APP_DIR}/usr -DCMAKE_INSTALL_MANDIR=man" -make install +make CMAKE_BUILD_TYPE="${NVIM_BUILD_TYPE}" +cmake --install build --prefix="$APP_BUILD_DIR/${APP_DIR}/usr" ######################################################################## # Get helper functions and move to AppDir @@ -52,7 +52,7 @@ fi chmod +x "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage # metainfo is not packaged automatically by linuxdeploy -mkdir "$APP_DIR/usr/share/metainfo/" +mkdir -p "$APP_DIR/usr/share/metainfo/" cp "$ROOT_DIR/runtime/nvim.appdata.xml" "$APP_DIR/usr/share/metainfo/" cd "$APP_DIR" || exit -- cgit From 28d1640dd6043f25cd3ae761dc9c4c8d374a8ce1 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 26 Jan 2024 11:38:19 +0000 Subject: feat: improve return type annotations for vim.api.* --- scripts/gen_eval_files.lua | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index 499c7dc9a7..78fc1c7d4f 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -1,3 +1,5 @@ +#!/usr/bin/env -S nvim -l + -- Generator for various vimdoc and Lua type files local DEP_API_METADATA = 'build/api_metadata.mpack' @@ -17,6 +19,31 @@ local DEP_API_DOC = 'runtime/doc/api.mpack' --- @field remote boolean --- @field since integer +local LUA_API_RETURN_OVERRIDES = { + nvim_buf_get_command = 'table', + nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item', + nvim_buf_get_extmarks = 'vim.api.keyset.get_extmark_item[]', + nvim_buf_get_keymap = 'vim.api.keyset.keymap[]', + nvim_get_autocmds = 'vim.api.keyset.get_autocmds.ret[]', + nvim_get_color_map = 'table', + nvim_get_command = 'table', + nvim_get_keymap = 'vim.api.keyset.keymap[]', + nvim_get_mark = 'vim.api.keyset.get_mark', + + -- Can also return table, however we need to + -- pick one to get some benefit. + -- REVISIT lewrus01 (26/01/24): we can maybe add + -- @overload fun(ns: integer, {}): table + nvim_get_hl = 'vim.api.keyset.hl_info', + + nvim_get_mode = 'vim.api.keyset.get_mode', + nvim_get_namespaces = 'table', + nvim_get_option_info = 'vim.api.keyset.get_option_info', + nvim_get_option_info2 = 'vim.api.keyset.get_option_info', + nvim_parse_cmd = 'vim.api.keyset.parse_cmd', + nvim_win_get_config = 'vim.api.keyset.float_config', +} + local LUA_META_HEADER = { '--- @meta _', '-- THIS FILE IS GENERATED', @@ -289,11 +316,9 @@ local function render_api_meta(_f, fun, write) end end if fun.returns ~= '' then - if fun.returns_desc then - write('--- @return ' .. fun.returns .. ' : ' .. fun.returns_desc) - else - write('--- @return ' .. fun.returns) - end + local ret_desc = fun.returns_desc and ' : ' .. fun.returns_desc or '' + local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns + write('--- @return ' .. ret .. ret_desc) end local param_str = table.concat(param_names, ', ') -- cgit From 01e82eba209a96f932d3497e580ab0ca749efafa Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Sun, 28 Jan 2024 17:22:39 -0500 Subject: build(docs): separate lint job to validate vimdoc #27227 Summary: Separate the lint job (`make lintdoc`) to validate runtime/doc, it is no longer as a part of functionaltest (help_spec). Build (cmake) and CI: - `make lintdoc`: validate vimdoc files and test-generate HTML docs. CI will run this as a part of the "docs" workflow. - `scripts/lintdoc.lua` is added as an entry point (executable script) for validating vimdoc files. scripts/gen_help_html.lua: - Move the tests for validating docs and generating HTMLs from `help_spec.lua` to `gen_help_html`. Added: - `gen_help_html.run_validate()`. - `gen_help_html.test_gen()`. - Do not hard-code `help_dir` to `build/runtime/doc`, but resolve from `$VIMRUNTIME`. Therefore, the `make lintdoc` job will check doc files on `./runtime/doc`, not on `./build/runtime/doc`. - Add type annotations for gen_help_html. --- scripts/gen_help_html.lua | 171 ++++++++++++++++++++++++++++++++++------------ scripts/lintdoc.lua | 20 ++++++ 2 files changed, 147 insertions(+), 44 deletions(-) create mode 100755 scripts/lintdoc.lua (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 5986a399d9..20174bab97 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -2,32 +2,38 @@ -- -- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). -- +-- USAGE (For CI/local testing purposes): Simply `make lintdoc` or `scripts/lintdoc.lua`, which +-- basically does the following: +-- 1. :helptags ALL +-- 2. nvim -V1 -es +"lua require('scripts.gen_help_html').run_validate()" +q +-- 3. nvim -V1 -es +"lua require('scripts.gen_help_html').test_gen()" +q +-- -- USAGE (GENERATE HTML): --- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). --- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" +-- 1. `:helptags ALL` first; this script depends on vim.fn.taglist(). +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('$VIMRUNTIME/doc', 'target/dir/')" +q -- - Read the docstring at gen(). -- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 -- 4. Visit http://localhost:4000/…/help.txt.html -- -- USAGE (VALIDATE): --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate()" +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate('$VIMRUNTIME/doc')" +q -- - validate() is 10x faster than gen(), so it is used in CI. -- -- SELF-TEST MODE: --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +q -- -- NOTES: --- * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too --- slow (~1 min) to run in per-commit CI. +-- * gen() and validate() are the primary (programmatic) entrypoints. validate() only exists +-- because gen() is too slow (~1 min) to run in per-commit CI. -- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. -- * visit_validate() is the core function used by validate(). -- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. local tagmap = nil local helpfiles = nil -local invalid_links = {} -local invalid_urls = {} -local invalid_spelling = {} +local invalid_links = {} ---@type table +local invalid_urls = {} ---@type table +local invalid_spelling = {} ---@type table> local spell_dict = { Neovim = 'Nvim', NeoVim = 'Nvim', @@ -150,7 +156,8 @@ end --- --- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc --- ---- @returns (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. +--- @param url string +--- @return string, string (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. local function fix_url(url) local removed_chars = '' local fixed_url = url @@ -656,8 +663,10 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end end -local function get_helpfiles(include) - local dir = './build/runtime/doc' +--- @param dir string e.g. '$VIMRUNTIME/doc' +--- @param include string[]|nil +--- @return string[] +local function get_helpfiles(dir, include) local rv = {} for f, type in vim.fs.dir(dir) do if @@ -1113,25 +1122,34 @@ local function gen_css(fname) tofile(fname, css) end -function M._test() - tagmap = get_helptags('./build/runtime/doc') - helpfiles = get_helpfiles() +-- Testing - local function ok(cond, expected, actual) +local function ok(cond, expected, actual, message) + assert( + (not expected and not actual) or (expected and actual), + 'if "expected" is given, "actual" is also required' + ) + if expected then assert( - (not expected and not actual) or (expected and actual), - 'if "expected" is given, "actual" is also required' + cond, + ('%sexpected %s, got: %s'):format( + message and (message .. '\n') or '', + vim.inspect(expected), + vim.inspect(actual) + ) ) - if expected then - assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual))) - return cond - else - return assert(cond) - end - end - local function eq(expected, actual) - return ok(expected == actual, expected, actual) + return cond + else + return assert(cond) end +end +local function eq(expected, actual, message) + return ok(vim.deep_equal(expected, actual), expected, actual, message) +end + +function M._test() + tagmap = get_helptags('$VIMRUNTIME/doc') + helpfiles = get_helpfiles(vim.fn.expand('$VIMRUNTIME/doc')) ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) ok( @@ -1169,20 +1187,25 @@ function M._test() eq('https://example.com', fixed_url) eq('', removed_chars) - print('all tests passed') + print('all tests passed.\n') end +--- @class nvim.gen_help_html.gen_result +--- @field helpfiles string[] list of generated HTML files, from the source docs {include} +--- @field err_count integer number of parse errors in :help docs +--- @field invalid_links table + --- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`. --- --- Example: --- ---- gen('./build/runtime/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) +--- gen('$VIMRUNTIME/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) --- --- @param help_dir string Source directory containing the :help files. Must run `make helptags` first. --- @param to_dir string Target directory where the .html files will be written. ---- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} +--- @param include string[]|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} --- ---- @returns info dict +--- @return nvim.gen_help_html.gen_result result function M.gen(help_dir, to_dir, include, commit, parser_path) vim.validate { help_dir = { @@ -1207,7 +1230,7 @@ function M.gen(help_dir, to_dir, include, commit, parser_path) local err_count = 0 ensure_runtimepath() tagmap = get_helptags(vim.fn.expand(help_dir)) - helpfiles = get_helpfiles(include) + helpfiles = get_helpfiles(help_dir, include) to_dir = vim.fn.expand(to_dir) parser_path = parser_path and vim.fn.expand(parser_path) or nil @@ -1233,6 +1256,7 @@ function M.gen(help_dir, to_dir, include, commit, parser_path) print(('total errors: %d'):format(err_count)) print(('invalid tags:\n%s'):format(vim.inspect(invalid_links))) + --- @type nvim.gen_help_html.gen_result return { helpfiles = helpfiles, err_count = err_count, @@ -1240,13 +1264,21 @@ function M.gen(help_dir, to_dir, include, commit, parser_path) } end --- Validates all :help files found in `help_dir`: --- - checks that |tag| links point to valid helptags. --- - recursively counts parse errors ("ERROR" nodes) --- --- This is 10x faster than gen(), for use in CI. --- --- @returns results dict +--- @class nvim.gen_help_html.validate_result +--- @field helpfiles integer number of generated helpfiles +--- @field err_count integer number of parse errors +--- @field parse_errors table +--- @field invalid_links table invalid tags in :help docs +--- @field invalid_urls table invalid URLs in :help docs +--- @field invalid_spelling table> invalid spelling in :help docs + +--- Validates all :help files found in `help_dir`: +--- - checks that |tag| links point to valid helptags. +--- - recursively counts parse errors ("ERROR" nodes) +--- +--- This is 10x faster than gen(), for use in CI. +--- +--- @return nvim.gen_help_html.validate_result result function M.validate(help_dir, include, parser_path) vim.validate { help_dir = { @@ -1265,15 +1297,15 @@ function M.validate(help_dir, include, parser_path) 'valid vimdoc.{so,dll} filepath', }, } - local err_count = 0 - local files_to_errors = {} + local err_count = 0 ---@type integer + local files_to_errors = {} ---@type table ensure_runtimepath() tagmap = get_helptags(vim.fn.expand(help_dir)) - helpfiles = get_helpfiles(include) + helpfiles = get_helpfiles(help_dir, include) parser_path = parser_path and vim.fn.expand(parser_path) or nil for _, f in ipairs(helpfiles) do - local helpfile = vim.fs.basename(f) + local helpfile = assert(vim.fs.basename(f)) local rv = validate_one(f, parser_path) print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) if #rv.parse_errors > 0 then @@ -1285,14 +1317,65 @@ function M.validate(help_dir, include, parser_path) err_count = err_count + #rv.parse_errors end + ---@type nvim.gen_help_html.validate_result return { helpfiles = #helpfiles, err_count = err_count, + parse_errors = files_to_errors, invalid_links = invalid_links, invalid_urls = invalid_urls, invalid_spelling = invalid_spelling, - parse_errors = files_to_errors, } end +--- Validates vimdoc files on $VIMRUNTIME. and print human-readable error messages if fails. +--- +--- If this fails, try these steps (in order): +--- 1. Fix/cleanup the :help docs. +--- 2. Fix the parser: https://github.com/neovim/tree-sitter-vimdoc +--- 3. File a parser bug, and adjust the tolerance of this test in the meantime. +--- +--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc' +function M.run_validate(help_dir) + help_dir = vim.fn.expand(help_dir or '$VIMRUNTIME/doc') + print('doc path = ' .. vim.uv.fs_realpath(help_dir)) + + local rv = M.validate(help_dir) + + -- Check that we actually found helpfiles. + ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles) + + eq({}, rv.parse_errors, 'no parse errors') + eq(0, rv.err_count, 'no parse errors') + eq({}, rv.invalid_links, 'invalid tags in :help docs') + eq({}, rv.invalid_urls, 'invalid URLs in :help docs') + eq( + {}, + rv.invalid_spelling, + 'invalid spelling in :help docs (see spell_dict in scripts/gen_help_html.lua)' + ) +end + +--- Test-generates HTML from docs. +--- +--- 1. Test that gen_help_html.lua actually works. +--- 2. Test that parse errors did not increase wildly. Because we explicitly test only a few +--- :help files, we can be precise about the tolerances here. +--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc' +function M.test_gen(help_dir) + local tmpdir = assert(vim.fs.dirname(vim.fn.tempname())) + help_dir = vim.fn.expand(help_dir or '$VIMRUNTIME/doc') + print('doc path = ' .. vim.uv.fs_realpath(help_dir)) + + local rv = M.gen( + help_dir, + tmpdir, + -- Because gen() is slow (~30s), this test is limited to a few files. + { 'pi_health.txt', 'help.txt', 'index.txt', 'nvim.txt' } + ) + eq(4, #rv.helpfiles) + eq(0, rv.err_count, 'parse errors in :help docs') + eq({}, rv.invalid_links, 'invalid tags in :help docs') +end + return M diff --git a/scripts/lintdoc.lua b/scripts/lintdoc.lua new file mode 100755 index 0000000000..5e78b4cdcb --- /dev/null +++ b/scripts/lintdoc.lua @@ -0,0 +1,20 @@ +#!/usr/bin/env -S nvim -l + +-- Validate vimdoc files on $VIMRUNTIME/doc, and test generating HTML docs. +-- Checks for duplicate/missing tags, parse errors, and invalid links/urls/spellings. +-- See also `make lintdoc`. +-- +-- Usage: +-- $ nvim -l scripts/lintdoc.lua +-- $ make lintdoc + +print('Running lintdoc ...') + +-- gen_help_html requires :helptags to be generated on $VIMRUNTIME/doc +-- :helptags checks for duplicate tags. +vim.cmd [[ helptags ALL ]] + +require('scripts.gen_help_html').run_validate() +require('scripts.gen_help_html').test_gen() + +print('lintdoc PASSED.') -- cgit From 5b1b765610ae12ebd6400aafd068903569ee441a Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Sun, 28 Jan 2024 20:53:14 -0500 Subject: docs: enforce "treesitter" spelling #27110 It's the "tree-sitter" project, but "treesitter" in our code and docs. --- scripts/gen_help_html.lua | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 20174bab97..8bc7d99985 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -42,9 +42,14 @@ local spell_dict = { VimL = 'Vimscript', vimL = 'Vimscript', viml = 'Vimscript', + ['tree-sitter'] = 'treesitter', + ['Tree-sitter'] = 'Treesitter', } +--- specify the list of keywords to ignore (i.e. allow), or true to disable spell check completely. +--- @type table local spell_ignore_files = { - ['backers.txt'] = 'true', + ['backers.txt'] = true, + ['news.txt'] = { 'tree-sitter' }, -- in news, may refer to the upstream "tree-sitter" library } local language = nil @@ -398,9 +403,15 @@ local function visit_validate(root, level, lang_tree, opt, stats) then local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. local fname_basename = assert(vim.fs.basename(opt.fname)) - if spell_dict[text_nopunct] and not spell_ignore_files[fname_basename] then - invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} - invalid_spelling[text_nopunct][fname_basename] = node_text(root:parent()) + if spell_dict[text_nopunct] then + local should_ignore = ( + spell_ignore_files[fname_basename] == true + or vim.tbl_contains(spell_ignore_files[fname_basename] --[[ @as string[] ]], text_nopunct) + ) + if not should_ignore then + invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} + invalid_spelling[text_nopunct][fname_basename] = node_text(root:parent()) + end end elseif node_name == 'url' then local fixed_url, _ = fix_url(trim(text)) -- cgit From be5cf338362244a944cdeebf07520f85d7d93226 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Mon, 29 Jan 2024 14:02:10 -0500 Subject: fix(gen_help_html): type warnings, spell_ignore_files #27254 - Add type annotations, fix most of the type warnings. - Fix a minor bug on `spell_ignore_files`: nil error when an invalid spelling is found but the file is not ignored. --- scripts/gen_help_html.lua | 72 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 8bc7d99985..5cda16bbe6 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -10,13 +10,13 @@ -- -- USAGE (GENERATE HTML): -- 1. `:helptags ALL` first; this script depends on vim.fn.taglist(). --- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('$VIMRUNTIME/doc', 'target/dir/')" +q +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./runtime/doc', 'target/dir/')" +q -- - Read the docstring at gen(). -- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 -- 4. Visit http://localhost:4000/…/help.txt.html -- -- USAGE (VALIDATE): --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate('$VIMRUNTIME/doc')" +q +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate('./runtime/doc')" +q -- - validate() is 10x faster than gen(), so it is used in CI. -- -- SELF-TEST MODE: @@ -29,8 +29,8 @@ -- * visit_validate() is the core function used by validate(). -- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. -local tagmap = nil -local helpfiles = nil +local tagmap = nil ---@type table +local helpfiles = nil ---@type string[] local invalid_links = {} ---@type table local invalid_urls = {} ---@type table local invalid_spelling = {} ---@type table> @@ -113,8 +113,9 @@ local function tofile(fname, text) end end +---@type fun(s: string): string local function html_esc(s) - return s:gsub('&', '&'):gsub('<', '<'):gsub('>', '>') + return (s:gsub('&', '&'):gsub('<', '<'):gsub('>', '>')) end local function url_encode(s) @@ -129,7 +130,7 @@ local function url_encode(s) end local function expandtabs(s) - return s:gsub('\t', (' '):rep(8)) + return s:gsub('\t', (' '):rep(8)) --[[ @as string ]] end local function to_titlecase(s) @@ -153,6 +154,7 @@ local function is_blank(s) return not not s:find([[^[\t ]*$]]) end +---@type fun(s: string, dir?:0|1|2): string local function trim(s, dir) return vim.fn.trim(s, '\r\t\n ', dir or 0) end @@ -274,6 +276,9 @@ local function trim_indent(s) end --- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. +---@param node TSNode +---@param bufnr integer +---@param offset integer local function getbuflinestr(node, bufnr, offset) local line1, _, line2, _ = node:range() line1 = line1 - offset @@ -284,8 +289,12 @@ end --- Gets the whitespace just before `node` from the raw buffer text. --- Needed for preformatted `old` lines. +---@param node TSNode +---@param bufnr integer +---@return string local function getws(node, bufnr) local line1, c1, line2, _ = node:range() + ---@type string local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] local text_before = raw:sub(1, c1) local leading_ws = text_before:match('%s+$') or '' @@ -322,9 +331,10 @@ local function ignore_parse_error(fname, s) return s:find("^[`'|*]") end +---@param node TSNode local function has_ancestor(node, ancestor_name) - local p = node - while true do + local p = node ---@type TSNode? + while p do p = p:parent() if not p or p:type() == 'help_file' then break @@ -336,6 +346,7 @@ local function has_ancestor(node, ancestor_name) end --- Gets the first matching child node matching `name`. +---@param node TSNode local function first(node, name) for c, _ in node:iter_children() do if c:named() and c:type() == name then @@ -369,6 +380,11 @@ local function validate_url(text, fname) end --- Traverses the tree at `root` and checks that |tag| links point to valid helptags. +---@param root TSNode +---@param level integer +---@param lang_tree TSTree +---@param opt table +---@param stats table local function visit_validate(root, level, lang_tree, opt, stats) level = level or 0 local node_name = (root.named and root:named()) and root:type() or nil @@ -406,7 +422,10 @@ local function visit_validate(root, level, lang_tree, opt, stats) if spell_dict[text_nopunct] then local should_ignore = ( spell_ignore_files[fname_basename] == true - or vim.tbl_contains(spell_ignore_files[fname_basename] --[[ @as string[] ]], text_nopunct) + or vim.tbl_contains( + (spell_ignore_files[fname_basename] or {}) --[[ @as string[] ]], + text_nopunct + ) ) if not should_ignore then invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} @@ -423,6 +442,8 @@ end -- Fix tab alignment issues caused by concealed characters like |, `, * in tags -- and code blocks. +---@param text string +---@param next_node_text string local function fix_tab_after_conceal(text, next_node_text) -- Vim tabs take into account the two concealed characters even though they -- are invisible, so we need to add back in the two spaces if this is @@ -433,7 +454,18 @@ local function fix_tab_after_conceal(text, next_node_text) return text end +---@class (exact) nvim.gen_help_html.heading +---@field name string +---@field subheadings nvim.gen_help_html.heading[] +---@field tag string + -- Generates HTML from node `root` recursively. +---@param root TSNode +---@param level integer +---@param lang_tree TSTree +---@param headings nvim.gen_help_html.heading[] +---@param opt table +---@param stats table local function visit_node(root, level, lang_tree, headings, opt, stats) level = level or 0 @@ -451,7 +483,6 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) -- Parent kind (string). local parent = root:parent() and root:parent():type() or nil local text = '' - local trimmed -- Gets leading whitespace of `node`. local function ws(node) node = node or root @@ -469,6 +500,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) end + local trimmed ---@type string if root:named_child_count() == 0 or node_name == 'ERROR' then text = node_text() trimmed = html_esc(trim(text)) @@ -503,7 +535,9 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname) if node_name == 'h1' or #headings == 0 then - table.insert(headings, { name = hname, subheadings = {}, tag = tagname }) + ---@type nvim.gen_help_html.heading + local heading = { name = hname, subheadings = {}, tag = tagname } + headings[#headings + 1] = heading else table.insert( headings[#headings].subheadings, @@ -593,7 +627,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if is_blank(text) then return '' end - local code + local code ---@type string if language then code = ('
%s
'):format( language, @@ -718,13 +752,15 @@ end --- --- @param fname string help file to parse --- @param parser_path string? path to non-default vimdoc.so ---- @returns lang_tree, bufnr +--- @return LanguageTree, integer (lang_tree, bufnr) local function parse_buf(fname, parser_path) - local buf + local buf ---@type integer if type(fname) == 'string' then vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename. buf = vim.api.nvim_get_current_buf() else + -- Left for debugging + ---@diagnostic disable-next-line: no-unknown buf = fname vim.cmd('sbuffer ' .. tostring(fname)) -- Buffer number. end @@ -741,7 +777,7 @@ end --- --- @param fname string help file to validate --- @param parser_path string? path to non-default vimdoc.so ---- @returns { invalid_links: number, parse_errors: string[] } +--- @return { invalid_links: number, parse_errors: string[] } local function validate_one(fname, parser_path) local stats = { parse_errors = {}, @@ -762,7 +798,8 @@ end --- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) --- @param parser_path string? path to non-default vimdoc.so --- ---- @returns html, stats +--- @return string html +--- @return table stats local function gen_one(fname, to_fname, old, commit, parser_path) local stats = { noise_lines = {}, @@ -770,6 +807,7 @@ local function gen_one(fname, to_fname, old, commit, parser_path) first_tags = {}, -- Track the first few tags in doc. } local lang_tree, buf = parse_buf(fname, parser_path) + ---@type nvim.gen_help_html.heading[] local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. local title = to_titlecase(basename_noext(fname)) @@ -903,6 +941,7 @@ local function gen_one(fname, to_fname, old, commit, parser_path) main ) + ---@type string local toc = [[
@@ -916,6 +955,7 @@ local function gen_one(fname, to_fname, old, commit, parser_path) n = n + 1 + #h1.subheadings end for _, h1 in ipairs(headings) do + ---@type string toc = toc .. ('
%s\n'):format(h1.tag, h1.name) if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. for _, h2 in ipairs(h1.subheadings) do -- cgit From 0a8e66898d73ee90a9c52d0944141f498804c99d Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Wed, 31 Jan 2024 21:45:30 +0800 Subject: build: update builtin terminfo (#27272) - Update to ncurses 6.4.20230520 - Disable smglp and smgrp for vtpcon and conemu - Add xterm+sl to vtpcon, fix #26880 --- scripts/windows.ti | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/windows.ti b/scripts/windows.ti index c3a367e6d4..34028b8e00 100644 --- a/scripts/windows.ti +++ b/scripts/windows.ti @@ -34,7 +34,7 @@ conemu|ANIS X3.64 and Xterm 256 colors for ConEmu with libuv, smcup=\E[?1049h, rmir@, rmkx@, rmm@, rs1@, rs2@, setab=\E[48;5;%p1%dm, setaf=\E[38;5;%p1%dm, sgr=\E[0%?%p1%p3%|%t;7%;%?%p2%t;4%;%?%p6%t;1%;m, - sgr0=\E[0m, smam@, smglr@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, + sgr0=\E[0m, smam@, smglp@, smgrp@, smglr@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, Cr@, Cs@, Ms@, XM@, kDC3@, kDC4@, kDC5@, kDC6@, kDC7@, kDN@, kDN3@, kDN4@, kDN5@, kDN6@, kDN7@, kEND3@, kEND4@, kEND5@, kEND6@, kEND7@, @@ -57,7 +57,7 @@ vtpcon|ANIS emulation for console virtual terminal sequence with libuv, mc0@, mc4@, mc5@, meml@, memu@, oc@, rmam@, rmcup=\E[?1049l, smcup=\E[?1049h, rmir@, rmkx@, rmm@, rs1@, rs2@, sgr=\E[0%?%p1%p3%|%t;7%;%?%p2%t;4%;%?%p6%t;1%;m, - sgr0=\E[0m, smam@, smglr@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, + sgr0=\E[0m, smam@, smglp@, smgrp@, smglr@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, Cr@, Cs@, Ms@, XM@, kDC3@, kDC4@, kDC5@, kDC6@, kDC7@, kDN@, kDN3@, kDN4@, kDN5@, kDN6@, kDN7@, kEND3@, kEND4@, kEND5@, kEND6@, kEND7@, @@ -68,4 +68,4 @@ vtpcon|ANIS emulation for console virtual terminal sequence with libuv, kPRV3@, kPRV4@, kPRV5@, kPRV6@, kPRV7@, kRIT3@, kRIT4@, kRIT5@, kRIT6@, kRIT7@, kUP3@, kUP4@, kUP5@, kUP6@, kUP7@, rmxx@, smxx@, xm@, - use=libuv+basekey, use=libuv+exkey, use=xterm+256color, use=xterm-new, + use=libuv+basekey, use=libuv+exkey, use=xterm+256color, use=xterm+sl, use=xterm-new, -- cgit From 4c91194611086916c833d61e28e2f5e689316e83 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Sat, 27 Jan 2024 19:10:41 +0100 Subject: build: various fixes - Consistently use the variable CMAKE_BUILD_TYPE to select build type. - Remove broken `doc_html` target. - Remove swap files created by oldtest when cleaning. - Only rerun `lintdoc` if any documentation files has changed. --- scripts/genappimage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh index 0ad5e803e4..e8aac42a9c 100755 --- a/scripts/genappimage.sh +++ b/scripts/genappimage.sh @@ -26,7 +26,7 @@ APP_DIR="$APP.AppDir" ######################################################################## # Build and install nvim into the AppImage -make CMAKE_BUILD_TYPE="${NVIM_BUILD_TYPE}" +make CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" cmake --install build --prefix="$APP_BUILD_DIR/${APP_DIR}/usr" ######################################################################## -- cgit From c4417ae70c03815c2fb64edb479017e79d223cf7 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 6 Feb 2024 15:08:17 +0000 Subject: fix(doc): prevent doxygen confusion --- scripts/lua2dox.lua | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 4f9973449e..0b3daa59b2 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -447,6 +447,8 @@ end function Lua2DoxFilter:filter(filename) local in_stream = StreamRead.new(filename) + local last_was_magic = false + while not in_stream:eof() do local line = in_stream:getLine() @@ -457,6 +459,16 @@ function Lua2DoxFilter:filter(filename) end if out_line then + -- Ensure all magic blocks associate with some object to prevent doxygen + -- from getting confused. + if vim.startswith(out_line, '///') then + last_was_magic = true + else + if last_was_magic and out_line:match('^// zz: [^-]+') then + writeln('local_function _ignore() {}') + end + last_was_magic = false + end writeln(out_line) end end -- cgit From f9d81c43d2296d212c9cebcbdce401cd76cf0f1f Mon Sep 17 00:00:00 2001 From: bfredl Date: Wed, 31 Jan 2024 22:02:06 +0100 Subject: refactor(api): use keydict and arena for more api return values Implement api_keydict_to_dict as the complement to api_dict_to_keydict Fix a conversion error when nvim_get_win_config gets called from lua, where Float values "x" and "y" didn't get converted to lua numbers. --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 4cb90a4588..c1a2183f24 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -1688,7 +1688,7 @@ def filter_source(filename, keep_tmpfiles): else: """Filters the source to fix macros that confuse Doxygen.""" with open(filename, 'rt') as fp: - print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))', + print(re.sub(r'^(ArrayOf|DictionaryOf|Dict)(\(.*?\))', lambda m: m.group(1)+'_'.join( re.split(r'[^\w]+', m.group(2))), fp.read(), flags=re.M)) -- cgit From d040b7341ec53317178cd75e9706c0a0ac0194d2 Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Fri, 16 Feb 2024 20:19:26 +0800 Subject: build(vim-patch.sh): don't add vim/vim to issue of another repo (#27493) --- scripts/vim-patch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index c7ccc3992d..660b8b6bb0 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -174,7 +174,7 @@ assign_commit_details() { vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}" vim_message="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:%B' "${vim_commit}" \ - | sed -Ee 's/(#[0-9]{1,})/vim\/vim\1/g')" + | sed -Ee 's/(\W)(#[0-9]{1,})/\1vim\/vim\2/g')" local vim_coauthor0 vim_coauthor0="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:Co-authored-by: %an <%ae>' "${vim_commit}")" # Extract co-authors from the commit message. -- cgit From ac0e8323dc82622a201f49efcdfcb79567a8f75e Mon Sep 17 00:00:00 2001 From: Maria José Solano Date: Wed, 21 Feb 2024 03:31:56 -0800 Subject: fix(lsp): add parentheses to generated union array types (#27560) --- scripts/gen_lsp.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua index b332a17fdc..19fad7bab4 100644 --- a/scripts/gen_lsp.lua +++ b/scripts/gen_lsp.lua @@ -224,7 +224,11 @@ function M.gen(opt) -- ArrayType elseif type.kind == 'array' then - return parse_type(type.element, prefix) .. '[]' + local parsed_items = parse_type(type.element, prefix) + if type.element.items and #type.element.items > 1 then + parsed_items = '(' .. parsed_items .. ')' + end + return parsed_items .. '[]' -- OrType elseif type.kind == 'or' then -- cgit From 741a6684e0c881bbad72d8235ad55f55282da0ef Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Thu, 22 Feb 2024 19:39:58 +0800 Subject: docs(builtin): show tag at first line with multiple signatures (#27577) Problem: When a function has multiple signatures, putting its tag at the last one may make one think that's its only signature. Solution: When a function has multiple signatures, put its tag at the first one. --- scripts/gen_eval_files.lua | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index 78fc1c7d4f..b7f17a2d58 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -411,9 +411,6 @@ local function render_eval_meta(f, fun, write) write(render_fun_sig(funname, params)) end ---- @type table -local rendered_tags = {} - --- @param name string --- @param fun vim.EvalFn --- @param write fun(line: string) @@ -455,24 +452,17 @@ local function render_eval_doc(f, fun, write) return end - local desc = fun.desc - - if not desc then + if f:find('__%d+$') then write(fun.signature) - return + else + render_sig_and_tag(fun.name or f, fun, write) end - local name = fun.name or f - - if rendered_tags[name] then - write(fun.signature) - else - render_sig_and_tag(name, fun, write) - rendered_tags[name] = true + if not fun.desc then + return end - desc = vim.trim(desc) - local desc_l = split(desc) + local desc_l = split(vim.trim(fun.desc)) for _, l in ipairs(desc_l) do l = l:gsub('^ ', '') if vim.startswith(l, '<') and not l:match('^<[^ \t]+>') then -- cgit From 9beb40a4db5613601fc1a4b828a44e5977eca046 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Thu, 15 Feb 2024 17:16:04 +0000 Subject: feat(docs): replace lua2dox.lua Problem: The documentation flow (`gen_vimdoc.py`) has several issues: - it's not very versatile - depends on doxygen - doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C. - The intermediate XML files and filters makes it too much like a rube goldberg machine. Solution: Re-implement the flow using Lua, LPEG and treesitter. - `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic. - `lua2dox.lua` is gone! - No more XML files. - Doxygen is now longer used and instead we now use: - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`). - LPEG for C parsing (see `scripts/cdoc_parser.lua`) - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`). - Treesitter for Markdown parsing (see `scripts/text_utils.lua`). - The generated `runtime/doc/*.mpack` files have been removed. - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly. - Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change). --- scripts/cdoc_grammar.lua | 87 +++ scripts/cdoc_parser.lua | 223 ++++++ scripts/gen_eval_files.lua | 100 ++- scripts/gen_vimdoc.lua | 787 +++++++++++++++++++ scripts/gen_vimdoc.py | 1766 ------------------------------------------- scripts/lua2dox.lua | 544 ------------- scripts/luacats_grammar.lua | 218 ++++++ scripts/luacats_parser.lua | 521 +++++++++++++ scripts/text_utils.lua | 239 ++++++ 9 files changed, 2135 insertions(+), 2350 deletions(-) create mode 100644 scripts/cdoc_grammar.lua create mode 100644 scripts/cdoc_parser.lua create mode 100755 scripts/gen_vimdoc.lua delete mode 100755 scripts/gen_vimdoc.py delete mode 100644 scripts/lua2dox.lua create mode 100644 scripts/luacats_grammar.lua create mode 100644 scripts/luacats_parser.lua create mode 100644 scripts/text_utils.lua (limited to 'scripts') diff --git a/scripts/cdoc_grammar.lua b/scripts/cdoc_grammar.lua new file mode 100644 index 0000000000..6a7610883b --- /dev/null +++ b/scripts/cdoc_grammar.lua @@ -0,0 +1,87 @@ +--[[! +LPEG grammar for C doc comments +]] + +--- @class nvim.cdoc.Param +--- @field kind 'param' +--- @field name string +--- @field desc? string + +--- @class nvim.cdoc.Return +--- @field kind 'return' +--- @field desc string + +--- @class nvim.cdoc.Note +--- @field desc? string + +--- @alias nvim.cdoc.grammar.result +--- | nvim.cdoc.Param +--- | nvim.cdoc.Return +--- | nvim.cdoc.Note + +--- @class nvim.cdoc.grammar +--- @field match fun(self, input: string): nvim.cdoc.grammar.result? + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local Ct, Cg = lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local nl = P('\r\n') + P('\n') +local ws = rep1(S(' \t') + nl) + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local ident = letter * rep(letter + R('09')) + +local io = P('[') * (P('in') + P('out') + P('inout')) * P(']') + +--- @param x string +local function Pf(x) + return opt(ws) * P(x) * opt(ws) +end + +--- @type table +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +local grammar = P { + rep1(P('@') * v.ats), + + ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc, + + at_param = Ct( + Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc')) + ), + + at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))), + + at_deprecated = Ct(Cg(P('deprecated'), 'kind')), + + at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + + at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_nodoc = Ct(Cg(P('nodoc'), 'kind')), +} + +return grammar --[[@as nvim.cdoc.grammar]] diff --git a/scripts/cdoc_parser.lua b/scripts/cdoc_parser.lua new file mode 100644 index 0000000000..5f0dc7be2c --- /dev/null +++ b/scripts/cdoc_parser.lua @@ -0,0 +1,223 @@ +local cdoc_grammar = require('scripts.cdoc_grammar') +local c_grammar = require('src.nvim.generators.c_grammar') + +--- @class nvim.cdoc.parser.param +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.note +--- @field desc string + +--- @class nvim.cdoc.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.cdoc.parser.fun +--- @field name string +--- @field params nvim.cdoc.parser.param[] +--- @field returns nvim.cdoc.parser.return[] +--- @field desc string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field notes? nvim.cdoc.parser.note[] +--- @field see? nvim.cdoc.parser.note[] + +--- @class nvim.cdoc.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.cdoc.parser.obj +--- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.cdoc.parser.obj +--- | nvim.cdoc.parser.fun +--- | nvim.cdoc.parser.brief + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.cdoc.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.cdoc.parser.State +local function process_doc_line(line, state) + line = line:gsub('^%s+@', '@') + + local parsed = cdoc_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + + local kind = parsed.kind + + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + state.last_doc_item = { + name = parsed.name, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = { { + desc = parsed.desc, + } } + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[1] + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param item table +--- @param state nvim.cdoc.parser.State +local function process_proto(item, state) + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + cur_obj.name = item.name + cur_obj.params = cur_obj.params or {} + + for _, p in ipairs(item.parameters) do + local param = { name = p[2], type = p[1] } + local added = false + for _, cp in ipairs(cur_obj.params) do + if cp.name == param.name then + cp.type = param.type + added = true + break + end + end + + if not added then + table.insert(cur_obj.params, param) + end + end + + cur_obj.returns = cur_obj.returns or { {} } + cur_obj.returns[1].type = item.return_type + + for _, a in ipairs({ + 'fast', + 'remote_only', + 'lua_only', + 'textlock', + 'textlock_allow_cmdwin', + }) do + if item[a] then + cur_obj.attrs = cur_obj.attrs or {} + table.insert(cur_obj.attrs, a) + end + end + + cur_obj.deprecated_since = item.deprecated_since + + -- Remove some arguments + for i = #cur_obj.params, 1, -1 do + local p = cur_obj.params[i] + if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then + table.remove(cur_obj.params, i) + end + end +end + +local M = {} + +--- @param filename string +--- @return {} classes +--- @return nvim.cdoc.parser.fun[] funs +--- @return string[] briefs +function M.parse(filename) + local funs = {} --- @type nvim.cdoc.parser.fun[] + local briefs = {} --- @type string[] + local state = {} --- @type nvim.cdoc.parser.State + + local txt = assert(io.open(filename, 'r')):read('*all') + + local parsed = c_grammar.grammar:match(txt) + for _, item in ipairs(parsed) do + if item.comment then + process_doc_line(item.comment, state) + else + add_doc_lines_to_obj(state) + if item[1] == 'proto' then + process_proto(item, state) + table.insert(funs, state.cur_obj) + end + local cur_obj = state.cur_obj + if cur_obj and not item.static then + if cur_obj.kind == 'brief' then + table.insert(briefs, cur_obj.desc) + end + end + state = {} + end + end + + return {}, funs, briefs +end + +-- M.parse('src/nvim/api/vim.c') + +return M diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index b7f17a2d58..895033d5af 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -3,7 +3,6 @@ -- Generator for various vimdoc and Lua type files local DEP_API_METADATA = 'build/api_metadata.mpack' -local DEP_API_DOC = 'runtime/doc/api.mpack' --- @class vim.api.metadata --- @field name string @@ -210,44 +209,65 @@ end --- @return table local function get_api_meta() - local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) - local metadata = vim.mpack.decode(mpack_f:read('*all')) --[[@as vim.api.metadata[] ]] local ret = {} --- @type table - local doc_mpack_f = assert(io.open(DEP_API_DOC, 'rb')) - local doc_metadata = vim.mpack.decode(doc_mpack_f:read('*all')) --[[@as table]] - - for _, fun in ipairs(metadata) do - if fun.lua then - local fdoc = doc_metadata[fun.name] - - local params = {} --- @type {[1]:string,[2]:string}[] - for _, p in ipairs(fun.parameters) do - local ptype, pname = p[1], p[2] - params[#params + 1] = { - pname, - api_type(ptype), - fdoc and fdoc.parameters_doc[pname] or nil, - } - end + local cdoc_parser = require('scripts.cdoc_parser') - local r = { - signature = 'NA', - name = fun.name, - params = params, - returns = api_type(fun.return_type), - deprecated = fun.deprecated_since ~= nil, - } + local f = 'src/nvim/api' - if fdoc then - if #fdoc.doc > 0 then - r.desc = table.concat(fdoc.doc, '\n') + local function include(fun) + if not vim.startswith(fun.name, 'nvim_') then + return false + end + if vim.tbl_contains(fun.attrs or {}, 'lua_only') then + return true + end + if vim.tbl_contains(fun.attrs or {}, 'remote_only') then + return false + end + return true + end + + --- @type table + local functions = {} + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + local filename = vim.fs.joinpath(f, path) + local _, funs = cdoc_parser.parse(filename) + for _, fn in ipairs(funs) do + if include(fn) then + functions[fn.name] = fn end - r.return_desc = (fdoc['return'] or {})[1] end + end + end + + for _, fun in pairs(functions) do + local deprecated = fun.deprecated_since ~= nil + + local params = {} --- @type {[1]:string,[2]:string}[] + for _, p in ipairs(fun.params) do + params[#params + 1] = { + p.name, + api_type(p.type), + not deprecated and p.desc or nil, + } + end + + local r = { + signature = 'NA', + name = fun.name, + params = params, + returns = api_type(fun.returns[1].type), + deprecated = deprecated, + } - ret[fun.name] = r + if not deprecated then + r.desc = fun.desc + r.return_desc = fun.returns[1].desc end + + ret[fun.name] = r end return ret end @@ -275,12 +295,10 @@ end --- @param fun vim.EvalFn --- @param write fun(line: string) local function render_api_meta(_f, fun, write) - if not vim.startswith(fun.name, 'nvim_') then - return - end - write('') + local text_utils = require('scripts.text_utils') + if vim.startswith(fun.name, 'nvim__') then write('--- @private') end @@ -291,10 +309,10 @@ local function render_api_meta(_f, fun, write) local desc = fun.desc if desc then + desc = text_utils.md_to_vimdoc(desc, 0, 0, 74) for _, l in ipairs(split(norm_text(desc))) do write('--- ' .. l) end - write('---') end local param_names = {} --- @type string[] @@ -303,8 +321,11 @@ local function render_api_meta(_f, fun, write) param_names[#param_names + 1] = p[1] local pdesc = p[3] if pdesc then - local pdesc_a = split(norm_text(pdesc)) - write('--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' .. pdesc_a[1]) + local s = '--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' + local indent = #('@param ' .. p[1] .. ' ') + pdesc = text_utils.md_to_vimdoc(pdesc, #s, indent, 74, true) + local pdesc_a = split(vim.trim(norm_text(pdesc))) + write(s .. pdesc_a[1]) for i = 2, #pdesc_a do if not pdesc_a[i] then break @@ -317,6 +338,7 @@ local function render_api_meta(_f, fun, write) end if fun.returns ~= '' then local ret_desc = fun.returns_desc and ' : ' .. fun.returns_desc or '' + ret_desc = text_utils.md_to_vimdoc(ret_desc, 0, 0, 74) local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns write('--- @return ' .. ret .. ret_desc) end @@ -328,8 +350,6 @@ end --- @return table local function get_api_keysets_meta() local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) - - --- @diagnostic disable-next-line:no-unknown local metadata = assert(vim.mpack.decode(mpack_f:read('*all'))) local ret = {} --- @type table diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua new file mode 100755 index 0000000000..290cd83fbc --- /dev/null +++ b/scripts/gen_vimdoc.lua @@ -0,0 +1,787 @@ +#!/usr/bin/env -S nvim -l +--- Generates Nvim :help docs from Lua/C docstrings +--- +--- The generated :help text for each function is formatted as follows: +--- - Max width of 78 columns (`TEXT_WIDTH`). +--- - Indent with spaces (not tabs). +--- - Indent of 4 columns for body text (`INDENTATION`). +--- - Function signature and helptag (right-aligned) on the same line. +--- - Signature and helptag must have a minimum of 8 spaces between them. +--- - If the signature is too long, it is placed on the line after the helptag. +--- Signature wraps with subsequent lines indented to the open parenthesis. +--- - Subsection bodies are indented an additional 4 spaces. +--- - Body consists of function description, parameters, return description, and +--- C declaration (`INCLUDE_C_DECL`). +--- - Parameters are omitted for the `void` and `Error *` types, or if the +--- parameter is marked as [out]. +--- - Each function documentation is separated by a single line. + +local luacats_parser = require('scripts.luacats_parser') +local cdoc_parser = require('scripts.cdoc_parser') +local text_utils = require('scripts.text_utils') + +local fmt = string.format + +local wrap = text_utils.wrap +local md_to_vimdoc = text_utils.md_to_vimdoc + +local TEXT_WIDTH = 78 +local INDENTATION = 4 + +--- @class (exact) nvim.gen_vimdoc.Config +--- +--- Generated documentation target, e.g. api.txt +--- @field filename string +--- +--- @field section_order string[] +--- +--- List of files/directories for doxygen to read, relative to `base_dir`. +--- @field files string[] +--- +--- @field exclude_types? true +--- +--- Section name overrides. Key: filename (e.g., vim.c) +--- @field section_name? table +--- +--- @field fn_name_pat? string +--- +--- @field fn_xform? fun(fun: nvim.luacats.parser.fun) +--- +--- For generated section names. +--- @field section_fmt fun(name: string): string +--- +--- @field helptag_fmt fun(name: string): string +--- +--- Per-function helptag. +--- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string +--- +--- @field append_only? string[] + +local function contains(t, xs) + return vim.tbl_contains(xs, t) +end + +--- @type {level:integer, prerelease:boolean}? +local nvim_api_info_ + +--- @return {level: integer, prerelease:boolean} +local function nvim_api_info() + if not nvim_api_info_ then + --- @type integer?, boolean? + local level, prerelease + for l in io.lines('CMakeLists.txt') do + --- @cast l string + if level and prerelease then + break + end + local m1 = l:match('^set%(NVIM_API_LEVEL%s+(%d+)%)') + if m1 then + level = tonumber(m1) --[[@as integer]] + end + local m2 = l:match('^set%(NVIM_API_PRERELEASE%s+(%w+)%)') + if m2 then + prerelease = m2 == 'true' + end + end + nvim_api_info_ = { level = level, prerelease = prerelease } + end + + return nvim_api_info_ +end + +--- @param fun nvim.luacats.parser.fun +--- @return string +local function fn_helptag_fmt_common(fun) + local fn_sfx = fun.table and '' or '()' + if fun.classvar then + return fmt('*%s:%s%s*', fun.classvar, fun.name, fn_sfx) + end + if fun.module then + return fmt('*%s.%s%s*', fun.module, fun.name, fn_sfx) + end + return fmt('*%s%s*', fun.name, fn_sfx) +end + +--- @type table +local config = { + api = { + filename = 'api.txt', + section_order = { + 'vim.c', + 'vimscript.c', + 'command.c', + 'options.c', + 'buffer.c', + 'extmark.c', + 'window.c', + 'win_config.c', + 'tabpage.c', + 'autocmd.c', + 'ui.c', + }, + exclude_types = true, + fn_name_pat = 'nvim_.*', + files = { 'src/nvim/api' }, + section_name = { + ['vim.c'] = 'Global', + }, + section_fmt = function(name) + return name .. ' Functions' + end, + helptag_fmt = function(name) + return fmt('*api-%s*', name:lower()) + end, + }, + lua = { + filename = 'lua.txt', + section_order = { + 'highlight.lua', + 'diff.lua', + 'mpack.lua', + 'json.lua', + 'base64.lua', + 'spell.lua', + 'builtin.lua', + '_options.lua', + '_editor.lua', + '_inspector.lua', + 'shared.lua', + 'loader.lua', + 'uri.lua', + 'ui.lua', + 'filetype.lua', + 'keymap.lua', + 'fs.lua', + 'glob.lua', + 'lpeg.lua', + 're.lua', + 'regex.lua', + 'secure.lua', + 'version.lua', + 'iter.lua', + 'snippet.lua', + 'text.lua', + }, + files = { + 'runtime/lua/vim/iter.lua', + 'runtime/lua/vim/_editor.lua', + 'runtime/lua/vim/_options.lua', + 'runtime/lua/vim/shared.lua', + 'runtime/lua/vim/loader.lua', + 'runtime/lua/vim/uri.lua', + 'runtime/lua/vim/ui.lua', + 'runtime/lua/vim/filetype.lua', + 'runtime/lua/vim/keymap.lua', + 'runtime/lua/vim/fs.lua', + 'runtime/lua/vim/highlight.lua', + 'runtime/lua/vim/secure.lua', + 'runtime/lua/vim/version.lua', + 'runtime/lua/vim/_inspector.lua', + 'runtime/lua/vim/snippet.lua', + 'runtime/lua/vim/text.lua', + 'runtime/lua/vim/glob.lua', + 'runtime/lua/vim/_meta/builtin.lua', + 'runtime/lua/vim/_meta/diff.lua', + 'runtime/lua/vim/_meta/mpack.lua', + 'runtime/lua/vim/_meta/json.lua', + 'runtime/lua/vim/_meta/base64.lua', + 'runtime/lua/vim/_meta/regex.lua', + 'runtime/lua/vim/_meta/lpeg.lua', + 'runtime/lua/vim/_meta/re.lua', + 'runtime/lua/vim/_meta/spell.lua', + }, + fn_xform = function(fun) + if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then + fun.module = 'vim' + end + + if fun.module == 'vim' and contains(fun.name, { 'cmd', 'inspect' }) then + fun.table = nil + end + + if fun.classvar or vim.startswith(fun.name, 'vim.') or fun.module == 'vim.iter' then + return + end + + fun.name = fmt('%s.%s', fun.module, fun.name) + end, + section_name = { + ['_inspector.lua'] = 'inspector', + }, + section_fmt = function(name) + name = name:lower() + if name == '_editor' then + return 'Lua module: vim' + elseif name == '_options' then + return 'LUA-VIMSCRIPT BRIDGE' + elseif name == 'builtin' then + return 'VIM' + end + if + contains(name, { + 'highlight', + 'mpack', + 'json', + 'base64', + 'diff', + 'spell', + 'regex', + 'lpeg', + 're', + }) + then + return 'VIM.' .. name:upper() + end + return 'Lua module: vim.' .. name + end, + helptag_fmt = function(name) + if name == '_editor' then + return '*lua-vim*' + elseif name == '_options' then + return '*lua-vimscript*' + end + return '*vim.' .. name:lower() .. '*' + end, + fn_helptag_fmt = function(fun) + local name = fun.name + + if vim.startswith(name, 'vim.') then + local fn_sfx = fun.table and '' or '()' + return fmt('*%s%s*', name, fn_sfx) + elseif fun.classvar == 'Option' then + return fmt('*vim.opt:%s()*', name) + end + + return fn_helptag_fmt_common(fun) + end, + append_only = { + 'shared.lua', + }, + }, + lsp = { + filename = 'lsp.txt', + section_order = { + 'lsp.lua', + 'buf.lua', + 'diagnostic.lua', + 'codelens.lua', + 'inlay_hint.lua', + 'tagfunc.lua', + 'semantic_tokens.lua', + 'handlers.lua', + 'util.lua', + 'log.lua', + 'rpc.lua', + 'protocol.lua', + }, + files = { + 'runtime/lua/vim/lsp', + 'runtime/lua/vim/lsp.lua', + }, + fn_xform = function(fun) + fun.name = fun.name:gsub('result%.', '') + end, + section_fmt = function(name) + if name:lower() == 'lsp' then + return 'Lua module: vim.lsp' + end + return 'Lua module: vim.lsp.' .. name:lower() + end, + helptag_fmt = function(name) + if name:lower() == 'lsp' then + return '*lsp-core*' + end + return fmt('*lsp-%s*', name:lower()) + end, + }, + diagnostic = { + filename = 'diagnostic.txt', + section_order = { + 'diagnostic.lua', + }, + files = { 'runtime/lua/vim/diagnostic.lua' }, + section_fmt = function() + return 'Lua module: vim.diagnostic' + end, + helptag_fmt = function() + return '*diagnostic-api*' + end, + }, + treesitter = { + filename = 'treesitter.txt', + section_order = { + 'treesitter.lua', + 'language.lua', + 'query.lua', + 'highlighter.lua', + 'languagetree.lua', + 'dev.lua', + }, + files = { + 'runtime/lua/vim/treesitter.lua', + 'runtime/lua/vim/treesitter/', + }, + section_fmt = function(name) + if name:lower() == 'treesitter' then + return 'Lua module: vim.treesitter' + end + return 'Lua module: vim.treesitter.' .. name:lower() + end, + helptag_fmt = function(name) + if name:lower() == 'treesitter' then + return '*lua-treesitter-core*' + end + return '*lua-treesitter-' .. name:lower() .. '*' + end, + }, +} + +--- @param ty string +--- @param generics table +--- @return string +local function replace_generics(ty, generics) + if ty:sub(-2) == '[]' then + local ty0 = ty:sub(1, -3) + if generics[ty0] then + return generics[ty0] .. '[]' + end + elseif ty:sub(-1) == '?' then + local ty0 = ty:sub(1, -2) + if generics[ty0] then + return generics[ty0] .. '?' + end + end + + return generics[ty] or ty +end + +--- @param ty string +--- @param generics? table +local function render_type(ty, generics) + if generics then + ty = replace_generics(ty, generics) + end + ty = ty:gsub('%s*|%s*nil', '?') + ty = ty:gsub('nil%s*|%s*(.*)', '%1?') + ty = ty:gsub('%s*|%s*', '|') + return fmt('(`%s`)', ty) +end + +--- @param p nvim.luacats.parser.param|nvim.luacats.parser.field +local function should_render_param(p) + return not p.access and not contains(p.name, { '_', 'self' }) +end + +--- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[] +--- @param generics? table +--- @param exclude_types? true +local function render_fields_or_params(xs, generics, exclude_types) + local ret = {} --- @type string[] + + xs = vim.tbl_filter(should_render_param, xs) + + local indent = 0 + for _, p in ipairs(xs) do + if p.type or p.desc then + indent = math.max(indent, #p.name + 3) + end + if exclude_types then + p.type = nil + end + end + + for _, p in ipairs(xs) do + local nm, ty = p.name, p.type + local desc = p.desc + local pnm = fmt(' • %-' .. indent .. 's', '{' .. nm .. '}') + if ty then + local pty = render_type(ty, generics) + if desc then + desc = fmt('%s %s', pty, desc) + table.insert(ret, pnm) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + else + table.insert(ret, fmt('%s %s\n', pnm, pty)) + end + else + if desc then + table.insert(ret, pnm) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + end + end + end + + return table.concat(ret) +end + +-- --- @param class lua2vimdoc.class +-- local function render_class(class) +-- writeln(fmt('*%s*', class.name)) +-- writeln() +-- if #class.fields > 0 then +-- writeln(' Fields: ~') +-- render_fields_or_params(class.fields) +-- end +-- writeln() +-- end + +-- --- @param cls table +-- local function render_classes(cls) +-- --- @diagnostic disable-next-line:no-unknown +-- for _, class in vim.spairs(cls) do +-- render_class(class) +-- end +-- end + +--- @param fun nvim.luacats.parser.fun +--- @param cfg nvim.gen_vimdoc.Config +local function render_fun_header(fun, cfg) + local ret = {} --- @type string[] + + local args = {} --- @type string[] + for _, p in ipairs(fun.params or {}) do + if p.name ~= 'self' then + args[#args + 1] = fmt('{%s}', p.name:gsub('%?$', '')) + end + end + + local nm = fun.name + if fun.classvar then + nm = fmt('%s:%s', fun.classvar, nm) + end + + local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')' + + if not cfg.fn_helptag_fmt then + cfg.fn_helptag_fmt = fn_helptag_fmt_common + end + + local tag = cfg.fn_helptag_fmt(fun) + + if #proto + #tag > TEXT_WIDTH - 8 then + table.insert(ret, fmt('%78s\n', tag)) + local name, pargs = proto:match('([^(]+%()(.*)') + table.insert(ret, name) + table.insert(ret, wrap(pargs, 0, #name, TEXT_WIDTH)) + else + local pad = TEXT_WIDTH - #proto - #tag + table.insert(ret, proto .. string.rep(' ', pad) .. tag) + end + + return table.concat(ret) +end + +--- @param returns nvim.luacats.parser.return[] +--- @param generics? table +--- @param exclude_types boolean +local function render_returns(returns, generics, exclude_types) + local ret = {} --- @type string[] + + returns = vim.deepcopy(returns) + if exclude_types then + for _, r in ipairs(returns) do + r.type = nil + end + end + + if #returns > 1 then + table.insert(ret, ' Return (multiple): ~\n') + elseif #returns == 1 and next(returns[1]) then + table.insert(ret, ' Return: ~\n') + end + + for _, p in ipairs(returns) do + local rnm, ty, desc = p.name, p.type, p.desc + local blk = '' + if ty then + blk = render_type(ty, generics) + end + if rnm then + blk = blk .. ' ' .. rnm + end + if desc then + blk = blk .. ' ' .. desc + end + table.insert(ret, md_to_vimdoc(blk, 8, 8, TEXT_WIDTH, true)) + end + + return table.concat(ret) +end + +--- @param fun nvim.luacats.parser.fun +--- @param cfg nvim.gen_vimdoc.Config +local function render_fun(fun, cfg) + if fun.access or fun.deprecated or fun.nodoc then + return + end + + if cfg.fn_name_pat and not fun.name:match(cfg.fn_name_pat) then + return + end + + if vim.startswith(fun.name, '_') or fun.name:find('[:.]_') then + return + end + + local ret = {} --- @type string[] + + table.insert(ret, render_fun_header(fun, cfg)) + table.insert(ret, '\n') + + if fun.desc then + table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH)) + end + + if fun.since then + local since = tonumber(fun.since) + local info = nvim_api_info() + if since and (since > info.level or since == info.level and info.prerelease) then + fun.notes = fun.notes or {} + table.insert(fun.notes, { desc = 'This API is pre-release (unstable).' }) + end + end + + if fun.notes then + table.insert(ret, '\n Note: ~\n') + for _, p in ipairs(fun.notes) do + table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true)) + end + end + + if fun.attrs then + table.insert(ret, '\n Attributes: ~\n') + for _, attr in ipairs(fun.attrs) do + local attr_str = ({ + textlock = 'not allowed when |textlock| is active or in the |cmdwin|', + textlock_allow_cmdwin = 'not allowed when |textlock| is active', + fast = '|api-fast|', + remote_only = '|RPC| only', + lua_only = 'Lua |vim.api| only', + })[attr] or attr + table.insert(ret, fmt(' %s\n', attr_str)) + end + end + + if fun.params and #fun.params > 0 then + local param_txt = render_fields_or_params(fun.params, fun.generics, cfg.exclude_types) + if not param_txt:match('^%s*$') then + table.insert(ret, '\n Parameters: ~\n') + ret[#ret + 1] = param_txt + end + end + + if fun.returns then + local txt = render_returns(fun.returns, fun.generics, cfg.exclude_types) + if not txt:match('^%s*$') then + table.insert(ret, '\n') + ret[#ret + 1] = txt + end + end + + if fun.see then + table.insert(ret, '\n See also: ~\n') + for _, p in ipairs(fun.see) do + table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true)) + end + end + + table.insert(ret, '\n') + return table.concat(ret) +end + +--- @param funs nvim.luacats.parser.fun[] +--- @param cfg nvim.gen_vimdoc.Config +local function render_funs(funs, cfg) + local ret = {} --- @type string[] + + for _, f in ipairs(funs) do + if cfg.fn_xform then + cfg.fn_xform(f) + end + ret[#ret + 1] = render_fun(f, cfg) + end + + -- Sort via prototype + table.sort(ret, function(a, b) + local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n') + local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n') + return a1:lower() < b1:lower() + end) + + return table.concat(ret) +end + +--- @return string +local function get_script_path() + local str = debug.getinfo(2, 'S').source:sub(2) + return str:match('(.*[/\\])') or './' +end + +local script_path = get_script_path() +local base_dir = vim.fs.dirname(assert(vim.fs.dirname(script_path))) + +local function delete_lines_below(doc_file, tokenstr) + local lines = {} --- @type string[] + local found = false + for line in io.lines(doc_file) do + if line:find(vim.pesc(tokenstr)) then + found = true + break + end + lines[#lines + 1] = line + end + if not found then + error(fmt('not found: %s in %s', tokenstr, doc_file)) + end + lines[#lines] = nil + local fp = assert(io.open(doc_file, 'w')) + fp:write(table.concat(lines, '\n')) + fp:write('\n') + fp:close() +end + +--- @param x string +local function mktitle(x) + if x == 'ui' then + return 'UI' + end + return x:sub(1, 1):upper() .. x:sub(2) +end + +--- @class nvim.gen_vimdoc.Section +--- @field name string +--- @field title string +--- @field help_tag string +--- @field funs_txt string +--- @field doc? string[] + +--- @param filename string +--- @param cfg nvim.gen_vimdoc.Config +--- @param section_docs table +--- @param funs_txt string +--- @return nvim.gen_vimdoc.Section? +local function make_section(filename, cfg, section_docs, funs_txt) + -- filename: e.g., 'autocmd.c' + -- name: e.g. 'autocmd' + local name = filename:match('(.*)%.[a-z]+') + + -- Formatted (this is what's going to be written in the vimdoc) + -- e.g., "Autocmd Functions" + local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name) + + -- section tag: e.g., "*api-autocmd*" + local help_tag = cfg.helptag_fmt(sectname) + + if funs_txt == '' and #section_docs == 0 then + return + end + + return { + name = sectname, + title = cfg.section_fmt(sectname), + help_tag = help_tag, + funs_txt = funs_txt, + doc = section_docs, + } +end + +--- @param section nvim.gen_vimdoc.Section +--- @param add_header? boolean +local function render_section(section, add_header) + local doc = {} --- @type string[] + + if add_header ~= false then + vim.list_extend(doc, { + string.rep('=', TEXT_WIDTH), + '\n', + section.title, + fmt('%' .. (TEXT_WIDTH - section.title:len()) .. 's', section.help_tag), + }) + end + + if section.doc and #section.doc > 0 then + table.insert(doc, '\n\n') + vim.list_extend(doc, section.doc) + end + + if section.funs_txt then + table.insert(doc, '\n\n') + table.insert(doc, section.funs_txt) + end + + return table.concat(doc) +end + +local parsers = { + lua = luacats_parser.parse, + c = cdoc_parser.parse, + h = cdoc_parser.parse, +} + +--- @param files string[] +local function expand_files(files) + for k, f in pairs(files) do + if vim.fn.isdirectory(f) == 1 then + table.remove(files, k) + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + table.insert(files, vim.fs.joinpath(f, path)) + end + end + end + end +end + +--- @param cfg nvim.gen_vimdoc.Config +local function gen_target(cfg) + local sections = {} --- @type table + + expand_files(cfg.files) + + for _, f in pairs(cfg.files) do + local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']] + local parser = assert(parsers[ext]) + local _, funs, briefs = parser(f) + local briefs_txt = {} --- @type string[] + for _, b in ipairs(briefs) do + briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH) + end + local funs_txt = render_funs(funs, cfg) + -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua` + local f_base = assert(vim.fs.basename(f)) + sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt) + end + + local first_section_tag = sections[cfg.section_order[1]].help_tag + local docs = {} --- @type string[] + for _, f in ipairs(cfg.section_order) do + local section = sections[f] + if section then + local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f) + table.insert(docs, render_section(section, add_sep_and_header)) + end + end + + table.insert( + docs, + fmt(' vim:tw=78:ts=8:sw=%d:sts=%d:et:ft=help:norl:\n', INDENTATION, INDENTATION) + ) + + local doc_file = vim.fs.joinpath(base_dir, 'runtime', 'doc', cfg.filename) + + if vim.uv.fs_stat(doc_file) then + delete_lines_below(doc_file, first_section_tag) + end + + local fp = assert(io.open(doc_file, 'a')) + fp:write(table.concat(docs, '\n')) + fp:close() +end + +local function run() + for _, cfg in pairs(config) do + gen_target(cfg) + end +end + +run() diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py deleted file mode 100755 index c1a2183f24..0000000000 --- a/scripts/gen_vimdoc.py +++ /dev/null @@ -1,1766 +0,0 @@ -#!/usr/bin/env python3 - -r"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. - -Also generates *.mpack files. To inspect the *.mpack structure: - :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B'))) - -Flow: - main - extract_from_xml - fmt_node_as_vimhelp \ - para_as_map } recursive - update_params_map / - render_node - -TODO: eliminate this script and use Lua+treesitter (requires parsers for C and -Lua markdown-style docstrings). - -The generated :help text for each function is formatted as follows: - - - Max width of 78 columns (`text_width`). - - Indent with spaces (not tabs). - - Indent of 4 columns for body text (`indentation`). - - Function signature and helptag (right-aligned) on the same line. - - Signature and helptag must have a minimum of 8 spaces between them. - - If the signature is too long, it is placed on the line after the helptag. - Signature wraps at `text_width - 8` characters with subsequent - lines indented to the open parenthesis. - - Subsection bodies are indented an additional 4 spaces. - - Body consists of function description, parameters, return description, and - C declaration (`INCLUDE_C_DECL`). - - Parameters are omitted for the `void` and `Error *` types, or if the - parameter is marked as [out]. - - Each function documentation is separated by a single line. -""" - -from __future__ import annotations # PEP-563, python 3.7+ - -import argparse -import collections -import dataclasses -import logging -import os -import re -import shutil -import subprocess -import sys -import textwrap -from pathlib import Path -from typing import Any, Callable, Dict, List, Tuple -from xml.dom import minidom - -if sys.version_info >= (3, 8): - from typing import Literal - -import msgpack - -Element = minidom.Element -Document = minidom.Document - -MIN_PYTHON_VERSION = (3, 7) -MIN_DOXYGEN_VERSION = (1, 9, 0) - -if sys.version_info < MIN_PYTHON_VERSION: - print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION)) - sys.exit(1) - -doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"], - universal_newlines=True).split()[0].split('.'))) - -if doxygen_version < MIN_DOXYGEN_VERSION: - print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION)) - print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version)) - sys.exit(1) - - -# Need a `nvim` that supports `-l`, try the local build -nvim_path = Path(__file__).parent / "../build/bin/nvim" -if nvim_path.exists(): - nvim = nvim_path.resolve() -else: - # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works. - nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True) - nvim_version = [line for line in nvim_out.split('\n') - if '-l ' in line] - if len(nvim_version) == 0: - print(( - "\nYou need to have a local Neovim build or a `nvim` version 0.9 for `-l` " - "support to build the documentation.")) - sys.exit(1) - nvim = 'nvim' - - -# DEBUG = ('DEBUG' in os.environ) -INCLUDE_C_DECL = os.environ.get('INCLUDE_C_DECL', '0') != '0' -INCLUDE_DEPRECATED = os.environ.get('INCLUDE_DEPRECATED', '0') != '0' - -log = logging.getLogger(__name__) - -LOG_LEVELS = { - logging.getLevelName(level): level for level in [ - logging.DEBUG, logging.INFO, logging.ERROR - ] -} - -text_width = 78 -indentation = 4 -SECTION_SEP = '=' * text_width - -script_path = os.path.abspath(__file__) -base_dir = os.path.dirname(os.path.dirname(script_path)) -out_dir = os.path.join(base_dir, 'tmp-{target}-doc') -filter_cmd = '%s %s' % (sys.executable, script_path) -msgs = [] # Messages to show on exit. -lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') - - -SectionName = str - -Docstring = str # Represents (formatted) vimdoc string - -FunctionName = str - - -@dataclasses.dataclass -class Config: - """Config for documentation.""" - - mode: Literal['c', 'lua'] - - filename: str - """Generated documentation target, e.g. api.txt""" - - section_order: List[str] - """Section ordering.""" - - files: List[str] - """List of files/directories for doxygen to read, relative to `base_dir`.""" - - file_patterns: str - """file patterns used by doxygen.""" - - section_name: Dict[str, SectionName] - """Section name overrides. Key: filename (e.g., vim.c)""" - - section_fmt: Callable[[SectionName], str] - """For generated section names.""" - - helptag_fmt: Callable[[SectionName], str] - """Section helptag.""" - - fn_helptag_fmt: Callable[[str, str, bool], str] - """Per-function helptag.""" - - module_override: Dict[str, str] - """Module name overrides (for Lua).""" - - append_only: List[str] - """Append the docs for these modules, do not start a new section.""" - - fn_name_prefix: str - """Only function with this prefix are considered""" - - fn_name_fmt: Callable[[str, str], str] | None = None - - include_tables: bool = True - - -CONFIG: Dict[str, Config] = { - 'api': Config( - mode='c', - filename = 'api.txt', - # Section ordering. - section_order=[x for x in [ - 'vim.c', - 'vimscript.c', - 'command.c', - 'options.c', - 'buffer.c', - 'extmark.c', - 'window.c', - 'win_config.c', - 'tabpage.c', - 'autocmd.c', - 'ui.c', - 'deprecated.c' if INCLUDE_DEPRECATED else '' - ] if x], - files=['src/nvim/api'], - file_patterns = '*.h *.c', - fn_name_prefix = 'nvim_', - section_name={ - 'vim.c': 'Global', - }, - section_fmt=lambda name: f'{name} Functions', - helptag_fmt=lambda name: f'*api-{name.lower()}*', - fn_helptag_fmt=lambda fstem, name, istbl: f'*{name}()*', - module_override={}, - append_only=[], - ), - 'lua': Config( - mode='lua', - filename='lua.txt', - section_order=[ - 'highlight.lua', - 'diff.lua', - 'mpack.lua', - 'json.lua', - 'base64.lua', - 'spell.lua', - 'builtin.lua', - '_options.lua', - '_editor.lua', - '_inspector.lua', - 'shared.lua', - 'loader.lua', - 'uri.lua', - 'ui.lua', - 'filetype.lua', - 'keymap.lua', - 'fs.lua', - 'glob.lua', - 'lpeg.lua', - 're.lua', - 'regex.lua', - 'secure.lua', - 'version.lua', - 'iter.lua', - 'snippet.lua', - 'text.lua', - ], - files=[ - 'runtime/lua/vim/iter.lua', - 'runtime/lua/vim/_editor.lua', - 'runtime/lua/vim/_options.lua', - 'runtime/lua/vim/shared.lua', - 'runtime/lua/vim/loader.lua', - 'runtime/lua/vim/uri.lua', - 'runtime/lua/vim/ui.lua', - 'runtime/lua/vim/filetype.lua', - 'runtime/lua/vim/keymap.lua', - 'runtime/lua/vim/fs.lua', - 'runtime/lua/vim/highlight.lua', - 'runtime/lua/vim/secure.lua', - 'runtime/lua/vim/version.lua', - 'runtime/lua/vim/_inspector.lua', - 'runtime/lua/vim/snippet.lua', - 'runtime/lua/vim/text.lua', - 'runtime/lua/vim/glob.lua', - 'runtime/lua/vim/_meta/builtin.lua', - 'runtime/lua/vim/_meta/diff.lua', - 'runtime/lua/vim/_meta/mpack.lua', - 'runtime/lua/vim/_meta/json.lua', - 'runtime/lua/vim/_meta/base64.lua', - 'runtime/lua/vim/_meta/regex.lua', - 'runtime/lua/vim/_meta/lpeg.lua', - 'runtime/lua/vim/_meta/re.lua', - 'runtime/lua/vim/_meta/spell.lua', - ], - file_patterns='*.lua', - fn_name_prefix='', - fn_name_fmt=lambda fstem, name: ( - name if fstem in [ 'vim.iter' ] else - f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else - f'vim.{name}' if fstem == '_options' and not name[0].isupper() else - f'{fstem}.{name}' if fstem.startswith('vim') else - name - ), - section_name={ - 'lsp.lua': 'core', - '_inspector.lua': 'inspector', - }, - section_fmt=lambda name: ( - 'Lua module: vim' if name.lower() == '_editor' else - 'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else - f'VIM.{name.upper()}' if name.lower() in [ - 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', - 'regex', 'lpeg', 're', - ] else - 'VIM' if name.lower() == 'builtin' else - f'Lua module: vim.{name.lower()}'), - helptag_fmt=lambda name: ( - '*lua-vim*' if name.lower() == '_editor' else - '*lua-vimscript*' if name.lower() == '_options' else - f'*vim.{name.lower()}*'), - fn_helptag_fmt=lambda fstem, name, istbl: ( - f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else - # Exclude fstem for methods - f'*{name}()*' if ':' in name else - f'*vim.{name}()*' if fstem.lower() == '_editor' else - f'*vim.{name}*' if fstem.lower() == '_options' and istbl else - # Prevents vim.regex.regex - f'*{fstem}()*' if fstem.endswith('.' + name) else - f'*{fstem}.{name}{"" if istbl else "()"}*' - ), - module_override={ - # `shared` functions are exposed on the `vim` module. - 'shared': 'vim', - '_inspector': 'vim', - 'uri': 'vim', - 'ui': 'vim.ui', - 'loader': 'vim.loader', - 'filetype': 'vim.filetype', - 'keymap': 'vim.keymap', - 'fs': 'vim.fs', - 'highlight': 'vim.highlight', - 'secure': 'vim.secure', - 'version': 'vim.version', - 'iter': 'vim.iter', - 'diff': 'vim', - 'builtin': 'vim', - 'mpack': 'vim.mpack', - 'json': 'vim.json', - 'base64': 'vim.base64', - 'regex': 'vim.regex', - 'lpeg': 'vim.lpeg', - 're': 'vim.re', - 'spell': 'vim.spell', - 'snippet': 'vim.snippet', - 'text': 'vim.text', - 'glob': 'vim.glob', - }, - append_only=[ - 'shared.lua', - ], - ), - 'lsp': Config( - mode='lua', - filename='lsp.txt', - section_order=[ - 'lsp.lua', - 'buf.lua', - 'diagnostic.lua', - 'codelens.lua', - 'inlay_hint.lua', - 'tagfunc.lua', - 'semantic_tokens.lua', - 'handlers.lua', - 'util.lua', - 'log.lua', - 'rpc.lua', - 'protocol.lua', - ], - files=[ - 'runtime/lua/vim/lsp', - 'runtime/lua/vim/lsp.lua', - ], - file_patterns='*.lua', - fn_name_prefix='', - section_name={'lsp.lua': 'lsp'}, - section_fmt=lambda name: ( - 'Lua module: vim.lsp' - if name.lower() == 'lsp' - else f'Lua module: vim.lsp.{name.lower()}'), - helptag_fmt=lambda name: ( - '*lsp-core*' - if name.lower() == 'lsp' - else f'*lsp-{name.lower()}*'), - fn_helptag_fmt=lambda fstem, name, istbl: ( - f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else - # HACK. TODO(justinmk): class/structure support in lua2dox - '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else - f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'), - module_override={}, - append_only=[], - ), - 'diagnostic': Config( - mode='lua', - filename='diagnostic.txt', - section_order=[ - 'diagnostic.lua', - ], - files=['runtime/lua/vim/diagnostic.lua'], - file_patterns='*.lua', - fn_name_prefix='', - include_tables=False, - section_name={'diagnostic.lua': 'diagnostic'}, - section_fmt=lambda _: 'Lua module: vim.diagnostic', - helptag_fmt=lambda _: '*diagnostic-api*', - fn_helptag_fmt=lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*', - module_override={}, - append_only=[], - ), - 'treesitter': Config( - mode='lua', - filename='treesitter.txt', - section_order=[ - 'treesitter.lua', - 'language.lua', - 'query.lua', - 'highlighter.lua', - 'languagetree.lua', - 'dev.lua', - ], - files=[ - 'runtime/lua/vim/treesitter.lua', - 'runtime/lua/vim/treesitter/', - ], - file_patterns='*.lua', - fn_name_prefix='', - section_name={}, - section_fmt=lambda name: ( - 'Lua module: vim.treesitter' - if name.lower() == 'treesitter' - else f'Lua module: vim.treesitter.{name.lower()}'), - helptag_fmt=lambda name: ( - '*lua-treesitter-core*' - if name.lower() == 'treesitter' - else f'*lua-treesitter-{name.lower()}*'), - fn_helptag_fmt=lambda fstem, name, istbl: ( - f'*vim.{fstem}.{name}()*' - if fstem == 'treesitter' - else f'*{name}()*' - if name[0].isupper() - else f'*vim.treesitter.{fstem}.{name}()*'), - module_override={}, - append_only=[], - ), -} - -param_exclude = ( - 'channel_id', -) - -# Annotations are displayed as line items after API function descriptions. -annotation_map = { - 'FUNC_API_FAST': '|api-fast|', - 'FUNC_API_TEXTLOCK': 'not allowed when |textlock| is active or in the |cmdwin|', - 'FUNC_API_TEXTLOCK_ALLOW_CMDWIN': 'not allowed when |textlock| is active', - 'FUNC_API_REMOTE_ONLY': '|RPC| only', - 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only', -} - - -def nvim_api_info() -> Tuple[int, bool]: - """Returns NVIM_API_LEVEL, NVIM_API_PRERELEASE from CMakeLists.txt""" - if not hasattr(nvim_api_info, 'LEVEL'): - script_dir = os.path.dirname(os.path.abspath(__file__)) - cmake_file_path = os.path.join(script_dir, '..', 'CMakeLists.txt') - with open(cmake_file_path, 'r') as cmake_file: - cmake_content = cmake_file.read() - - api_level_match = re.search(r'set\(NVIM_API_LEVEL (\d+)\)', cmake_content) - api_prerelease_match = re.search( - r'set\(NVIM_API_PRERELEASE (\w+)\)', cmake_content - ) - - if not api_level_match or not api_prerelease_match: - raise RuntimeError( - 'Could not find NVIM_API_LEVEL or NVIM_API_PRERELEASE in CMakeLists.txt' - ) - - nvim_api_info.LEVEL = int(api_level_match.group(1)) - nvim_api_info.PRERELEASE = api_prerelease_match.group(1).lower() == 'true' - - return nvim_api_info.LEVEL, nvim_api_info.PRERELEASE - - -# Raises an error with details about `o`, if `cond` is in object `o`, -# or if `cond()` is callable and returns True. -def debug_this(o, cond=True): - name = '' - if cond is False: - return - if not isinstance(o, str): - try: - name = o.nodeName - o = o.toprettyxml(indent=' ', newl='\n') - except Exception: - pass - if (cond is True - or (callable(cond) and cond()) - or (not callable(cond) and cond in o)): - raise RuntimeError('xxx: {}\n{}'.format(name, o)) - - -# Appends a message to a list which will be printed on exit. -def msg(s): - msgs.append(s) - - -# Print all collected messages. -def msg_report(): - for m in msgs: - print(f' {m}') - - -# Print collected messages, then throw an exception. -def fail(s): - msg_report() - raise RuntimeError(s) - - -def find_first(parent, name): - """Finds the first matching node within parent.""" - sub = parent.getElementsByTagName(name) - if not sub: - return None - return sub[0] - - -def iter_children(parent, name): - """Yields matching child nodes within parent.""" - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: - yield child - - -def get_child(parent, name): - """Gets the first matching child node.""" - for child in iter_children(parent, name): - return child - return None - - -def self_or_child(n): - """Gets the first child node, or self.""" - if len(n.childNodes) == 0: - return n - return n.childNodes[0] - - -def align_tags(line): - tag_regex = r"\s(\*.+?\*)(?:\s|$)" - tags = re.findall(tag_regex, line) - - if len(tags) > 0: - line = re.sub(tag_regex, "", line) - tags = " " + " ".join(tags) - line = line + (" " * (78 - len(line) - len(tags))) + tags - return line - - -def clean_lines(text): - """Removes superfluous lines. - - The beginning and end of the string is trimmed. Empty lines are collapsed. - """ - return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text)) - - -def is_blank(text): - return '' == clean_lines(text) - - -def get_text(n): - """Recursively concatenates all text in a node tree.""" - text = '' - if n.nodeType == n.TEXT_NODE: - return n.data - if n.nodeName == 'computeroutput': - for node in n.childNodes: - text += get_text(node) - return '`{}`'.format(text) - if n.nodeName == 'sp': # space, used in "programlisting" nodes - return ' ' - for node in n.childNodes: - if node.nodeType == node.TEXT_NODE: - text += node.data - elif node.nodeType == node.ELEMENT_NODE: - text += get_text(node) - return text - - -# Gets the length of the last line in `text`, excluding newline ("\n") char. -def len_lastline(text): - lastnl = text.rfind('\n') - if -1 == lastnl: - return len(text) - if '\n' == text[-1]: - return lastnl - (1 + text.rfind('\n', 0, lastnl)) - return len(text) - (1 + lastnl) - - -def len_lastline_withoutindent(text, indent): - n = len_lastline(text) - return (n - len(indent)) if n > len(indent) else 0 - - -# Returns True if node `n` contains only inline (not block-level) elements. -def is_inline(n): - # if len(n.childNodes) == 0: - # return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput' - for c in n.childNodes: - if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput': - return False - if not is_inline(c): - return False - return True - - -def doc_wrap(text, prefix='', width=70, func=False, indent=None) -> str: - """Wraps text to `width`. - - First line is prefixed with `prefix`, subsequent lines are aligned. - If `func` is True, only wrap at commas. - """ - if not width: - # return prefix + text - return text - - # Whitespace used to indent all lines except the first line. - indent = ' ' * len(prefix) if indent is None else indent - indent_only = (prefix == '' and indent is not None) - - if func: - lines = [prefix] - for part in text.split(', '): - if part[-1] not in ');': - part += ', ' - if len(lines[-1]) + len(part) > width: - lines.append(indent) - lines[-1] += part - return '\n'.join(x.rstrip() for x in lines).rstrip() - - # XXX: Dummy prefix to force TextWrapper() to wrap the first line. - if indent_only: - prefix = indent - - tw = textwrap.TextWrapper(break_long_words=False, - break_on_hyphens=False, - width=width, - initial_indent=prefix, - subsequent_indent=indent) - result = '\n'.join(tw.wrap(text.strip())) - - # XXX: Remove the dummy prefix. - if indent_only: - result = result[len(indent):] - - return result - - -def max_name(names): - if len(names) == 0: - return 0 - return max(len(name) for name in names) - - -def update_params_map(parent, ret_map, width=text_width - indentation): - """Updates `ret_map` with name:desc key-value pairs extracted - from Doxygen XML node `parent`. - """ - params = collections.OrderedDict() - for node in parent.childNodes: - if node.nodeType == node.TEXT_NODE: - continue - name_node = find_first(node, 'parametername') - if name_node.getAttribute('direction') == 'out': - continue - name = get_text(name_node) - if name in param_exclude: - continue - params[name.strip()] = node - max_name_len = max_name(params.keys()) + 8 - # `ret_map` is a name:desc map. - for name, node in params.items(): - desc = '' - desc_node = get_child(node, 'parameterdescription') - if desc_node: - desc = fmt_node_as_vimhelp( - desc_node, width=width, indent=(' ' * max_name_len)) - ret_map[name] = desc - return ret_map - - -def render_node(n: Element, text: str, prefix='', *, - indent: str = '', - width: int = (text_width - indentation), - fmt_vimhelp: bool = False): - """Renders a node as Vim help text, recursively traversing all descendants.""" - - def ind(s): - return s if fmt_vimhelp else '' - - # Get the current column offset from the last line of `text` - # (needed to appropriately wrap multiple and contiguous inline elements) - col_offset: int = len_lastline(text) - - text = '' - # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) - # text += (int(not space_preceding) * ' ') - - if n.nodeName == 'preformatted': - o = get_text(n) - ensure_nl = '' if o[-1] == '\n' else '\n' - if o[0:4] == 'lua\n': - text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1]) - elif o[0:4] == 'vim\n': - text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1]) - elif o[0:5] == 'help\n': - text += o[4:-1] - else: - text += '>{}{}\n<'.format(ensure_nl, o) - elif n.nodeName == 'programlisting': # codeblock (```) - o = get_text(n) - text += '>' - if 'filename' in n.attributes: - filename = n.attributes['filename'].value - text += filename.lstrip('.') - - text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4)) - elif is_inline(n): - o = get_text(n).strip() - if o: - DEL = chr(127) # a dummy character to pad for proper line wrap - assert len(DEL) == 1 - dummy_padding = DEL * max(0, col_offset - len(prefix)) - text += doc_wrap(dummy_padding + o, - prefix=prefix, indent=indent, width=width - ).replace(DEL, "") - elif n.nodeName == 'verbatim': - # TODO: currently we don't use this. The "[verbatim]" hint is there as - # a reminder that we must decide how to format this if we do use it. - text += ' [verbatim] {}'.format(get_text(n)) - elif n.nodeName == 'listitem': - for c in n.childNodes: - result = render_node( - c, - text, - indent=indent + (' ' * len(prefix)), - width=width - ) - if is_blank(result): - continue - text += indent + prefix + result - elif n.nodeName in ('para', 'heading'): - did_prefix = False - for c in n.childNodes: - c_text = render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width) - if (is_inline(c) - and '' != c_text.strip() - and text - and text[-1] not in (' ', '(', '|') - and not c_text.startswith(')')): - text += ' ' - text += c_text - did_prefix = True - elif n.nodeName == 'itemizedlist': - for c in n.childNodes: - text += '{}\n'.format(render_node(c, text, prefix='• ', - indent=indent, width=width)) - elif n.nodeName == 'orderedlist': - i = 1 - for c in n.childNodes: - if is_blank(get_text(c)): - text += '\n' - continue - text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i), - indent=indent, width=width)) - i = i + 1 - elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'): - text += ind(' ') - for c in n.childNodes: - if is_blank(render_node(c, text, prefix='• ', indent=' ', width=width)): - continue - text += render_node(c, text, prefix='• ', indent=' ', width=width) - # text += '\n' - elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'): - text += 'Warning:\n ' - for c in n.childNodes: - text += render_node(c, text, indent=' ', width=width) - text += '\n' - elif n.nodeName == 'simplesect' and 'see' == n.getAttribute('kind'): - text += ind(' ') - # Example: - # - # |autocommand| - # - for c in n.childNodes: - text += render_node(c, text, prefix='• ', indent=' ', width=width) - elif n.nodeName == 'simplesect' and 'return' == n.getAttribute('kind'): - text += ind(' ') - for c in n.childNodes: - text += render_node(c, text, indent=' ', width=width) - elif n.nodeName == 'computeroutput': - return get_text(n) - else: - raise RuntimeError('unhandled node type: {}\n{}'.format( - n.nodeName, n.toprettyxml(indent=' ', newl='\n'))) - - return text - - -def para_as_map(parent: Element, - indent: str = '', - width: int = (text_width - indentation), - ): - """Extracts a Doxygen XML node to a map. - - Keys: - 'text': Text from this element - 'note': List of @note strings - 'params': map - 'return': List of @return strings - 'seealso': List of @see strings - 'xrefs': ? - """ - chunks = { - 'text': '', - 'note': [], - 'params': collections.OrderedDict(), - 'return': [], - 'seealso': [], - 'prerelease': False, - 'xrefs': [] - } - - # Ordered dict of ordered lists. - groups = collections.OrderedDict([ - ('note', []), - ('params', []), - ('return', []), - ('seealso', []), - ('xrefs', []), - ]) - - # Gather nodes into groups. Mostly this is because we want "parameterlist" - # nodes to appear together. - text = '' - kind = '' - if is_inline(parent): - # Flatten inline text from a tree of non-block nodes. - text = doc_wrap(render_node(parent, ""), - indent=indent, width=width) - else: - prev = None # Previous node - for child in parent.childNodes: - if child.nodeName == 'parameterlist': - groups['params'].append(child) - elif child.nodeName == 'xrefsect': - groups['xrefs'].append(child) - elif child.nodeName == 'simplesect': - kind = child.getAttribute('kind') - if kind == 'note': - groups['note'].append(child) - elif kind == 'return': - groups['return'].append(child) - elif kind == 'see': - groups['seealso'].append(child) - elif kind == 'warning': - text += render_node(child, text, indent=indent, width=width) - elif kind == 'since': - since_match = re.match(r'^(\d+)', get_text(child)) - since = int(since_match.group(1)) if since_match else 0 - NVIM_API_LEVEL, NVIM_API_PRERELEASE = nvim_api_info() - if since > NVIM_API_LEVEL or ( - since == NVIM_API_LEVEL and NVIM_API_PRERELEASE - ): - chunks['prerelease'] = True - else: - raise RuntimeError('unhandled simplesect: {}\n{}'.format( - child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) - else: - child_text = render_node(child, text, indent=indent, width=width) - if (prev is not None - and is_inline(self_or_child(prev)) - and is_inline(self_or_child(child)) - and '' != get_text(self_or_child(child)).strip() - and text - and text[-1] not in (' ', '(', '|') - and not child_text.startswith(')')): - text += ' ' - - text += child_text - prev = child - - chunks['text'] += text - - # Generate map from the gathered items. - if len(groups['params']) > 0: - for child in groups['params']: - update_params_map(child, ret_map=chunks['params'], width=width) - for child in groups['note']: - chunks['note'].append(render_node( - child, '', indent=indent, width=width).rstrip()) - for child in groups['return']: - chunks['return'].append(render_node( - child, '', indent=indent, width=width)) - for child in groups['seealso']: - # Example: - # - # |autocommand| - # - chunks['seealso'].append(render_node( - child, '', indent=indent, width=width)) - - xrefs = set() - for child in groups['xrefs']: - # XXX: Add a space (or any char) to `title` here, otherwise xrefs - # ("Deprecated" section) acts very weird... - title = get_text(get_child(child, 'xreftitle')) + ' ' - xrefs.add(title) - xrefdesc = get_text(get_child(child, 'xrefdescription')) - chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title), - width=width) + '\n') - - return chunks, xrefs - - -def is_program_listing(para): - """ - Return True if `para` contains a "programlisting" (i.e. a Markdown code - block ```). - - Sometimes a element will have only a single "programlisting" child - node, but othertimes it will have extra whitespace around the - "programlisting" node. - - @param para XML node - @return True if is a programlisting - """ - - # Remove any child text nodes that are only whitespace - children = [ - n for n in para.childNodes - if n.nodeType != n.TEXT_NODE or n.data.strip() != '' - ] - - return len(children) == 1 and children[0].nodeName == 'programlisting' - - -FunctionParam = Tuple[ - str, # type - str, # parameter name -] - -@dataclasses.dataclass -class FunctionDoc: - """Data structure for function documentation. Also exported as msgpack.""" - - annotations: List[str] - """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map""" - - notes: List[Docstring] - """Notes: (@note strings)""" - - signature: str - """Function signature with *tags*.""" - - parameters: List[FunctionParam] - """Parameters: (type, name)""" - - parameters_doc: Dict[str, Docstring] - """Parameters documentation. Key is parameter name, value is doc.""" - - doc: List[Docstring] - """Main description for the function. Separated by paragraph.""" - - return_: List[Docstring] - """Return:, or Return (multiple): (@return strings)""" - - seealso: List[Docstring] - """See also: (@see strings)""" - - xrefs: List[Docstring] - """XRefs. Currently only used to track Deprecated functions.""" - - # for INCLUDE_C_DECL - c_decl: str | None = None - - prerelease: bool = False - - def export_mpack(self) -> Dict[str, Any]: - """Convert a dict to be exported as mpack data.""" - exported = self.__dict__.copy() - del exported['notes'] - del exported['c_decl'] - del exported['prerelease'] - del exported['xrefs'] - exported['return'] = exported.pop('return_') - return exported - - def doc_concatenated(self) -> Docstring: - """Concatenate all the paragraphs in `doc` into a single string, but - remove blank lines before 'programlisting' blocks. #25127 - - BEFORE (without programlisting processing): - ```vimdoc - Example: - - >vim - :echo nvim_get_color_by_name("Pink") - < - ``` - - AFTER: - ```vimdoc - Example: >vim - :echo nvim_get_color_by_name("Pink") - < - ``` - """ - def is_program_listing(paragraph: str) -> bool: - lines = paragraph.strip().split('\n') - return lines[0].startswith('>') and lines[-1] == '<' - - rendered = [] - for paragraph in self.doc: - if is_program_listing(paragraph): - rendered.append(' ') # Example: >vim - elif rendered: - rendered.append('\n\n') - rendered.append(paragraph) - return ''.join(rendered) - - def render(self) -> Docstring: - """Renders function documentation as Vim :help text.""" - rendered_blocks: List[Docstring] = [] - - def fmt_param_doc(m): - """Renders a params map as Vim :help text.""" - max_name_len = max_name(m.keys()) + 4 - out = '' - for name, desc in m.items(): - if name == 'self': - continue - name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) - out += '{}{}\n'.format(name, desc) - return out.rstrip() - - # Generate text from the gathered items. - chunks: List[Docstring] = [self.doc_concatenated()] - - notes = [] - if self.prerelease: - notes = [" This API is pre-release (unstable)."] - notes += self.notes - if len(notes) > 0: - chunks.append('\nNote: ~') - for s in notes: - chunks.append(' ' + s) - - if self.parameters_doc: - chunks.append('\nParameters: ~') - chunks.append(fmt_param_doc(self.parameters_doc)) - - if self.return_: - chunks.append('\nReturn (multiple): ~' if len(self.return_) > 1 - else '\nReturn: ~') - for s in self.return_: - chunks.append(' ' + s) - - if self.seealso: - chunks.append('\nSee also: ~') - for s in self.seealso: - chunks.append(' ' + s) - - # Note: xrefs are currently only used to remark "Deprecated: " - # for deprecated functions; visible when INCLUDE_DEPRECATED is set - for s in self.xrefs: - chunks.append('\n' + s) - - rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) - rendered_blocks.append('') - - return clean_lines('\n'.join(rendered_blocks).strip()) - - -def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=''): - """Renders (nested) Doxygen nodes as Vim :help text. - - Only handles "text" nodes. Used for individual elements (see render_node()) - and in extract_defgroups(). - - NB: Blank lines in a docstring manifest as tags. - """ - rendered_blocks = [] - - for child in parent.childNodes: - para, _ = para_as_map(child, indent, width) - - # 'programlisting' blocks are Markdown code blocks. Do not include - # these as a separate paragraph, but append to the last non-empty line - # in the text - if is_program_listing(child): - while rendered_blocks and rendered_blocks[-1] == '': - rendered_blocks.pop() - rendered_blocks[-1] += ' ' + para['text'] - continue - - # Generate text from the gathered items. - chunks = [para['text']] - - rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) - rendered_blocks.append('') - - return clean_lines('\n'.join(rendered_blocks).strip()) - - -def extract_from_xml(filename, target, *, - width: int, fmt_vimhelp: bool) -> Tuple[ - Dict[FunctionName, FunctionDoc], - Dict[FunctionName, FunctionDoc], -]: - """Extracts Doxygen info as maps without formatting the text. - - Returns two maps: - 1. Functions - 2. Deprecated functions - - The `fmt_vimhelp` variable controls some special cases for use by - fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) - """ - config: Config = CONFIG[target] - - fns: Dict[FunctionName, FunctionDoc] = {} - deprecated_fns: Dict[FunctionName, FunctionDoc] = {} - - dom = minidom.parse(filename) - compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) - for member in dom.getElementsByTagName('memberdef'): - if member.getAttribute('static') == 'yes' or \ - member.getAttribute('kind') != 'function' or \ - member.getAttribute('prot') == 'private' or \ - get_text(get_child(member, 'name')).startswith('_'): - continue - - loc = find_first(member, 'location') - if 'private' in loc.getAttribute('file'): - continue - - return_type = get_text(get_child(member, 'type')) - if return_type == '': - continue - - if 'local_function' in return_type: # Special from lua2dox.lua. - continue - - istbl = return_type.startswith('table') # Special from lua2dox.lua. - if istbl and not config.include_tables: - continue - - if return_type.startswith(('ArrayOf', 'DictionaryOf')): - parts = return_type.strip('_').split('_') - return_type = '{}({})'.format(parts[0], ', '.join(parts[1:])) - - name = get_text(get_child(member, 'name')) - - annotations = get_text(get_child(member, 'argsstring')) - if annotations and ')' in annotations: - annotations = annotations.rsplit(')', 1)[-1].strip() - # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of - # non-void functions. Special-case void functions here. - if name == 'nvim_get_mode' and len(annotations) == 0: - annotations += 'FUNC_API_FAST' - annotations = filter(None, map(lambda x: annotation_map.get(x), - annotations.split())) - - params = [] - type_length = 0 - - for param in iter_children(member, 'param'): - param_type = get_text(get_child(param, 'type')).strip() - param_name = '' - declname = get_child(param, 'declname') - if declname: - param_name = get_text(declname).strip() - elif config.mode == 'lua': - # XXX: this is what lua2dox gives us... - param_name = param_type - param_type = '' - - if param_name in param_exclude: - continue - - if fmt_vimhelp and param_type.endswith('*'): - param_type = param_type.strip('* ') - param_name = '*' + param_name - - type_length = max(type_length, len(param_type)) - params.append((param_type, param_name)) - - # Handle Object Oriented style functions here. - # We make sure they have "self" in the parameters, - # and a parent function - if return_type.startswith('function') \ - and len(return_type.split(' ')) >= 2 \ - and any(x[1] == 'self' for x in params): - split_return = return_type.split(' ') - name = f'{split_return[1]}:{name}' - params = [x for x in params if x[1] != 'self'] - - c_args = [] - for param_type, param_name in params: - c_args.append((' ' if fmt_vimhelp else '') + ( - '%s %s' % (param_type.ljust(type_length), param_name)).strip()) - - if not fmt_vimhelp: - pass - else: - fstem = '?' - if '.' in compoundname: - fstem = compoundname.split('.')[0] - fstem = config.module_override.get(fstem, fstem) - vimtag = config.fn_helptag_fmt(fstem, name, istbl) - - if config.fn_name_fmt: - name = config.fn_name_fmt(fstem, name) - - if istbl: - aopen, aclose = '', '' - else: - aopen, aclose = '(', ')' - - prefix = name + aopen - suffix = ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error', 'Arena', - 'lua_State')) + aclose - - if not fmt_vimhelp: - c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) - signature = prefix + suffix - else: - c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, - ',\n'.join(c_args)), - ' ') - - # Minimum 8 chars between signature and vimtag - lhs = (width - 8) - len(vimtag) - - if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(width) + '\n' - signature += doc_wrap(suffix, width=width, prefix=prefix, - func=True) - else: - signature = prefix + suffix - signature += vimtag.rjust(width - len(signature)) - - # Tracks `xrefsect` titles. As of this writing, used only for separating - # deprecated functions. - xrefs_all = set() - paras: List[Dict[str, Any]] = [] # paras means paragraphs! - brief_desc = find_first(member, 'briefdescription') - if brief_desc: - for child in brief_desc.childNodes: - para, xrefs = para_as_map(child) - paras.append(para) - xrefs_all.update(xrefs) - - desc = find_first(member, 'detaileddescription') - if desc: - paras_detail = [] # override briefdescription - for child in desc.childNodes: - para, xrefs = para_as_map(child) - paras_detail.append(para) - xrefs_all.update(xrefs) - log.debug( - textwrap.indent( - re.sub(r'\n\s*\n+', '\n', - desc.toprettyxml(indent=' ', newl='\n')), - ' ' * indentation)) - - # override briefdescription, if detaileddescription is not empty - # (note: briefdescription can contain some erroneous luadoc - # comments from preceding comments, this is a bug of lua2dox) - if any((para['text'] or para['note'] or para['params'] or - para['return'] or para['seealso'] - ) for para in paras_detail): - paras = paras_detail - - fn = FunctionDoc( - annotations=list(annotations), - notes=[], - signature=signature, - parameters=params, - parameters_doc=collections.OrderedDict(), - doc=[], - return_=[], - seealso=[], - xrefs=[], - ) - - for m in paras: - if m.get('text', ''): - fn.doc.append(m['text']) - if 'params' in m: - # Merge OrderedDicts. - fn.parameters_doc.update(m['params']) - if 'return' in m and len(m['return']) > 0: - fn.return_ += m['return'] - if 'seealso' in m and len(m['seealso']) > 0: - fn.seealso += m['seealso'] - if m.get('prerelease', False): - fn.prerelease = True - if 'note' in m: - fn.notes += m['note'] - if 'xrefs' in m: - fn.xrefs += m['xrefs'] - - if INCLUDE_C_DECL: - fn.c_decl = c_decl - - if 'Deprecated' in str(xrefs_all): - deprecated_fns[name] = fn - elif name.startswith(config.fn_name_prefix): - fns[name] = fn - - # sort functions by name (lexicographically) - fns = collections.OrderedDict(sorted( - fns.items(), - key=lambda key_item_tuple: key_item_tuple[0].lower(), - )) - deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items())) - return fns, deprecated_fns - - -def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: - """Entrypoint for generating Vim :help from from Doxygen XML. - - Returns 2 items: - 1. Vim help text for functions found in `filename`. - 2. Vim help text for deprecated functions. - """ - config: Config = CONFIG[target] - - fns_txt = {} # Map of func_name:vim-help-text. - deprecated_fns_txt = {} # Map of func_name:vim-help-text. - - fns: Dict[FunctionName, FunctionDoc] - deprecated_fns: Dict[FunctionName, FunctionDoc] - fns, deprecated_fns = extract_from_xml( - filename, target, width=text_width, fmt_vimhelp=True) - - def _handle_fn(fn_name: FunctionName, fn: FunctionDoc, - fns_txt: Dict[FunctionName, Docstring], deprecated=False): - # Generate Vim :help for parameters. - - # Generate body from FunctionDoc, not XML nodes - doc = fn.render() - if not doc and fn_name.startswith("nvim__"): - return - if not doc: - doc = ('TODO: Documentation' if not deprecated - else 'Deprecated.') - - # Annotations: put before Parameters - annotations: str = '\n'.join(fn.annotations) - if annotations: - annotations = ('\n\nAttributes: ~\n' + - textwrap.indent(annotations, ' ')) - i = doc.rfind('Parameters: ~') - if i == -1: - doc += annotations - else: - doc = doc[:i] + annotations + '\n\n' + doc[i:] - - # C Declaration: (debug only) - if INCLUDE_C_DECL: - doc += '\n\nC Declaration: ~\n>\n' - assert fn.c_decl is not None - doc += fn.c_decl - doc += '\n<' - - # Start of function documentations. e.g., - # nvim_cmd({*cmd}, {*opts}) *nvim_cmd()* - func_doc = fn.signature + '\n' - func_doc += textwrap.indent(clean_lines(doc), ' ' * indentation) - - # Verbatim handling. - func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) - - def process_helptags(func_doc: str) -> str: - lines: List[str] = func_doc.split('\n') - # skip ">lang ... <" regions - is_verbatim: bool = False - for i in range(len(lines)): - if re.search(' >([a-z])*$', lines[i]): - is_verbatim = True - elif is_verbatim and lines[i].strip() == '<': - is_verbatim = False - if not is_verbatim: - lines[i] = align_tags(lines[i]) - return "\n".join(lines) - - func_doc = process_helptags(func_doc) - - if (fn_name.startswith(config.fn_name_prefix) - and fn_name != "nvim_error_event"): - fns_txt[fn_name] = func_doc - - for fn_name, fn in fns.items(): - _handle_fn(fn_name, fn, fns_txt) - for fn_name, fn in deprecated_fns.items(): - _handle_fn(fn_name, fn, deprecated_fns_txt, deprecated=True) - - return ( - '\n\n'.join(list(fns_txt.values())), - '\n\n'.join(list(deprecated_fns_txt.values())), - ) - - -def delete_lines_below(filename, tokenstr): - """Deletes all lines below the line containing `tokenstr`, the line itself, - and one line above it. - """ - lines = open(filename).readlines() - i = 0 - found = False - for i, line in enumerate(lines, 1): - if tokenstr in line: - found = True - break - if not found: - raise RuntimeError(f'not found: "{tokenstr}"') - i = max(0, i - 2) - with open(filename, 'wt') as fp: - fp.writelines(lines[0:i]) - - -def extract_defgroups(base: str, dom: Document) -> Dict[SectionName, Docstring]: - '''Generate module-level (section) docs (@defgroup).''' - section_docs = {} - - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'group': - continue - - # Doxygen "@defgroup" directive. - groupname = get_text(find_first(compound, 'name')) - groupxml = os.path.join(base, '%s.xml' % - compound.getAttribute('refid')) - - group_parsed = minidom.parse(groupxml) - doc_list = [] - brief_desc = find_first(group_parsed, 'briefdescription') - if brief_desc: - for child in brief_desc.childNodes: - doc_list.append(fmt_node_as_vimhelp(child)) - - desc = find_first(group_parsed, 'detaileddescription') - if desc: - doc = fmt_node_as_vimhelp(desc) - - if doc: - doc_list.append(doc) - - # Can't use '.' in @defgroup, so convert to '--' - # "vim.json" => "vim-dot-json" - groupname = groupname.replace('-dot-', '.') - - section_docs[groupname] = "\n".join(doc_list) - - return section_docs - - -@dataclasses.dataclass -class Section: - """Represents a section. Includes section heading (defgroup) - and all the FunctionDoc that belongs to this section.""" - - name: str - '''Name of the section. Usually derived from basename of lua/c src file. - Example: "Autocmd".''' - - title: str - '''Formatted section config. see config.section_fmt(). - Example: "Autocmd Functions". ''' - - helptag: str - '''see config.helptag_fmt(). Example: *api-autocmd*''' - - @property - def id(self) -> str: - '''section id: Module/Section id matched against @defgroup. - e.g., "*api-autocmd*" => "api-autocmd" - ''' - return self.helptag.strip('*') - - doc: str = "" - '''Section heading docs extracted from @defgroup.''' - - # TODO: Do not carry rendered text, but handle FunctionDoc for better OOP - functions_text: Docstring | None = None - '''(Rendered) doc of all the functions that belong to this section.''' - - deprecated_functions_text: Docstring | None = None - '''(Rendered) doc of all the deprecated functions that belong to this - section.''' - - def __repr__(self): - return f"Section(title='{self.title}', helptag='{self.helptag}')" - - @classmethod - def make_from(cls, filename: str, config: Config, - section_docs: Dict[SectionName, str], - *, - functions_text: Docstring, - deprecated_functions_text: Docstring, - ): - # filename: e.g., 'autocmd.c' - # name: e.g. 'autocmd' - name = os.path.splitext(filename)[0].lower() - - # section name: e.g. "Autocmd" - sectname: SectionName - sectname = name.upper() if name == 'ui' else name.title() - sectname = config.section_name.get(filename, sectname) - - # Formatted (this is what's going to be written in the vimdoc) - # e.g., "Autocmd Functions" - title: str = config.section_fmt(sectname) - - # section tag: e.g., "*api-autocmd*" - section_tag: str = config.helptag_fmt(sectname) - - section = cls(name=sectname, title=title, helptag=section_tag, - functions_text=functions_text, - deprecated_functions_text=deprecated_functions_text, - ) - section.doc = section_docs.get(section.id) or '' - return section - - def render(self, add_header=True) -> str: - """Render as vimdoc.""" - doc = '' - - if add_header: - doc += SECTION_SEP - doc += '\n{}{}'.format( - self.title, - self.helptag.rjust(text_width - len(self.title)) - ) - - if self.doc: - doc += '\n\n' + self.doc - - if self.functions_text: - doc += '\n\n' + self.functions_text - - if INCLUDE_DEPRECATED and self.deprecated_functions_text: - doc += f'\n\n\nDeprecated {self.name} Functions: ~\n\n' - doc += self.deprecated_functions_text - - return doc - - def __bool__(self) -> bool: - """Whether this section has contents. Used for skipping empty ones.""" - return bool(self.doc or self.functions_text or - (INCLUDE_DEPRECATED and self.deprecated_functions_text)) - - -def main(doxygen_config, args): - """Generates: - - 1. Vim :help docs - 2. *.mpack files for use by API clients - - Doxygen is called and configured through stdin. - """ - for target in CONFIG: - if args.target is not None and target != args.target: - continue - - config: Config = CONFIG[target] - - mpack_file = os.path.join( - base_dir, 'runtime', 'doc', - config.filename.replace('.txt', '.mpack')) - if os.path.exists(mpack_file): - os.remove(mpack_file) - - output_dir = out_dir.format(target=target) - log.info("Generating documentation for %s in folder %s", - target, output_dir) - debug = args.log_level >= logging.DEBUG - p = subprocess.Popen( - ['doxygen', '-'], - stdin=subprocess.PIPE, - # silence warnings - # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found - stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL)) - p.communicate( - doxygen_config.format( - input=' '.join([f'"{file}"' for file in config.files]), - output=output_dir, - filter=filter_cmd, - file_patterns=config.file_patterns) - .encode('utf8') - ) - if p.returncode: - sys.exit(p.returncode) - - # Collects all functions as each module is processed. - fn_map_full: Dict[FunctionName, FunctionDoc] = {} - # key: filename (e.g. autocmd.c) - sections: Dict[str, Section] = {} - - base = os.path.join(output_dir, 'xml') - dom = minidom.parse(os.path.join(base, 'index.xml')) - - # Collect all @defgroups (section headings after the '===...' separator - section_docs: Dict[SectionName, Docstring] = extract_defgroups(base, dom) - - # Generate docs for all functions in the current module. - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'file': - continue - - filename = get_text(find_first(compound, 'name')) - if not ( - filename.endswith('.c') or - filename.endswith('.lua') - ): - continue - - xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) - - # Extract unformatted (*.mpack). - fn_map, _ = extract_from_xml( - xmlfile, target, width=9999, fmt_vimhelp=False) - - # Extract formatted (:help). - functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( - xmlfile, target) - - if not functions_text and not deprecated_text: - continue - - filename = os.path.basename(filename) - - section: Section = Section.make_from( - filename, config, section_docs, - functions_text=functions_text, - deprecated_functions_text=deprecated_text, - ) - - if section: # if not empty - sections[filename] = section - fn_map_full.update(fn_map) - else: - log.debug("Skipping empty section: %s", section) - - if len(sections) == 0: - fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') - if len(sections) > len(config.section_order): - raise RuntimeError( - '{}: found new modules {}; ' - 'update the "section_order" map'.format( - target, - set(sections).difference(config.section_order)) - ) - first_section_tag = sections[config.section_order[0]].helptag - - docs = '' - - for filename in config.section_order: - try: - section: Section = sections.pop(filename) - except KeyError: - msg(f'warning: empty docs, skipping (target={target}): {filename}') - msg(f' existing docs: {sections.keys()}') - continue - - add_sep_and_header = filename not in config.append_only - docs += section.render(add_header=add_sep_and_header) - docs += '\n\n\n' - - docs = docs.rstrip() + '\n\n' - docs += f' vim:tw=78:ts=8:sw={indentation}:sts={indentation}:et:ft=help:norl:\n' - - doc_file = os.path.join(base_dir, 'runtime', 'doc', config.filename) - - if os.path.exists(doc_file): - delete_lines_below(doc_file, first_section_tag) - with open(doc_file, 'ab') as fp: - fp.write(docs.encode('utf8')) - - fn_map_full_exported = collections.OrderedDict(sorted( - (name, fn_doc.export_mpack()) for (name, fn_doc) in fn_map_full.items() - )) - with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(fn_map_full_exported, use_bin_type=True)) # type: ignore - - if not args.keep_tmpfiles: - shutil.rmtree(output_dir) - - msg_report() - - -def filter_source(filename, keep_tmpfiles): - output_dir = out_dir.format(target='lua2dox') - name, extension = os.path.splitext(filename) - if extension == '.lua': - args = [str(nvim), '-l', lua2dox, filename] + (['--outdir', output_dir] if keep_tmpfiles else []) - p = subprocess.run(args, stdout=subprocess.PIPE) - op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8')) - print(op) - else: - """Filters the source to fix macros that confuse Doxygen.""" - with open(filename, 'rt') as fp: - print(re.sub(r'^(ArrayOf|DictionaryOf|Dict)(\(.*?\))', - lambda m: m.group(1)+'_'.join( - re.split(r'[^\w]+', m.group(2))), - fp.read(), flags=re.M)) - - -def parse_args(): - targets = ', '.join(CONFIG.keys()) - ap = argparse.ArgumentParser( - description="Generate helpdoc from source code") - ap.add_argument( - "--log-level", "-l", choices=LOG_LEVELS.keys(), - default=logging.getLevelName(logging.ERROR), help="Set log verbosity" - ) - ap.add_argument('source_filter', nargs='*', - help="Filter source file(s)") - ap.add_argument('-k', '--keep-tmpfiles', action='store_true', - help="Keep temporary files (tmp-xx-doc/ directories, including tmp-lua2dox-doc/ for lua2dox.lua quasi-C output)") - ap.add_argument('-t', '--target', - help=f'One of ({targets}), defaults to "all"') - return ap.parse_args() - - -Doxyfile = textwrap.dedent(''' - OUTPUT_DIRECTORY = {output} - INPUT = {input} - INPUT_ENCODING = UTF-8 - FILE_PATTERNS = {file_patterns} - RECURSIVE = YES - INPUT_FILTER = "{filter}" - EXCLUDE = - EXCLUDE_SYMLINKS = NO - EXCLUDE_PATTERNS = */private/* */health.lua */_*.lua - EXCLUDE_SYMBOLS = - EXTENSION_MAPPING = lua=C - EXTRACT_PRIVATE = NO - - GENERATE_HTML = NO - GENERATE_DOCSET = NO - GENERATE_HTMLHELP = NO - GENERATE_QHP = NO - GENERATE_TREEVIEW = NO - GENERATE_LATEX = NO - GENERATE_RTF = NO - GENERATE_MAN = NO - GENERATE_DOCBOOK = NO - GENERATE_AUTOGEN_DEF = NO - - GENERATE_XML = YES - XML_OUTPUT = xml - XML_PROGRAMLISTING = NO - - ENABLE_PREPROCESSING = YES - MACRO_EXPANSION = YES - EXPAND_ONLY_PREDEF = NO - MARKDOWN_SUPPORT = YES -''') - -if __name__ == "__main__": - args = parse_args() - print("Setting log level to %s" % args.log_level) - args.log_level = LOG_LEVELS[args.log_level] - log.setLevel(args.log_level) - log.addHandler(logging.StreamHandler()) - - # When invoked as a filter, args won't be passed, so use an env var. - if args.keep_tmpfiles: - os.environ['NVIM_KEEP_TMPFILES'] = '1' - keep_tmpfiles = ('NVIM_KEEP_TMPFILES' in os.environ) - - if len(args.source_filter) > 0: - filter_source(args.source_filter[0], keep_tmpfiles) - else: - main(Doxyfile, args) - -# vim: set ft=python ts=4 sw=4 tw=79 et : diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua deleted file mode 100644 index 0b3daa59b2..0000000000 --- a/scripts/lua2dox.lua +++ /dev/null @@ -1,544 +0,0 @@ ------------------------------------------------------------------------------ --- Copyright (C) 2012 by Simon Dales -- --- simon@purrsoft.co.uk -- --- -- --- This program is free software; you can redistribute it and/or modify -- --- it under the terms of the GNU General Public License as published by -- --- the Free Software Foundation; either version 2 of the License, or -- --- (at your option) any later version. -- --- -- --- This program is distributed in the hope that it will be useful, -- --- but WITHOUT ANY WARRANTY; without even the implied warranty of -- --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- --- GNU General Public License for more details. -- --- -- --- You should have received a copy of the GNU General Public License -- --- along with this program; if not, write to the -- --- Free Software Foundation, Inc., -- --- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -- ------------------------------------------------------------------------------ - ---[[! -Lua-to-Doxygen converter - -Partially from lua2dox -http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm - -RUNNING -------- - -This script "lua2dox.lua" gets called by "gen_vimdoc.py". - -DEBUGGING/DEVELOPING ---------------------- - -1. To debug, run gen_vimdoc.py with --keep-tmpfiles: - python3 scripts/gen_vimdoc.py -t treesitter --keep-tmpfiles -2. The filtered result will be written to ./tmp-lua2dox-doc/….lua.c - -Doxygen must be on your system. You can experiment like so: - -- Run "doxygen -g" to create a default Doxyfile. -- Then alter it to let it recognise lua. Add the following line: - FILE_PATTERNS = *.lua -- Then run "doxygen". - -The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. -It only has to be good enough for doxygen to see it as legal. - -One limitation is that each line is treated separately (except for long comments). -The implication is that class and function declarations must be on the same line. - -There is hack that will insert the "missing" close paren. -The effect is that you will get the function documented, but not with the parameter list you might expect. -]] - -local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' } - -local luacats_parser = require('src/nvim/generators/luacats_grammar') - -local debug_outfile = nil --- @type string? -local debug_output = {} - ---- write to stdout ---- @param str? string -local function write(str) - if not str then - return - end - - io.write(str) - if debug_outfile then - table.insert(debug_output, str) - end -end - ---- write to stdout ---- @param str? string -local function writeln(str) - write(str) - write('\n') -end - ---- an input file buffer ---- @class StreamRead ---- @field currentLine string? ---- @field contentsLen integer ---- @field currentLineNo integer ---- @field filecontents string[] -local StreamRead = {} - ---- @return StreamRead ---- @param filename string -function StreamRead.new(filename) - assert(filename, ('invalid file: %s'):format(filename)) - -- get lines from file - -- syphon lines to our table - local filecontents = {} --- @type string[] - for line in io.lines(filename) do - filecontents[#filecontents + 1] = line - end - - return setmetatable({ - filecontents = filecontents, - contentsLen = #filecontents, - currentLineNo = 1, - }, { __index = StreamRead }) -end - --- get a line -function StreamRead:getLine() - if self.currentLine then - self.currentLine = nil - return self.currentLine - end - - -- get line - if self.currentLineNo <= self.contentsLen then - local line = self.filecontents[self.currentLineNo] - self.currentLineNo = self.currentLineNo + 1 - return line - end - - return '' -end - --- save line fragment ---- @param line_fragment string -function StreamRead:ungetLine(line_fragment) - self.currentLine = line_fragment -end - --- is it eof? -function StreamRead:eof() - return not self.currentLine and self.currentLineNo > self.contentsLen -end - --- input filter ---- @class Lua2DoxFilter -local Lua2DoxFilter = { - generics = {}, --- @type table - block_ignore = false, --- @type boolean -} -setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter }) - -function Lua2DoxFilter:reset() - self.generics = {} - self.block_ignore = false -end - ---- trim comment off end of string ---- ---- @param line string ---- @return string, string? -local function removeCommentFromLine(line) - local pos_comment = line:find('%-%-') - if not pos_comment then - return line - end - return line:sub(1, pos_comment - 1), line:sub(pos_comment) -end - ---- @param parsed luacats.Return ---- @return string -local function get_return_type(parsed) - local elems = {} --- @type string[] - for _, v in ipairs(parsed) do - local e = v.type --- @type string - if v.name then - e = e .. ' ' .. v.name --- @type string - end - elems[#elems + 1] = e - end - return '(' .. table.concat(elems, ', ') .. ')' -end - ---- @param name string ---- @return string -local function process_name(name, optional) - if optional then - name = name:sub(1, -2) --- @type string - end - return name -end - ---- @param ty string ---- @param generics table ---- @return string -local function process_type(ty, generics, optional) - -- replace generic types - for k, v in pairs(generics) do - ty = ty:gsub(k, v) --- @type string - end - - -- strip parens - ty = ty:gsub('^%((.*)%)$', '%1') - - if optional and not ty:find('nil') then - ty = ty .. '?' - end - - -- remove whitespace in unions - ty = ty:gsub('%s*|%s*', '|') - - -- replace '|nil' with '?' - ty = ty:gsub('|nil', '?') - ty = ty:gsub('nil|(.*)', '%1?') - - return '(`' .. ty .. '`)' -end - ---- @param parsed luacats.Param ---- @param generics table ---- @return string -local function process_param(parsed, generics) - local name, ty = parsed.name, parsed.type - local optional = vim.endswith(name, '?') - - return table.concat({ - '/// @param', - process_name(name, optional), - process_type(ty, generics, optional), - parsed.desc, - }, ' ') -end - ---- @param parsed luacats.Return ---- @param generics table ---- @return string -local function process_return(parsed, generics) - local ty, name --- @type string, string - if #parsed == 1 then - ty, name = parsed[1].type, parsed[1].name or '' - else - ty, name = get_return_type(parsed), '' - end - - local optional = vim.endswith(name, '?') - - return table.concat({ - '/// @return', - process_type(ty, generics, optional), - process_name(name, optional), - parsed.desc, - }, ' ') -end - ---- Processes "@…" directives in a docstring line. ---- ---- @param line string ---- @return string? -function Lua2DoxFilter:process_magic(line) - line = line:gsub('^%s+@', '@') - line = line:gsub('@package', '@private') - line = line:gsub('@nodoc', '@private') - - if self.block_ignore then - return '// gg:" ' .. line .. '"' - end - - if not vim.startswith(line, '@') then -- it's a magic comment - return '/// ' .. line - end - - local magic_split = vim.split(line, ' ', { plain = true }) - local directive = magic_split[1] - - if - vim.list_contains({ - '@cast', - '@diagnostic', - '@overload', - '@meta', - '@type', - }, directive) - then - -- Ignore LSP directives - return '// gg:"' .. line .. '"' - elseif directive == '@defgroup' or directive == '@addtogroup' then - -- Can't use '.' in defgroup, so convert to '--' - return '/// ' .. line:gsub('%.', '-dot-') - end - - if directive == '@alias' then - -- this contiguous block should be all ignored. - self.block_ignore = true - return '// gg:"' .. line .. '"' - end - - -- preprocess line before parsing - if directive == '@param' or directive == '@return' then - for _, type in ipairs(TYPES) do - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') - - line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1') - line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1') - line = line:gsub('^@return%s+.*%((' .. type .. '%?)%)', '@return %1') - end - end - - local parsed = luacats_parser:match(line) - - if not parsed then - return '/// ' .. line - end - - local kind = parsed.kind - - if kind == 'generic' then - self.generics[parsed.name] = parsed.type or 'any' - return - elseif kind == 'param' then - return process_param(parsed --[[@as luacats.Param]], self.generics) - elseif kind == 'return' then - return process_return(parsed --[[@as luacats.Return]], self.generics) - end - - error(string.format('unhandled parsed line %q: %s', line, parsed)) -end - ---- @param line string ---- @param in_stream StreamRead ---- @return string -function Lua2DoxFilter:process_block_comment(line, in_stream) - local comment_parts = {} --- @type string[] - local done --- @type boolean? - - while not done and not in_stream:eof() do - local thisComment --- @type string? - local closeSquare = line:find(']]') - if not closeSquare then -- need to look on another line - thisComment = line .. '\n' - line = in_stream:getLine() - else - thisComment = line:sub(1, closeSquare - 1) - done = true - - -- unget the tail of the line - -- in most cases it's empty. This may make us less efficient but - -- easier to program - in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2))) - end - comment_parts[#comment_parts + 1] = thisComment - end - - local comment = table.concat(comment_parts) - - if comment:sub(1, 1) == '@' then -- it's a long magic comment - return '/*' .. comment .. '*/ ' - end - - -- discard - return '/* zz:' .. comment .. '*/ ' -end - ---- @param line string ---- @return string -function Lua2DoxFilter:process_function_header(line) - local pos_fn = assert(line:find('function')) - -- we've got a function - local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8))) - - if fn:sub(1, 1) == '(' then - -- it's an anonymous function - return '// ZZ: ' .. line - end - -- fn has a name, so is interesting - - -- want to fix for iffy declarations - if fn:find('[%({]') then - -- we might have a missing close paren - if not fn:find('%)') then - fn = fn .. ' ___MissingCloseParenHere___)' - end - end - - -- Big hax - if fn:find(':') then - fn = fn:gsub(':', '.', 1) - - local paren_start = fn:find('(', 1, true) - local paren_finish = fn:find(')', 1, true) - - -- Nothing in between the parens - local comma --- @type string - if paren_finish == paren_start + 1 then - comma = '' - else - comma = ', ' - end - - fn = fn:sub(1, paren_start) .. 'self' .. comma .. fn:sub(paren_start + 1) - end - - if line:match('local') then - -- Special: tell gen_vimdoc.py this is a local function. - return 'local_function ' .. fn .. '{}' - end - - -- add vanilla function - return 'function ' .. fn .. '{}' -end - ---- @param line string ---- @param in_stream StreamRead ---- @return string? -function Lua2DoxFilter:process_line(line, in_stream) - local line_raw = line - line = vim.trim(line) - - if vim.startswith(line, '---') then - return Lua2DoxFilter:process_magic(line:sub(4)) - end - - if vim.startswith(line, '--' .. '[[') then -- it's a long comment - return Lua2DoxFilter:process_block_comment(line:sub(5), in_stream) - end - - -- Hax... I'm sorry - -- M.fun = vim.memoize(function(...) - -- -> - -- function M.fun(...) - line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') - - if line:find('^function') or line:find('^local%s+function') then - return Lua2DoxFilter:process_function_header(line) - end - - if not line:match('^local') then - local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=') - if v and v:match('%.') then - -- Special: this lets gen_vimdoc.py handle tables. - return 'table ' .. v .. '() {}' - end - end - - if #line > 0 then -- we don't know what this line means, so just comment it out - return '// zz: ' .. line - end - - return '' -end - --- Processes the file and writes filtered output to stdout. ----@param filename string -function Lua2DoxFilter:filter(filename) - local in_stream = StreamRead.new(filename) - - local last_was_magic = false - - while not in_stream:eof() do - local line = in_stream:getLine() - - local out_line = self:process_line(line, in_stream) - - if not vim.startswith(vim.trim(line), '---') then - self:reset() - end - - if out_line then - -- Ensure all magic blocks associate with some object to prevent doxygen - -- from getting confused. - if vim.startswith(out_line, '///') then - last_was_magic = true - else - if last_was_magic and out_line:match('^// zz: [^-]+') then - writeln('local_function _ignore() {}') - end - last_was_magic = false - end - writeln(out_line) - end - end -end - ---- @class TApp ---- @field timestamp string|osdate ---- @field name string ---- @field version string ---- @field copyright string ---- this application -local TApp = { - timestamp = os.date('%c %Z', os.time()), - name = 'Lua2DoX', - version = '0.2 20130128', - copyright = 'Copyright (c) Simon Dales 2012-13', -} - -setmetatable(TApp, { __index = TApp }) - -function TApp:getRunStamp() - return self.name .. ' (' .. self.version .. ') ' .. self.timestamp -end - -function TApp:getVersion() - return self.name .. ' (' .. self.version .. ') ' -end - ---main - -if arg[1] == '--help' then - writeln(TApp:getVersion()) - writeln(TApp.copyright) - writeln([[ - run as: - nvim -l scripts/lua2dox.lua - -------------- - Param: - : interprets filename - --version : show version/copyright info - --help : this help text]]) -elseif arg[1] == '--version' then - writeln(TApp:getVersion()) - writeln(TApp.copyright) -else -- It's a filter. - local filename = arg[1] - - if arg[2] == '--outdir' then - local outdir = arg[3] - if - type(outdir) ~= 'string' - or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) - then - error(('invalid --outdir: "%s"'):format(tostring(outdir))) - end - vim.fn.mkdir(outdir, 'p') - debug_outfile = string.format('%s/%s.c', outdir, vim.fs.basename(filename)) - end - - Lua2DoxFilter:filter(filename) - - -- output the tail - writeln('// #######################') - writeln('// app run:' .. TApp:getRunStamp()) - writeln('// #######################') - writeln() - - if debug_outfile then - local f = assert(io.open(debug_outfile, 'w')) - f:write(table.concat(debug_output)) - f:close() - end -end diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua new file mode 100644 index 0000000000..ee0f9d8e87 --- /dev/null +++ b/scripts/luacats_grammar.lua @@ -0,0 +1,218 @@ +--[[! +LPEG grammar for LuaCATS +]] + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local Ct, Cg = lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local nl = P('\r\n') + P('\n') +local ws = rep1(S(' \t') + nl) +local fill = opt(ws) + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local num = R('09') +local ident = letter * rep(letter + num + S '-.') +local string_single = P "'" * rep(any - P "'") * P "'" +local string_double = P '"' * rep(any - P '"') * P '"' + +local literal = (string_single + string_double + (opt(P '-') * num) + P 'false' + P 'true') + +local lname = (ident + P '...') * opt(P '?') + +--- @param x string +local function Pf(x) + return fill * P(x) * fill +end + +--- @param x string +local function Sf(x) + return fill * S(x) * fill +end + +--- @param x vim.lpeg.Pattern +local function comma(x) + return x * rep(Pf ',' * x) +end + +--- @param x vim.lpeg.Pattern +local function parenOpt(x) + return (Pf('(') * x * fill * P(')')) + x +end + +--- @type table +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +local desc_delim = Sf '#:' + ws + +--- @class nvim.luacats.Param +--- @field kind 'param' +--- @field name string +--- @field type string +--- @field desc? string + +--- @class nvim.luacats.Return +--- @field kind 'return' +--- @field [integer] { type: string, name?: string} +--- @field desc? string + +--- @class nvim.luacats.Generic +--- @field kind 'generic' +--- @field name string +--- @field type? string + +--- @class nvim.luacats.Class +--- @field kind 'class' +--- @field name string +--- @field parent? string + +--- @class nvim.luacats.Field +--- @field kind 'field' +--- @field name string +--- @field type string +--- @field desc? string +--- @field access? 'private'|'protected'|'package' + +--- @class nvim.luacats.Note +--- @field desc? string + +--- @alias nvim.luacats.grammar.result +--- | nvim.luacats.Param +--- | nvim.luacats.Return +--- | nvim.luacats.Generic +--- | nvim.luacats.Class +--- | nvim.luacats.Field +--- | nvim.luacats.Note + +--- @class nvim.luacats.grammar +--- @field match fun(self, input: string): nvim.luacats.grammar.result? + +local grammar = P { + rep1(P('@') * (v.ats + v.ext_ats)), + + ats = v.at_param + + v.at_return + + v.at_type + + v.at_cast + + v.at_generic + + v.at_class + + v.at_field + + v.at_access + + v.at_deprecated + + v.at_alias + + v.at_enum + + v.at_see + + v.at_diagnostic + + v.at_overload + + v.at_meta, + + ext_ats = v.ext_at_note + v.ext_at_since + v.ext_at_nodoc + v.ext_at_brief, + + at_param = Ct( + Cg(P('param'), 'kind') + * ws + * Cg(lname, 'name') + * ws + * parenOpt(Cg(v.ltype, 'type')) + * opt(desc_delim * Cg(rep(any), 'desc')) + ), + + at_return = Ct( + Cg(P('return'), 'kind') + * ws + * parenOpt(comma(Ct(Cg(v.ltype, 'type') * opt(ws * Cg(ident, 'name'))))) + * opt(desc_delim * Cg(rep(any), 'desc')) + ), + + at_type = Ct( + Cg(P('type'), 'kind') + * ws + * parenOpt(comma(Ct(Cg(v.ltype, 'type')))) + * opt(desc_delim * Cg(rep(any), 'desc')) + ), + + at_cast = Ct( + Cg(P('cast'), 'kind') * ws * Cg(lname, 'name') * ws * opt(Sf('+-')) * Cg(v.ltype, 'type') + ), + + at_generic = Ct( + Cg(P('generic'), 'kind') * ws * Cg(ident, 'name') * opt(Pf ':' * Cg(v.ltype, 'type')) + ), + + at_class = Ct( + Cg(P('class'), 'kind') + * ws + * opt(P('(exact)') * ws) + * Cg(lname, 'name') + * opt(Pf(':') * Cg(lname, 'parent')) + ), + + at_field = Ct( + Cg(P('field'), 'kind') + * ws + * opt(Cg(Pf('private') + Pf('package') + Pf('protected'), 'access')) + * Cg(lname, 'name') + * ws + * Cg(v.ltype, 'type') + * opt(desc_delim * Cg(rep(any), 'desc')) + ), + + at_access = Ct(Cg(P('private') + P('protected') + P('package'), 'kind')), + + at_deprecated = Ct(Cg(P('deprecated'), 'kind')), + + -- Types may be provided on subsequent lines + at_alias = Ct(Cg(P('alias'), 'kind') * ws * Cg(lname, 'name') * opt(ws * Cg(v.ltype, 'type'))), + + at_enum = Ct(Cg(P('enum'), 'kind') * ws * Cg(lname, 'name')), + + at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + at_diagnostic = Ct(Cg(P('diagnostic'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + at_overload = Ct(Cg(P('overload'), 'kind') * ws * Cg(v.ltype, 'type')), + at_meta = Ct(Cg(P('meta'), 'kind')), + + --- Custom extensions + ext_at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), + + -- TODO only consume 1 line + ext_at_since = Ct(Cg(P('since'), 'kind') * ws * Cg(rep(any), 'desc')), + + ext_at_nodoc = Ct(Cg(P('nodoc'), 'kind')), + ext_at_brief = Ct(Cg(P('brief'), 'kind') * opt(ws * Cg(rep(any), 'desc'))), + + ltype = v.ty_union + Pf '(' * v.ty_union * fill * P ')', + + ty_union = v.ty_opt * rep(Pf '|' * v.ty_opt), + ty = v.ty_fun + ident + v.ty_table + literal, + ty_param = Pf '<' * comma(v.ltype) * fill * P '>', + ty_opt = v.ty * opt(v.ty_param) * opt(P '[]') * opt(P '?'), + + table_key = (Pf '[' * literal * Pf ']') + lname, + table_elem = v.table_key * Pf ':' * v.ltype, + ty_table = Pf '{' * comma(v.table_elem) * Pf '}', + + fun_param = lname * opt(Pf ':' * v.ltype), + ty_fun = Pf 'fun(' * rep(comma(v.fun_param)) * fill * P ')' * opt(Pf ':' * comma(v.ltype)), +} + +return grammar --[[@as nvim.luacats.grammar]] diff --git a/scripts/luacats_parser.lua b/scripts/luacats_parser.lua new file mode 100644 index 0000000000..520272d1dc --- /dev/null +++ b/scripts/luacats_parser.lua @@ -0,0 +1,521 @@ +local luacats_grammar = require('scripts.luacats_grammar') + +--- @class nvim.luacats.parser.param +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.note +--- @field desc string + +--- @class nvim.luacats.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.luacats.parser.alias +--- @field kind 'alias' +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.fun +--- @field name string +--- @field params nvim.luacats.parser.param[] +--- @field returns nvim.luacats.parser.return[] +--- @field desc string +--- @field access? 'private'|'package'|'protected' +--- @field class? string +--- @field module? string +--- @field modvar? string +--- @field classvar? string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field generics? table +--- @field table? true +--- @field notes? nvim.luacats.parser.note[] +--- @field see? nvim.luacats.parser.note[] + +--- @class nvim.luacats.parser.field +--- @field name string +--- @field type string +--- @field desc string +--- @field access? 'private'|'package'|'protected' + +--- @class nvim.luacats.parser.class +--- @field kind 'class' +--- @field name string +--- @field desc string +--- @field fields nvim.luacats.parser.field[] +--- @field notes? string[] + +--- @class nvim.luacats.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.luacats.parser.obj +--- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.luacats.parser.obj +--- | nvim.luacats.parser.class +--- | nvim.luacats.parser.fun +--- | nvim.luacats.parser.brief + +-- Remove this when we document classes properly +--- Some doc lines have the form: +--- param name some.complex.type (table) description +--- if so then transform the line to remove the complex type: +--- param name (table) description +--- @param line string +local function use_type_alt(line) + for _, type in ipairs({ 'table', 'function' }) do + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') + + line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1') + line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1') + line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1') + end + return line +end + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.luacats.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.luacats.parser.State +local function process_doc_line(line, state) + line = line:sub(4):gsub('^%s+@', '@') + line = use_type_alt(line) + + local parsed = luacats_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + local kind = parsed.kind + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'class' then + --- @cast parsed nvim.luacats.Class + state.cur_obj = { + kind = 'class', + name = parsed.name, + parent = parsed.parent, + desc = '', + fields = {}, + } + elseif kind == 'field' then + --- @cast parsed nvim.luacats.Field + if not parsed.access then + parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil + if parsed.desc then + parsed.desc = vim.trim(parsed.desc) + end + table.insert(cur_obj.fields, parsed) + end + state.doc_lines = nil + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + if vim.endswith(parsed.name, '?') then + parsed.name = parsed.name:sub(1, -2) + parsed.type = parsed.type .. '?' + end + state.last_doc_item = { + name = parsed.name, + type = parsed.type, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = cur_obj.returns or {} + for _, t in ipairs(parsed) do + table.insert(cur_obj.returns, { + name = t.name, + type = t.type, + desc = parsed.desc, + }) + end + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[#cur_obj.returns] + elseif kind == 'private' then + cur_obj.access = 'private' + elseif kind == 'package' then + cur_obj.access = 'package' + elseif kind == 'protected' then + cur_obj.access = 'protected' + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + elseif kind == 'type' then + cur_obj.desc = parsed.desc + parsed.desc = nil + parsed.kind = nil + cur_obj.type = parsed + elseif kind == 'alias' then + state.cur_obj = { + kind = 'alias', + desc = parsed.desc, + } + elseif kind == 'enum' then + -- TODO + state.doc_lines = nil + elseif + vim.tbl_contains({ + 'diagnostic', + 'cast', + 'overload', + 'meta', + }, kind) + then + -- Ignore + return + elseif kind == 'generic' then + cur_obj.generics = cur_obj.generics or {} + cur_obj.generics[parsed.name] = parsed.type or 'any' + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param fun nvim.luacats.parser.fun +--- @return nvim.luacats.parser.field +local function fun2field(fun) + local parts = { 'fun(' } + for _, p in ipairs(fun.params or {}) do + parts[#parts + 1] = string.format('%s: %s', p.name, p.type) + end + parts[#parts + 1] = ')' + if fun.returns then + parts[#parts + 1] = ': ' + local tys = {} --- @type string[] + for _, p in ipairs(fun.returns) do + tys[#tys + 1] = p.type + end + parts[#parts + 1] = table.concat(tys, ', ') + end + + return { + name = fun.name, + type = table.concat(parts, ''), + access = fun.access, + desc = fun.desc, + } +end + +--- Function to normalize known form for declaring functions and normalize into a more standard +--- form. +--- @param line string +--- @return string +local function filter_decl(line) + -- M.fun = vim._memoize(function(...) + -- -> + -- function M.fun(...) + line = line:gsub('^local (.+) = .*_memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)') + line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') + return line +end + +--- @param line string +--- @param state nvim.luacats.parser.State +--- @param classes table +--- @param classvars table +--- @param has_indent boolean +local function process_lua_line(line, state, classes, classvars, has_indent) + line = filter_decl(line) + + if state.cur_obj and state.cur_obj.kind == 'class' then + local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=') + if nm then + classvars[nm] = state.cur_obj.name + end + return + end + + do + local parent_tbl, sep, fun_or_meth_nm = + line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(') + if parent_tbl then + -- Have a decl. Ensure cur_obj + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + -- Match `Class:foo` methods for defined classes + local class = classvars[parent_tbl] + if class then + --- @cast cur_obj nvim.luacats.parser.fun + cur_obj.name = fun_or_meth_nm + cur_obj.class = class + cur_obj.classvar = parent_tbl + -- Add self param to methods + if sep == ':' then + cur_obj.params = cur_obj.params or {} + table.insert(cur_obj.params, 1, { + name = 'self', + type = class, + }) + end + + -- Add method as the field to the class + table.insert(classes[class].fields, fun2field(cur_obj)) + return + end + + -- Match `M.foo` + if cur_obj and parent_tbl == cur_obj.modvar then + cur_obj.name = fun_or_meth_nm + return + end + end + end + + do + -- Handle: `function A.B.C.foo(...)` + local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(') + if fn_nm then + state.cur_obj = state.cur_obj or {} + state.cur_obj.name = fn_nm + return + end + end + + do + -- Handle: `M.foo = {...}` where `M` is the modvar + local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=') + if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + do + -- Handle: `foo = {...}` + local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=') + if tbl_nm and not has_indent then + state.cur_obj = state.cur_obj or {} + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + do + -- Handle: `vim.foo = {...}` + local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=') + if state.cur_obj and tbl_nm and not has_indent then + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + if state.cur_obj then + if line:find('^%s*%-%- luacheck:') then + state.cur_obj = nil + elseif line:find('^%s*local%s+') then + state.cur_obj = nil + elseif line:find('^%s*return%s+') then + state.cur_obj = nil + elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then + state.cur_obj = nil + end + end +end + +--- Determine the table name used to export functions of a module +--- Usually this is `M`. +--- @param filename string +--- @return string? +local function determine_modvar(filename) + local modvar --- @type string? + for line in io.lines(filename) do + do + --- @type string? + local m = line:match('^return%s+([a-zA-Z_]+)') + if m then + modvar = m + end + end + do + --- @type string? + local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),') + if m then + modvar = m + end + end + end + return modvar +end + +--- @param obj nvim.luacats.parser.obj +--- @param funs nvim.luacats.parser.fun[] +--- @param classes table +--- @param briefs string[] +--- @param uncommitted nvim.luacats.parser.obj[] +local function commit_obj(obj, classes, funs, briefs, uncommitted) + local commit = false + if obj.kind == 'class' then + --- @cast obj nvim.luacats.parser.class + if not classes[obj.name] then + classes[obj.name] = obj + commit = true + end + elseif obj.kind == 'alias' then + -- Just pretend + commit = true + elseif obj.kind == 'brief' then + --- @cast obj nvim.luacats.parser.brief` + briefs[#briefs + 1] = obj.desc + commit = true + else + --- @cast obj nvim.luacats.parser.fun` + if obj.name then + funs[#funs + 1] = obj + commit = true + end + end + if not commit then + table.insert(uncommitted, obj) + end + return commit +end + +--- @param filename string +--- @param uncommitted nvim.luacats.parser.obj[] +-- luacheck: no unused +local function dump_uncommitted(filename, uncommitted) + local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt' + if #uncommitted > 0 then + print(string.format('Could not commit %d objects in %s', #uncommitted, filename)) + vim.fn.mkdir(assert(vim.fs.dirname(out_path)), 'p') + local f = assert(io.open(out_path, 'w')) + for i, x in ipairs(uncommitted) do + f:write(i) + f:write(': ') + f:write(vim.inspect(x)) + f:write('\n') + end + f:close() + else + vim.fn.delete(out_path) + end +end + +local M = {} + +--- @param filename string +--- @return table classes +--- @return nvim.luacats.parser.fun[] funs +--- @return string[] briefs +--- @return nvim.luacats.parser.obj[] +function M.parse(filename) + local funs = {} --- @type nvim.luacats.parser.fun[] + local classes = {} --- @type table + local briefs = {} --- @type string[] + + local mod_return = determine_modvar(filename) + + --- @type string + local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename + module = module:gsub('/', '.') + + local classvars = {} --- @type table + + local state = {} --- @type nvim.luacats.parser.State + + -- Keep track of any partial objects we don't commit + local uncommitted = {} --- @type nvim.luacats.parser.obj[] + + for line in io.lines(filename) do + local has_indent = line:match('^%s+') ~= nil + line = vim.trim(line) + if vim.startswith(line, '---') then + process_doc_line(line, state) + else + add_doc_lines_to_obj(state) + + if state.cur_obj then + state.cur_obj.modvar = mod_return + state.cur_obj.module = module + end + + process_lua_line(line, state, classes, classvars, has_indent) + + -- Commit the object + local cur_obj = state.cur_obj + if cur_obj then + if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then + --- @diagnostic disable-next-line:inject-field + cur_obj.line = line + end + end + + state = {} + end + end + + -- dump_uncommitted(filename, uncommitted) + + return classes, funs, briefs, uncommitted +end + +return M diff --git a/scripts/text_utils.lua b/scripts/text_utils.lua new file mode 100644 index 0000000000..5167ec42f2 --- /dev/null +++ b/scripts/text_utils.lua @@ -0,0 +1,239 @@ +local fmt = string.format + +--- @class nvim.text_utils.MDNode +--- @field [integer] nvim.text_utils.MDNode +--- @field type string +--- @field text? string + +local INDENTATION = 4 + +local M = {} + +local function contains(t, xs) + return vim.tbl_contains(xs, t) +end + +--- @param text string +--- @return nvim.text_utils.MDNode +local function parse_md(text) + local parser = vim.treesitter.languagetree.new(text, 'markdown', { + injections = { markdown = '' }, + }) + + local root = parser:parse(true)[1]:root() + + local EXCLUDE_TEXT_TYPE = { + list = true, + list_item = true, + section = true, + document = true, + fenced_code_block = true, + fenced_code_block_delimiter = true, + } + + --- @param node TSNode + --- @return nvim.text_utils.MDNode? + local function extract(node) + local ntype = node:type() + + if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then + return + end + + --- @type table + local ret = { type = ntype } + + if not EXCLUDE_TEXT_TYPE[ntype] then + ret.text = vim.treesitter.get_node_text(node, text) + end + + for child, child_field in node:iter_children() do + local e = extract(child) + if child_field then + ret[child_field] = e + else + table.insert(ret, e) + end + end + + return ret + end + + return extract(root) or {} +end + +--- @param x string +--- @param start_indent integer +--- @param indent integer +--- @param text_width integer +--- @return string +function M.wrap(x, start_indent, indent, text_width) + local words = vim.split(vim.trim(x), '%s+') + local parts = { string.rep(' ', start_indent) } --- @type string[] + local count = indent + + for i, w in ipairs(words) do + if count > indent and count + #w > text_width - 1 then + parts[#parts + 1] = '\n' + parts[#parts + 1] = string.rep(' ', indent) + count = indent + elseif i ~= 1 then + parts[#parts + 1] = ' ' + count = count + 1 + end + count = count + #w + parts[#parts + 1] = w + end + + return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', '')) +end + +--- @param node nvim.text_utils.MDNode +--- @param start_indent integer +--- @param indent integer +--- @param text_width integer +--- @param level integer +--- @return string[] +local function render_md(node, start_indent, indent, text_width, level, is_list) + local parts = {} --- @type string[] + + -- For debugging + local add_tag = false + -- local add_tag = true + + if add_tag then + parts[#parts + 1] = '<' .. node.type .. '>' + end + + if node.type == 'paragraph' then + local text = assert(node.text) + text = text:gsub('(%s)%*(%w+)%*(%s)', '%1%2%3') + text = text:gsub('(%s)_(%w+)_(%s)', '%1%2%3') + text = text:gsub('\\|', '|') + text = text:gsub('\\%*', '*') + text = text:gsub('\\_', '_') + parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width) + parts[#parts + 1] = '\n' + elseif node.type == 'code_fence_content' then + local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n') + + local cindent = indent + INDENTATION + if level > 3 then + -- The tree-sitter markdown parser doesn't parse the code blocks indents + -- correctly in lists. Fudge it! + lines[1] = ' ' .. lines[1] -- ¯\_(ツ)_/¯ + cindent = indent - level + local _, initial_indent = lines[1]:find('^%s*') + initial_indent = initial_indent + cindent + if initial_indent < indent then + cindent = indent - INDENTATION + end + end + + for _, l in ipairs(lines) do + if #l > 0 then + parts[#parts + 1] = string.rep(' ', cindent) + parts[#parts + 1] = l + end + parts[#parts + 1] = '\n' + end + elseif node.type == 'fenced_code_block' then + parts[#parts + 1] = '>' + for _, child in ipairs(node) do + if child.type == 'info_string' then + parts[#parts + 1] = child.text + break + end + end + parts[#parts + 1] = '\n' + for _, child in ipairs(node) do + if child.type ~= 'info_string' then + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1)) + end + end + parts[#parts + 1] = '<\n' + elseif node.type == 'html_block' then + local text = node.text:gsub('^
help', '')
+    text = text:gsub('
%s*$', '') + parts[#parts + 1] = text + elseif node.type == 'list_marker_dot' then + parts[#parts + 1] = node.text + elseif contains(node.type, { 'list_marker_minus', 'list_marker_star' }) then + parts[#parts + 1] = '• ' + elseif node.type == 'list_item' then + parts[#parts + 1] = string.rep(' ', indent) + local offset = node[1].type == 'list_marker_dot' and 3 or 2 + for i, child in ipairs(node) do + local sindent = i <= 2 and 0 or (indent + offset) + vim.list_extend( + parts, + render_md(child, sindent, indent + offset, text_width, level + 1, true) + ) + end + else + if node.text then + error(fmt('cannot render:\n%s', vim.inspect(node))) + end + for i, child in ipairs(node) do + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1, is_list)) + if node.type ~= 'list' and i ~= #node then + if (node[i + 1] or {}).type ~= 'list' then + parts[#parts + 1] = '\n' + end + end + end + end + + if add_tag then + parts[#parts + 1] = '' + end + + return parts +end + +--- @param text_width integer +local function align_tags(text_width) + --- @param line string + --- @return string + return function(line) + local tag_pat = '%s+(%*[^ ]+%*)%s*$' + local tags = {} + for m in line:gmatch(tag_pat) do + table.insert(tags, m) + end + + if #tags > 0 then + line = line:gsub(tag_pat, '') + local tags_str = ' ' .. table.concat(tags, ' ') + local pad = string.rep(' ', text_width - #line - #tags_str) + return line .. pad .. tags_str + end + + return line + end +end + +--- @param text string +--- @param start_indent integer +--- @param indent integer +--- @param is_list? boolean +--- @return string +function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list) + -- Add an extra newline so the parser can properly capture ending ``` + local parsed = parse_md(text .. '\n') + local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list) + + local lines = vim.split(table.concat(ret), '\n') + + lines = vim.tbl_map(align_tags(text_width), lines) + + local s = table.concat(lines, '\n') + + -- Reduce whitespace in code-blocks + s = s:gsub('\n+%s*>([a-z]+)\n?\n', ' >%1\n') + s = s:gsub('\n+%s*>\n?\n', ' >\n') + + return s +end + +return M -- cgit From 07b4b7524fc7faefb69e5c94d2512eb0807f0593 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Wed, 28 Feb 2024 10:38:25 +0100 Subject: vim-patch:e84d2d4432cd runtime(sh): Update ftplugin, fix vim/vim#14101 (vim/vim#14102) Add the 'b' flag to 'comments', so that the shebang line is not detected as comment. Fixes vim/vim#14101. https://github.com/vim/vim/commit/e84d2d4432cd6e43f2bb300d02abc90d551bcf4a Co-authored-by: dkearns --- scripts/vim-patch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 660b8b6bb0..45dd7f5fee 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -174,7 +174,7 @@ assign_commit_details() { vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}" vim_message="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:%B' "${vim_commit}" \ - | sed -Ee 's/(\W)(#[0-9]{1,})/\1vim\/vim\2/g')" + | sed -Ee 's/([^A-Za-z0-9])(#[0-9]{1,})/\1vim\/vim\2/g')" local vim_coauthor0 vim_coauthor0="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:Co-authored-by: %an <%ae>' "${vim_commit}")" # Extract co-authors from the commit message. -- cgit From de5cf09cf98e20d8d3296ad6933ff2741acf83f7 Mon Sep 17 00:00:00 2001 From: bfredl Date: Mon, 26 Feb 2024 18:00:46 +0100 Subject: refactor(metadata): generate all metadata in lua Then we can just load metadata in C as a single msgpack blob. Which also can be used directly as binarly data, instead of first unpacking all the functions and ui_events metadata to immediately pack it again, which was a bit of a silly walk (and one extra usecase of `msgpack_rpc_from_object` which will get yak shaved in the next PR) --- scripts/gen_eval_files.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index 895033d5af..83e55b3bc4 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -2,7 +2,7 @@ -- Generator for various vimdoc and Lua type files -local DEP_API_METADATA = 'build/api_metadata.mpack' +local DEP_API_METADATA = 'build/funcs_metadata.mpack' --- @class vim.api.metadata --- @field name string -- cgit From 2f85bbe61513d12c746641fed6ad07559bd95719 Mon Sep 17 00:00:00 2001 From: altermo <107814000+altermo@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:42:51 -0800 Subject: feat!: rewrite TOhtml in lua Co-authored-by: wookayin Co-authored-by: clason Co-authored-by: Lewis Russell --- scripts/gen_vimdoc.lua | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua index 290cd83fbc..8acae66f49 100755 --- a/scripts/gen_vimdoc.lua +++ b/scripts/gen_vimdoc.lua @@ -161,6 +161,7 @@ local config = { 'iter.lua', 'snippet.lua', 'text.lua', + 'tohtml.lua', }, files = { 'runtime/lua/vim/iter.lua', @@ -189,6 +190,7 @@ local config = { 'runtime/lua/vim/_meta/lpeg.lua', 'runtime/lua/vim/_meta/re.lua', 'runtime/lua/vim/_meta/spell.lua', + 'runtime/lua/tohtml.lua', }, fn_xform = function(fun) if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then @@ -232,6 +234,9 @@ local config = { then return 'VIM.' .. name:upper() end + if name == 'tohtml' then + return 'Lua module: tohtml' + end return 'Lua module: vim.' .. name end, helptag_fmt = function(name) @@ -239,6 +244,8 @@ local config = { return '*lua-vim*' elseif name == '_options' then return '*lua-vimscript*' + elseif name == 'tohtml' then + return '*tohtml*' end return '*vim.' .. name:lower() .. '*' end, -- cgit From 813dd36b72979dfd05479eb6402b9becc0faea29 Mon Sep 17 00:00:00 2001 From: Will Hopkins Date: Fri, 1 Mar 2024 14:59:32 -0800 Subject: fix(types): rename win_get_config return type to win_config Follow-up to #27397 --- scripts/gen_eval_files.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index 83e55b3bc4..f1bba5c0a2 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -40,7 +40,7 @@ local LUA_API_RETURN_OVERRIDES = { nvim_get_option_info = 'vim.api.keyset.get_option_info', nvim_get_option_info2 = 'vim.api.keyset.get_option_info', nvim_parse_cmd = 'vim.api.keyset.parse_cmd', - nvim_win_get_config = 'vim.api.keyset.float_config', + nvim_win_get_config = 'vim.api.keyset.win_config', } local LUA_META_HEADER = { -- cgit From a5fe8f59d98398d04bed8586cee73864bbcdde92 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 27 Feb 2024 15:20:32 +0000 Subject: docs: improve/add documentation of Lua types - Added `@inlinedoc` so single use Lua types can be inlined into the functions docs. E.g. ```lua --- @class myopts --- @inlinedoc --- --- Documentation for some field --- @field somefield integer --- @param opts myOpts function foo(opts) end ``` Will be rendered as ``` foo(opts) Parameters: - {opts} (table) Object with the fields: - somefield (integer) Documentation for some field ``` - Marked many classes with with `@nodoc` or `(private)`. We can eventually introduce these when we want to. --- scripts/gen_help_html.lua | 2 +- scripts/gen_vimdoc.lua | 246 ++++++++++++++++++++++++++++++++++++-------- scripts/luacats_grammar.lua | 176 +++++++++++++------------------ scripts/luacats_parser.lua | 66 +++++++----- scripts/text_utils.lua | 6 +- 5 files changed, 320 insertions(+), 176 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 5cda16bbe6..43040151eb 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -752,7 +752,7 @@ end --- --- @param fname string help file to parse --- @param parser_path string? path to non-default vimdoc.so ---- @return LanguageTree, integer (lang_tree, bufnr) +--- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr) local function parse_buf(fname, parser_path) local buf ---@type integer if type(fname) == 'string' then diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua index 8acae66f49..8e2ff8193c 100755 --- a/scripts/gen_vimdoc.lua +++ b/scripts/gen_vimdoc.lua @@ -269,6 +269,7 @@ local config = { filename = 'lsp.txt', section_order = { 'lsp.lua', + 'client.lua', 'buf.lua', 'diagnostic.lua', 'codelens.lua', @@ -362,15 +363,25 @@ local function replace_generics(ty, generics) return generics[ty] or ty end +--- @param name string +local function fmt_field_name(name) + local name0, opt = name:match('^([^?]*)(%??)$') + return fmt('{%s}%s', name0, opt) +end + --- @param ty string --- @param generics? table -local function render_type(ty, generics) +--- @param default? string +local function render_type(ty, generics, default) if generics then ty = replace_generics(ty, generics) end ty = ty:gsub('%s*|%s*nil', '?') ty = ty:gsub('nil%s*|%s*(.*)', '%1?') ty = ty:gsub('%s*|%s*', '|') + if default then + return fmt('(`%s`, default: %s)', ty, default) + end return fmt('(`%s`)', ty) end @@ -379,10 +390,101 @@ local function should_render_param(p) return not p.access and not contains(p.name, { '_', 'self' }) end +--- @param desc? string +--- @return string?, string? +local function get_default(desc) + if not desc then + return + end + + local default = desc:match('\n%s*%([dD]efault: ([^)]+)%)') + if default then + desc = desc:gsub('\n%s*%([dD]efault: [^)]+%)', '') + end + + return desc, default +end + +--- @param ty string +--- @param classes? table +--- @return nvim.luacats.parser.class? +local function get_class(ty, classes) + if not classes then + return + end + + local cty = ty:gsub('%s*|%s*nil', '?'):gsub('?$', ''):gsub('%[%]$', '') + + return classes[cty] +end + +--- @param obj nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.field +--- @param classes? table +local function inline_type(obj, classes) + local ty = obj.type + if not ty then + return + end + + local cls = get_class(ty, classes) + + if not cls or cls.nodoc then + return + end + + if not cls.inlinedoc then + -- Not inlining so just add a: "See |tag|." + local tag = fmt('|%s|', cls.name) + if obj.desc and obj.desc:find(tag) then + -- Tag already there + return + end + + -- TODO(lewis6991): Aim to remove this. Need this to prevent dead + -- references to types defined in runtime/lua/vim/lsp/_meta/protocol.lua + if not vim.startswith(cls.name, 'vim.') then + return + end + + obj.desc = obj.desc or '' + local period = (obj.desc == '' or vim.endswith(obj.desc, '.')) and '' or '.' + obj.desc = obj.desc .. fmt('%s See %s.', period, tag) + return + end + + local ty_isopt = (ty:match('%?$') or ty:match('%s*|%s*nil')) ~= nil + local ty_islist = (ty:match('%[%]$')) ~= nil + ty = ty_isopt and 'table?' or ty_islist and 'table[]' or 'table' + + local desc = obj.desc or '' + if cls.desc then + desc = desc .. cls.desc + elseif desc == '' then + if ty_islist then + desc = desc .. 'A list of objects with the following fields:' + else + desc = desc .. 'A table with the following fields:' + end + end + + local desc_append = {} + for _, f in ipairs(cls.fields) do + local fdesc, default = get_default(f.desc) + local fty = render_type(f.type, nil, default) + local fnm = fmt_field_name(f.name) + table.insert(desc_append, table.concat({ '-', fnm, fty, fdesc }, ' ')) + end + + desc = desc .. '\n' .. table.concat(desc_append, '\n') + obj.type = ty + obj.desc = desc +end + --- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[] --- @param generics? table +--- @param classes? table --- @param exclude_types? true -local function render_fields_or_params(xs, generics, exclude_types) +local function render_fields_or_params(xs, generics, classes, exclude_types) local ret = {} --- @type string[] xs = vim.tbl_filter(should_render_param, xs) @@ -398,15 +500,27 @@ local function render_fields_or_params(xs, generics, exclude_types) end for _, p in ipairs(xs) do - local nm, ty = p.name, p.type - local desc = p.desc - local pnm = fmt(' • %-' .. indent .. 's', '{' .. nm .. '}') + local pdesc, default = get_default(p.desc) + p.desc = pdesc + + inline_type(p, classes) + local nm, ty, desc = p.name, p.type, p.desc + + local fnm = p.kind == 'operator' and fmt('op(%s)', nm) or fmt_field_name(nm) + local pnm = fmt(' • %-' .. indent .. 's', fnm) + if ty then - local pty = render_type(ty, generics) + local pty = render_type(ty, generics, default) + if desc then - desc = fmt('%s %s', pty, desc) table.insert(ret, pnm) - table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + if #pty > TEXT_WIDTH - indent then + vim.list_extend(ret, { ' ', pty, '\n' }) + table.insert(ret, md_to_vimdoc(desc, 9 + indent, 9 + indent, TEXT_WIDTH, true)) + else + desc = fmt('%s %s', pty, desc) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + end else table.insert(ret, fmt('%s %s\n', pnm, pty)) end @@ -421,24 +535,46 @@ local function render_fields_or_params(xs, generics, exclude_types) return table.concat(ret) end --- --- @param class lua2vimdoc.class --- local function render_class(class) --- writeln(fmt('*%s*', class.name)) --- writeln() --- if #class.fields > 0 then --- writeln(' Fields: ~') --- render_fields_or_params(class.fields) --- end --- writeln() --- end - --- --- @param cls table --- local function render_classes(cls) --- --- @diagnostic disable-next-line:no-unknown --- for _, class in vim.spairs(cls) do --- render_class(class) --- end --- end +--- @param class nvim.luacats.parser.class +local function render_class(class) + if class.access or class.nodoc or class.inlinedoc then + return + end + + local ret = {} --- @type string[] + + table.insert(ret, fmt('*%s*\n', class.name)) + + if class.parent then + local txt = fmt('Extends: |%s|', class.parent) + table.insert(ret, md_to_vimdoc(txt, INDENTATION, INDENTATION, TEXT_WIDTH)) + end + + if class.desc then + table.insert(ret, md_to_vimdoc(class.desc, INDENTATION, INDENTATION, TEXT_WIDTH)) + end + + local fields_txt = render_fields_or_params(class.fields) + if not fields_txt:match('^%s*$') then + table.insert(ret, '\n Fields: ~\n') + table.insert(ret, fields_txt) + end + table.insert(ret, '\n') + + return table.concat(ret) +end + +--- @param cls table +local function render_classes(cls) + local ret = {} --- @type string[] + + --- @diagnostic disable-next-line:no-unknown + for _, class in vim.spairs(cls) do + ret[#ret + 1] = render_class(class) + end + + return table.concat(ret) +end --- @param fun nvim.luacats.parser.fun --- @param cfg nvim.gen_vimdoc.Config @@ -448,7 +584,7 @@ local function render_fun_header(fun, cfg) local args = {} --- @type string[] for _, p in ipairs(fun.params or {}) do if p.name ~= 'self' then - args[#args + 1] = fmt('{%s}', p.name:gsub('%?$', '')) + args[#args + 1] = fmt_field_name(p.name) end end @@ -480,8 +616,9 @@ end --- @param returns nvim.luacats.parser.return[] --- @param generics? table +--- @param classes? table --- @param exclude_types boolean -local function render_returns(returns, generics, exclude_types) +local function render_returns(returns, generics, classes, exclude_types) local ret = {} --- @type string[] returns = vim.deepcopy(returns) @@ -498,26 +635,26 @@ local function render_returns(returns, generics, exclude_types) end for _, p in ipairs(returns) do + inline_type(p, classes) local rnm, ty, desc = p.name, p.type, p.desc - local blk = '' + + local blk = {} --- @type string[] if ty then - blk = render_type(ty, generics) - end - if rnm then - blk = blk .. ' ' .. rnm + blk[#blk + 1] = render_type(ty, generics) end - if desc then - blk = blk .. ' ' .. desc - end - table.insert(ret, md_to_vimdoc(blk, 8, 8, TEXT_WIDTH, true)) + blk[#blk + 1] = rnm + blk[#blk + 1] = desc + + table.insert(ret, md_to_vimdoc(table.concat(blk, ' '), 8, 8, TEXT_WIDTH, true)) end return table.concat(ret) end --- @param fun nvim.luacats.parser.fun +--- @param classes table --- @param cfg nvim.gen_vimdoc.Config -local function render_fun(fun, cfg) +local function render_fun(fun, classes, cfg) if fun.access or fun.deprecated or fun.nodoc then return end @@ -570,7 +707,7 @@ local function render_fun(fun, cfg) end if fun.params and #fun.params > 0 then - local param_txt = render_fields_or_params(fun.params, fun.generics, cfg.exclude_types) + local param_txt = render_fields_or_params(fun.params, fun.generics, classes, cfg.exclude_types) if not param_txt:match('^%s*$') then table.insert(ret, '\n Parameters: ~\n') ret[#ret + 1] = param_txt @@ -578,7 +715,7 @@ local function render_fun(fun, cfg) end if fun.returns then - local txt = render_returns(fun.returns, fun.generics, cfg.exclude_types) + local txt = render_returns(fun.returns, fun.generics, classes, cfg.exclude_types) if not txt:match('^%s*$') then table.insert(ret, '\n') ret[#ret + 1] = txt @@ -597,15 +734,16 @@ local function render_fun(fun, cfg) end --- @param funs nvim.luacats.parser.fun[] +--- @param classes table --- @param cfg nvim.gen_vimdoc.Config -local function render_funs(funs, cfg) +local function render_funs(funs, classes, cfg) local ret = {} --- @type string[] for _, f in ipairs(funs) do if cfg.fn_xform then cfg.fn_xform(f) end - ret[#ret + 1] = render_fun(f, cfg) + ret[#ret + 1] = render_fun(f, classes, cfg) end -- Sort via prototype @@ -745,15 +883,35 @@ local function gen_target(cfg) expand_files(cfg.files) + --- @type table, [2]: nvim.luacats.parser.fun[], [3]: string[]}> + local file_results = {} + + --- @type table + local all_classes = {} + + --- First pass so we can collect all classes for _, f in pairs(cfg.files) do local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']] local parser = assert(parsers[ext]) - local _, funs, briefs = parser(f) + local classes, funs, briefs = parser(f) + file_results[f] = { classes, funs, briefs } + all_classes = vim.tbl_extend('error', all_classes, classes) + end + + for f, r in pairs(file_results) do + local classes, funs, briefs = r[1], r[2], r[3] + local briefs_txt = {} --- @type string[] for _, b in ipairs(briefs) do briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH) end - local funs_txt = render_funs(funs, cfg) + local funs_txt = render_funs(funs, all_classes, cfg) + if next(classes) then + local classes_txt = render_classes(classes) + if vim.trim(classes_txt) ~= '' then + funs_txt = classes_txt .. '\n' .. funs_txt + end + end -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua` local f_base = assert(vim.fs.basename(f)) sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt) diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua index ee0f9d8e87..0beb1d2352 100644 --- a/scripts/luacats_grammar.lua +++ b/scripts/luacats_grammar.lua @@ -21,8 +21,7 @@ local function opt(x) return x ^ -1 end -local nl = P('\r\n') + P('\n') -local ws = rep1(S(' \t') + nl) +local ws = rep1(S(' \t')) local fill = opt(ws) local any = P(1) -- (consume one character) @@ -30,11 +29,11 @@ local letter = R('az', 'AZ') + S('_$') local num = R('09') local ident = letter * rep(letter + num + S '-.') local string_single = P "'" * rep(any - P "'") * P "'" -local string_double = P '"' * rep(any - P '"') * P '"' +local string_double = P('"') * rep(any - P('"')) * P('"') -local literal = (string_single + string_double + (opt(P '-') * num) + P 'false' + P 'true') +local literal = (string_single + string_double + (opt(P('-')) * num) + P('false') + P('true')) -local lname = (ident + P '...') * opt(P '?') +local lname = (ident + P('...')) * opt(P('?')) --- @param x string local function Pf(x) @@ -47,13 +46,23 @@ local function Sf(x) end --- @param x vim.lpeg.Pattern -local function comma(x) - return x * rep(Pf ',' * x) +local function paren(x) + return Pf('(') * x * fill * P(')') end --- @param x vim.lpeg.Pattern local function parenOpt(x) - return (Pf('(') * x * fill * P(')')) + x + return paren(x) + x +end + +--- @param x vim.lpeg.Pattern +local function comma1(x) + return parenOpt(x * rep(Pf(',') * x)) +end + +--- @param x vim.lpeg.Pattern +local function comma(x) + return opt(comma1(x)) end --- @type table @@ -63,7 +72,15 @@ local v = setmetatable({}, { end, }) +local colon = Pf(':') +local opt_exact = opt(Cg(Pf('(exact)'), 'access')) +local access = P('private') + P('protected') + P('package') +local caccess = Cg(access, 'access') local desc_delim = Sf '#:' + ws +local desc = Cg(rep(any), 'desc') +local opt_desc = opt(desc_delim * desc) +local cname = Cg(ident, 'name') +local opt_parent = opt(colon * Cg(ident, 'parent')) --- @class nvim.luacats.Param --- @field kind 'param' @@ -85,6 +102,7 @@ local desc_delim = Sf '#:' + ws --- @field kind 'class' --- @field name string --- @field parent? string +--- @field access? 'private'|'protected'|'package' --- @class nvim.luacats.Field --- @field kind 'field' @@ -107,112 +125,60 @@ local desc_delim = Sf '#:' + ws --- @class nvim.luacats.grammar --- @field match fun(self, input: string): nvim.luacats.grammar.result? +local function annot(nm, pat) + if type(nm) == 'string' then + nm = P(nm) + end + if pat then + return Ct(Cg(P(nm), 'kind') * fill * pat) + end + return Ct(Cg(P(nm), 'kind')) +end + local grammar = P { rep1(P('@') * (v.ats + v.ext_ats)), - ats = v.at_param - + v.at_return - + v.at_type - + v.at_cast - + v.at_generic - + v.at_class - + v.at_field - + v.at_access - + v.at_deprecated - + v.at_alias - + v.at_enum - + v.at_see - + v.at_diagnostic - + v.at_overload - + v.at_meta, - - ext_ats = v.ext_at_note + v.ext_at_since + v.ext_at_nodoc + v.ext_at_brief, - - at_param = Ct( - Cg(P('param'), 'kind') - * ws - * Cg(lname, 'name') - * ws - * parenOpt(Cg(v.ltype, 'type')) - * opt(desc_delim * Cg(rep(any), 'desc')) - ), - - at_return = Ct( - Cg(P('return'), 'kind') - * ws - * parenOpt(comma(Ct(Cg(v.ltype, 'type') * opt(ws * Cg(ident, 'name'))))) - * opt(desc_delim * Cg(rep(any), 'desc')) - ), - - at_type = Ct( - Cg(P('type'), 'kind') - * ws - * parenOpt(comma(Ct(Cg(v.ltype, 'type')))) - * opt(desc_delim * Cg(rep(any), 'desc')) - ), - - at_cast = Ct( - Cg(P('cast'), 'kind') * ws * Cg(lname, 'name') * ws * opt(Sf('+-')) * Cg(v.ltype, 'type') - ), - - at_generic = Ct( - Cg(P('generic'), 'kind') * ws * Cg(ident, 'name') * opt(Pf ':' * Cg(v.ltype, 'type')) - ), - - at_class = Ct( - Cg(P('class'), 'kind') - * ws - * opt(P('(exact)') * ws) - * Cg(lname, 'name') - * opt(Pf(':') * Cg(lname, 'parent')) - ), - - at_field = Ct( - Cg(P('field'), 'kind') - * ws - * opt(Cg(Pf('private') + Pf('package') + Pf('protected'), 'access')) - * Cg(lname, 'name') - * ws - * Cg(v.ltype, 'type') - * opt(desc_delim * Cg(rep(any), 'desc')) - ), - - at_access = Ct(Cg(P('private') + P('protected') + P('package'), 'kind')), - - at_deprecated = Ct(Cg(P('deprecated'), 'kind')), - - -- Types may be provided on subsequent lines - at_alias = Ct(Cg(P('alias'), 'kind') * ws * Cg(lname, 'name') * opt(ws * Cg(v.ltype, 'type'))), - - at_enum = Ct(Cg(P('enum'), 'kind') * ws * Cg(lname, 'name')), - - at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), - at_diagnostic = Ct(Cg(P('diagnostic'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), - at_overload = Ct(Cg(P('overload'), 'kind') * ws * Cg(v.ltype, 'type')), - at_meta = Ct(Cg(P('meta'), 'kind')), + ats = annot('param', Cg(lname, 'name') * ws * v.ctype * opt_desc) + + annot('return', comma1(Ct(v.ctype * opt(ws * cname))) * opt_desc) + + annot('type', comma1(Ct(v.ctype)) * opt_desc) + + annot('cast', cname * ws * opt(Sf('+-')) * v.ctype) + + annot('generic', cname * opt(colon * v.ctype)) + + annot('class', opt_exact * opt(paren(caccess)) * fill * cname * opt_parent) + + annot('field', opt(caccess * ws) * v.field_name * ws * v.ctype * opt_desc) + + annot('operator', cname * opt(paren(Cg(v.ltype, 'argtype'))) * colon * v.ctype) + + annot(access) + + annot('deprecated') + + annot('alias', cname * opt(ws * v.ctype)) + + annot('enum', cname) + + annot('overload', v.ctype) + + annot('see', opt(desc_delim) * desc) + + annot('diagnostic', opt(desc_delim) * desc) + + annot('meta'), --- Custom extensions - ext_at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), - - -- TODO only consume 1 line - ext_at_since = Ct(Cg(P('since'), 'kind') * ws * Cg(rep(any), 'desc')), + ext_ats = ( + annot('note', desc) + + annot('since', desc) + + annot('nodoc') + + annot('inlinedoc') + + annot('brief', desc) + ), - ext_at_nodoc = Ct(Cg(P('nodoc'), 'kind')), - ext_at_brief = Ct(Cg(P('brief'), 'kind') * opt(ws * Cg(rep(any), 'desc'))), + field_name = Cg(lname + (v.ty_index * opt(P('?'))), 'name'), - ltype = v.ty_union + Pf '(' * v.ty_union * fill * P ')', + ctype = parenOpt(Cg(v.ltype, 'type')), + ltype = parenOpt(v.ty_union), - ty_union = v.ty_opt * rep(Pf '|' * v.ty_opt), + ty_union = v.ty_opt * rep(Pf('|') * v.ty_opt), ty = v.ty_fun + ident + v.ty_table + literal, - ty_param = Pf '<' * comma(v.ltype) * fill * P '>', - ty_opt = v.ty * opt(v.ty_param) * opt(P '[]') * opt(P '?'), - - table_key = (Pf '[' * literal * Pf ']') + lname, - table_elem = v.table_key * Pf ':' * v.ltype, - ty_table = Pf '{' * comma(v.table_elem) * Pf '}', - - fun_param = lname * opt(Pf ':' * v.ltype), - ty_fun = Pf 'fun(' * rep(comma(v.fun_param)) * fill * P ')' * opt(Pf ':' * comma(v.ltype)), + ty_param = Pf('<') * comma1(v.ltype) * fill * P('>'), + ty_opt = v.ty * opt(v.ty_param) * opt(P('[]')) * opt(P('?')), + ty_index = (Pf('[') * v.ltype * Pf(']')), + table_key = v.ty_index + lname, + table_elem = v.table_key * colon * v.ltype, + ty_table = Pf('{') * comma1(v.table_elem) * Pf('}'), + fun_param = lname * opt(colon * v.ltype), + ty_fun = Pf('fun') * paren(comma(lname * opt(colon * v.ltype))) * opt(colon * comma1(v.ltype)), } return grammar --[[@as nvim.luacats.grammar]] diff --git a/scripts/luacats_parser.lua b/scripts/luacats_parser.lua index 520272d1dc..cd671fb9dc 100644 --- a/scripts/luacats_parser.lua +++ b/scripts/luacats_parser.lua @@ -19,7 +19,7 @@ local luacats_grammar = require('scripts.luacats_grammar') --- @class nvim.luacats.parser.alias --- @field kind 'alias' ---- @field type string +--- @field type string[] --- @field desc string --- @class nvim.luacats.parser.fun @@ -49,8 +49,12 @@ local luacats_grammar = require('scripts.luacats_grammar') --- @class nvim.luacats.parser.class --- @field kind 'class' +--- @field parent? string --- @field name string --- @field desc string +--- @field nodoc? true +--- @field inlinedoc? true +--- @field access? 'private'|'package'|'protected' --- @field fields nvim.luacats.parser.field[] --- @field notes? string[] @@ -64,6 +68,7 @@ local luacats_grammar = require('scripts.luacats_grammar') --- | nvim.luacats.parser.class --- | nvim.luacats.parser.fun --- | nvim.luacats.parser.brief +--- | nvim.luacats.parser.alias -- Remove this when we document classes properly --- Some doc lines have the form: @@ -142,22 +147,27 @@ local function process_doc_line(line, state) } elseif kind == 'class' then --- @cast parsed nvim.luacats.Class - state.cur_obj = { - kind = 'class', - name = parsed.name, - parent = parsed.parent, - desc = '', - fields = {}, - } + cur_obj.kind = 'class' + cur_obj.name = parsed.name + cur_obj.parent = parsed.parent + cur_obj.access = parsed.access + cur_obj.desc = state.doc_lines and table.concat(state.doc_lines, '\n') or nil + state.doc_lines = nil + cur_obj.fields = {} elseif kind == 'field' then --- @cast parsed nvim.luacats.Field - if not parsed.access then - parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil - if parsed.desc then - parsed.desc = vim.trim(parsed.desc) - end - table.insert(cur_obj.fields, parsed) + parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil + if parsed.desc then + parsed.desc = vim.trim(parsed.desc) + end + table.insert(cur_obj.fields, parsed) + state.doc_lines = nil + elseif kind == 'operator' then + parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil + if parsed.desc then + parsed.desc = vim.trim(parsed.desc) end + table.insert(cur_obj.fields, parsed) state.doc_lines = nil elseif kind == 'param' then state.last_doc_item_indent = nil @@ -191,6 +201,8 @@ local function process_doc_line(line, state) cur_obj.access = 'protected' elseif kind == 'deprecated' then cur_obj.deprecated = true + elseif kind == 'inlinedoc' then + cur_obj.inlinedoc = true elseif kind == 'nodoc' then cur_obj.nodoc = true elseif kind == 'since' then @@ -383,11 +395,11 @@ end --- Determine the table name used to export functions of a module --- Usually this is `M`. ---- @param filename string +--- @param str string --- @return string? -local function determine_modvar(filename) +local function determine_modvar(str) local modvar --- @type string? - for line in io.lines(filename) do + for line in vim.gsplit(str, '\n') do do --- @type string? local m = line:match('^return%s+([a-zA-Z_]+)') @@ -462,17 +474,12 @@ end local M = {} ---- @param filename string ---- @return table classes ---- @return nvim.luacats.parser.fun[] funs ---- @return string[] briefs ---- @return nvim.luacats.parser.obj[] -function M.parse(filename) +function M.parse_str(str, filename) local funs = {} --- @type nvim.luacats.parser.fun[] local classes = {} --- @type table local briefs = {} --- @type string[] - local mod_return = determine_modvar(filename) + local mod_return = determine_modvar(str) --- @type string local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename @@ -485,7 +492,7 @@ function M.parse(filename) -- Keep track of any partial objects we don't commit local uncommitted = {} --- @type nvim.luacats.parser.obj[] - for line in io.lines(filename) do + for line in vim.gsplit(str, '\n') do local has_indent = line:match('^%s+') ~= nil line = vim.trim(line) if vim.startswith(line, '---') then @@ -518,4 +525,13 @@ function M.parse(filename) return classes, funs, briefs, uncommitted end +--- @param filename string +function M.parse(filename) + local f = assert(io.open(filename, 'r')) + local txt = f:read('*all') + f:close() + + return M.parse_str(txt, filename) +end + return M diff --git a/scripts/text_utils.lua b/scripts/text_utils.lua index 5167ec42f2..ec0ccf668b 100644 --- a/scripts/text_utils.lua +++ b/scripts/text_utils.lua @@ -175,7 +175,11 @@ local function render_md(node, start_indent, indent, text_width, level, is_list) error(fmt('cannot render:\n%s', vim.inspect(node))) end for i, child in ipairs(node) do - vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1, is_list)) + local start_indent0 = i == 1 and start_indent or indent + vim.list_extend( + parts, + render_md(child, start_indent0, indent, text_width, level + 1, is_list) + ) if node.type ~= 'list' and i ~= #node then if (node[i + 1] or {}).type ~= 'list' then parts[#parts + 1] = '\n' -- cgit From a4290f462ed7dc81e17b09bd27877b106b24b6bd Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 5 Mar 2024 12:06:15 +0000 Subject: docs(lua): improvements for LSP and Diagnostic --- scripts/gen_vimdoc.lua | 15 ++++++++------- scripts/luacats_grammar.lua | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua index 8e2ff8193c..22df411a35 100755 --- a/scripts/gen_vimdoc.lua +++ b/scripts/gen_vimdoc.lua @@ -536,7 +536,8 @@ local function render_fields_or_params(xs, generics, classes, exclude_types) end --- @param class nvim.luacats.parser.class -local function render_class(class) +--- @param classes table +local function render_class(class, classes) if class.access or class.nodoc or class.inlinedoc then return end @@ -548,13 +549,14 @@ local function render_class(class) if class.parent then local txt = fmt('Extends: |%s|', class.parent) table.insert(ret, md_to_vimdoc(txt, INDENTATION, INDENTATION, TEXT_WIDTH)) + table.insert(ret, '\n') end if class.desc then table.insert(ret, md_to_vimdoc(class.desc, INDENTATION, INDENTATION, TEXT_WIDTH)) end - local fields_txt = render_fields_or_params(class.fields) + local fields_txt = render_fields_or_params(class.fields, nil, classes) if not fields_txt:match('^%s*$') then table.insert(ret, '\n Fields: ~\n') table.insert(ret, fields_txt) @@ -564,13 +566,12 @@ local function render_class(class) return table.concat(ret) end ---- @param cls table -local function render_classes(cls) +--- @param classes table +local function render_classes(classes) local ret = {} --- @type string[] - --- @diagnostic disable-next-line:no-unknown - for _, class in vim.spairs(cls) do - ret[#ret + 1] = render_class(class) + for _, class in vim.spairs(classes) do + ret[#ret + 1] = render_class(class, classes) end return table.concat(ret) diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua index 0beb1d2352..158541dc77 100644 --- a/scripts/luacats_grammar.lua +++ b/scripts/luacats_grammar.lua @@ -170,7 +170,7 @@ local grammar = P { ltype = parenOpt(v.ty_union), ty_union = v.ty_opt * rep(Pf('|') * v.ty_opt), - ty = v.ty_fun + ident + v.ty_table + literal, + ty = v.ty_fun + ident + v.ty_table + literal + paren(v.ty), ty_param = Pf('<') * comma1(v.ltype) * fill * P('>'), ty_opt = v.ty * opt(v.ty_param) * opt(P('[]')) * opt(P('?')), ty_index = (Pf('[') * v.ltype * Pf(']')), -- cgit From ade1b12f49c3b3914c74847d791eb90ea90b56b7 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Fri, 8 Mar 2024 12:25:18 +0000 Subject: docs: support inline markdown - Tags are now created with `[tag]()` - References are now created with `[tag]` - Code spans are no longer wrapped --- scripts/luacats_grammar.lua | 4 +- scripts/text_utils.lua | 154 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 138 insertions(+), 20 deletions(-) (limited to 'scripts') diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua index 158541dc77..ca26c70156 100644 --- a/scripts/luacats_grammar.lua +++ b/scripts/luacats_grammar.lua @@ -173,10 +173,10 @@ local grammar = P { ty = v.ty_fun + ident + v.ty_table + literal + paren(v.ty), ty_param = Pf('<') * comma1(v.ltype) * fill * P('>'), ty_opt = v.ty * opt(v.ty_param) * opt(P('[]')) * opt(P('?')), - ty_index = (Pf('[') * v.ltype * Pf(']')), + ty_index = (Pf('[') * (v.ltype + ident + rep1(num)) * fill * P(']')), table_key = v.ty_index + lname, table_elem = v.table_key * colon * v.ltype, - ty_table = Pf('{') * comma1(v.table_elem) * Pf('}'), + ty_table = Pf('{') * comma1(v.table_elem) * fill * P('}'), fun_param = lname * opt(colon * v.ltype), ty_fun = Pf('fun') * paren(comma(lname * opt(colon * v.ltype))) * opt(colon * comma1(v.ltype)), } diff --git a/scripts/text_utils.lua b/scripts/text_utils.lua index ec0ccf668b..937408c546 100644 --- a/scripts/text_utils.lua +++ b/scripts/text_utils.lua @@ -7,12 +7,99 @@ local fmt = string.format local INDENTATION = 4 +local NBSP = string.char(160) + local M = {} local function contains(t, xs) return vim.tbl_contains(xs, t) end +--- @param txt string +--- @param srow integer +--- @param scol integer +--- @param erow? integer +--- @param ecol? integer +--- @return string +local function slice_text(txt, srow, scol, erow, ecol) + local lines = vim.split(txt, '\n') + + if srow == erow then + return lines[srow + 1]:sub(scol + 1, ecol) + end + + if erow then + -- Trim the end + for _ = erow + 2, #lines do + table.remove(lines, #lines) + end + end + + -- Trim the start + for _ = 1, srow do + table.remove(lines, 1) + end + + lines[1] = lines[1]:sub(scol + 1) + lines[#lines] = lines[#lines]:sub(1, ecol) + + return table.concat(lines, '\n') +end + +--- @param text string +--- @return nvim.text_utils.MDNode +local function parse_md_inline(text) + local parser = vim.treesitter.languagetree.new(text, 'markdown_inline') + local root = parser:parse(true)[1]:root() + + --- @param node TSNode + --- @return nvim.text_utils.MDNode? + local function extract(node) + local ntype = node:type() + + if ntype:match('^%p$') then + return + end + + --- @type table + local ret = { type = ntype } + ret.text = vim.treesitter.get_node_text(node, text) + + local row, col = 0, 0 + + for child, child_field in node:iter_children() do + local e = extract(child) + if e and ntype == 'inline' then + local srow, scol = child:start() + if (srow == row and scol > col) or srow > row then + local t = slice_text(ret.text, row, col, srow, scol) + if t and t ~= '' then + table.insert(ret, { type = 'text', j = true, text = t }) + end + end + row, col = child:end_() + end + + if child_field then + ret[child_field] = e + else + table.insert(ret, e) + end + end + + if ntype == 'inline' and (row > 0 or col > 0) then + local t = slice_text(ret.text, row, col) + if t and t ~= '' then + table.insert(ret, { type = 'text', text = t }) + end + end + + return ret + end + + return extract(root) or {} +end + --- @param text string --- @return nvim.text_utils.MDNode local function parse_md(text) @@ -47,6 +134,10 @@ local function parse_md(text) ret.text = vim.treesitter.get_node_text(node, text) end + if ntype == 'inline' then + ret = parse_md_inline(ret.text) + end + for child, child_field in node:iter_children() do local e = extract(child) if child_field then @@ -101,20 +192,47 @@ local function render_md(node, start_indent, indent, text_width, level, is_list) local add_tag = false -- local add_tag = true + local ntype = node.type + if add_tag then - parts[#parts + 1] = '<' .. node.type .. '>' + parts[#parts + 1] = '<' .. ntype .. '>' end - if node.type == 'paragraph' then - local text = assert(node.text) - text = text:gsub('(%s)%*(%w+)%*(%s)', '%1%2%3') - text = text:gsub('(%s)_(%w+)_(%s)', '%1%2%3') - text = text:gsub('\\|', '|') - text = text:gsub('\\%*', '*') - text = text:gsub('\\_', '_') - parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width) + if ntype == 'text' then + parts[#parts + 1] = node.text + elseif ntype == 'html_tag' then + error('html_tag: ' .. node.text) + elseif ntype == 'inline_link' then + vim.list_extend(parts, { '*', node[1].text, '*' }) + elseif ntype == 'shortcut_link' then + if node[1].text:find('^<.*>$') then + parts[#parts + 1] = node[1].text + else + vim.list_extend(parts, { '|', node[1].text, '|' }) + end + elseif ntype == 'backslash_escape' then + parts[#parts + 1] = node.text + elseif ntype == 'emphasis' then + parts[#parts + 1] = node.text:sub(2, -2) + elseif ntype == 'code_span' then + vim.list_extend(parts, { '`', node.text:sub(2, -2):gsub(' ', NBSP), '`' }) + elseif ntype == 'inline' then + if #node == 0 then + local text = assert(node.text) + parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width) + else + for _, child in ipairs(node) do + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1)) + end + end + elseif ntype == 'paragraph' then + local pparts = {} + for _, child in ipairs(node) do + vim.list_extend(pparts, render_md(child, start_indent, indent, text_width, level + 1)) + end + parts[#parts + 1] = M.wrap(table.concat(pparts), start_indent, indent, text_width) parts[#parts + 1] = '\n' - elseif node.type == 'code_fence_content' then + elseif ntype == 'code_fence_content' then local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n') local cindent = indent + INDENTATION @@ -137,7 +255,7 @@ local function render_md(node, start_indent, indent, text_width, level, is_list) end parts[#parts + 1] = '\n' end - elseif node.type == 'fenced_code_block' then + elseif ntype == 'fenced_code_block' then parts[#parts + 1] = '>' for _, child in ipairs(node) do if child.type == 'info_string' then @@ -152,15 +270,15 @@ local function render_md(node, start_indent, indent, text_width, level, is_list) end end parts[#parts + 1] = '<\n' - elseif node.type == 'html_block' then + elseif ntype == 'html_block' then local text = node.text:gsub('^
help', '')
     text = text:gsub('
%s*$', '') parts[#parts + 1] = text - elseif node.type == 'list_marker_dot' then + elseif ntype == 'list_marker_dot' then parts[#parts + 1] = node.text - elseif contains(node.type, { 'list_marker_minus', 'list_marker_star' }) then + elseif contains(ntype, { 'list_marker_minus', 'list_marker_star' }) then parts[#parts + 1] = '• ' - elseif node.type == 'list_item' then + elseif ntype == 'list_item' then parts[#parts + 1] = string.rep(' ', indent) local offset = node[1].type == 'list_marker_dot' and 3 or 2 for i, child in ipairs(node) do @@ -180,7 +298,7 @@ local function render_md(node, start_indent, indent, text_width, level, is_list) parts, render_md(child, start_indent0, indent, text_width, level + 1, is_list) ) - if node.type ~= 'list' and i ~= #node then + if ntype ~= 'list' and i ~= #node then if (node[i + 1] or {}).type ~= 'list' then parts[#parts + 1] = '\n' end @@ -189,7 +307,7 @@ local function render_md(node, start_indent, indent, text_width, level, is_list) end if add_tag then - parts[#parts + 1] = '' + parts[#parts + 1] = '' end return parts @@ -227,7 +345,7 @@ function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list) local parsed = parse_md(text .. '\n') local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list) - local lines = vim.split(table.concat(ret), '\n') + local lines = vim.split(table.concat(ret):gsub(NBSP, ' '), '\n') lines = vim.tbl_map(align_tags(text_width), lines) -- cgit