diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/bump_deps.lua | 53 | ||||
| -rw-r--r-- | scripts/cdoc_grammar.lua | 87 | ||||
| -rw-r--r-- | scripts/cdoc_parser.lua | 223 | ||||
| -rwxr-xr-x | scripts/gen_eval_files.lua | 406 | ||||
| -rw-r--r-- | scripts/gen_filetype.lua | 102 | ||||
| -rw-r--r-- | scripts/gen_help_html.lua | 587 | ||||
| -rw-r--r-- | scripts/gen_lsp.lua | 294 | ||||
| -rwxr-xr-x | scripts/gen_vimdoc.lua | 953 | ||||
| -rwxr-xr-x | scripts/gen_vimdoc.py | 1447 | ||||
| -rwxr-xr-x | scripts/genappimage.sh | 6 | ||||
| -rw-r--r-- | scripts/lintcommit.lua | 76 | ||||
| -rwxr-xr-x | scripts/lintdoc.lua | 20 | ||||
| -rw-r--r-- | scripts/lua2dox.lua | 475 | ||||
| -rw-r--r-- | scripts/luacats_grammar.lua | 184 | ||||
| -rw-r--r-- | scripts/luacats_parser.lua | 537 | ||||
| -rw-r--r-- | scripts/text_utils.lua | 361 | ||||
| -rwxr-xr-x | scripts/vim-patch.sh | 6 | ||||
| -rwxr-xr-x | scripts/vimpatch.lua | 10 | ||||
| -rw-r--r-- | scripts/windows.ti | 6 |
19 files changed, 3429 insertions, 2404 deletions
diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua index 076ad374cf..c5294893e0 100755 --- a/scripts/bump_deps.lua +++ b/scripts/bump_deps.lua @@ -138,9 +138,10 @@ local function get_archive_info(repo, ref) 'Failed to download archive from GitHub' ) - local shacmd = (vim.fn.executable('sha256sum') == 1 - and{ 'sha256sum', archive_path } - or { 'shasum', '-a', '256', archive_path }) + local shacmd = ( + vim.fn.executable('sha256sum') == 1 and { 'sha256sum', archive_path } + or { 'shasum', '-a', '256', archive_path } + ) local archive_sha = run(shacmd):gmatch('%w+')() return { url = archive_url, sha = archive_sha } end @@ -152,18 +153,7 @@ local function write_cmakelists_line(symbol, kind, value) 'sed', '-i', '-e', - 's/' - .. symbol - .. '_' - .. kind - .. '.*$' - .. '/' - .. symbol - .. '_' - .. kind - .. ' ' - .. value - .. '/', + 's/' .. symbol .. '_' .. kind .. '.*$' .. '/' .. symbol .. '_' .. kind .. ' ' .. value .. '/', deps_file, }, 'Failed to write ' .. deps_file) end @@ -203,16 +193,13 @@ local function update_cmakelists(dependency, archive, comment) p('Updating ' .. dependency.name .. ' to ' .. archive.url .. '\n') write_cmakelists_line(dependency.symbol, 'URL', archive.url:gsub('/', '\\/')) write_cmakelists_line(dependency.symbol, 'SHA256', archive.sha) - run_die( - { - 'git', - 'commit', - deps_file, - '-m', - commit_prefix .. 'bump ' .. dependency.name .. ' to ' .. comment, - }, - 'git failed to commit' - ) + run_die({ + 'git', + 'commit', + deps_file, + '-m', + commit_prefix .. 'bump ' .. dependency.name .. ' to ' .. comment, + }, 'git failed to commit') end local function verify_cmakelists_committed() @@ -318,9 +305,9 @@ function M.commit(dependency_name, commit) end function M.version(dependency_name, version) - vim.validate{ - dependency_name={dependency_name,'s'}, - version={version,'s'}, + vim.validate { + dependency_name = { dependency_name, 's' }, + version = { version, 's' }, } local dependency = assert(get_dependency(dependency_name)) verify_cmakelists_committed() @@ -384,7 +371,7 @@ function M.submit_pr() end local function usage() - local this_script = _G.arg[0]:match("[^/]*.lua$") + local this_script = _G.arg[0]:match('[^/]*.lua$') print(([=[ Bump Nvim dependencies @@ -421,13 +408,13 @@ local function parseargs() elseif _G.arg[i] == '--pr' then args.pr = true elseif _G.arg[i] == '--branch' then - args.branch = _G.arg[i+1] + args.branch = _G.arg[i + 1] elseif _G.arg[i] == '--dep' then - args.dep = _G.arg[i+1] + args.dep = _G.arg[i + 1] elseif _G.arg[i] == '--version' then - args.version = _G.arg[i+1] + args.version = _G.arg[i + 1] elseif _G.arg[i] == '--commit' then - args.commit = _G.arg[i+1] + args.commit = _G.arg[i + 1] elseif _G.arg[i] == '--head' then args.head = true end diff --git a/scripts/cdoc_grammar.lua b/scripts/cdoc_grammar.lua new file mode 100644 index 0000000000..6a7610883b --- /dev/null +++ b/scripts/cdoc_grammar.lua @@ -0,0 +1,87 @@ +--[[! +LPEG grammar for C doc comments +]] + +--- @class nvim.cdoc.Param +--- @field kind 'param' +--- @field name string +--- @field desc? string + +--- @class nvim.cdoc.Return +--- @field kind 'return' +--- @field desc string + +--- @class nvim.cdoc.Note +--- @field desc? string + +--- @alias nvim.cdoc.grammar.result +--- | nvim.cdoc.Param +--- | nvim.cdoc.Return +--- | nvim.cdoc.Note + +--- @class nvim.cdoc.grammar +--- @field match fun(self, input: string): nvim.cdoc.grammar.result? + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local Ct, Cg = lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local nl = P('\r\n') + P('\n') +local ws = rep1(S(' \t') + nl) + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local ident = letter * rep(letter + R('09')) + +local io = P('[') * (P('in') + P('out') + P('inout')) * P(']') + +--- @param x string +local function Pf(x) + return opt(ws) * P(x) * opt(ws) +end + +--- @type table<string,vim.lpeg.Pattern> +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +local grammar = P { + rep1(P('@') * v.ats), + + ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc, + + at_param = Ct( + Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc')) + ), + + at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))), + + at_deprecated = Ct(Cg(P('deprecated'), 'kind')), + + at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + + at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_nodoc = Ct(Cg(P('nodoc'), 'kind')), +} + +return grammar --[[@as nvim.cdoc.grammar]] diff --git a/scripts/cdoc_parser.lua b/scripts/cdoc_parser.lua new file mode 100644 index 0000000000..5f0dc7be2c --- /dev/null +++ b/scripts/cdoc_parser.lua @@ -0,0 +1,223 @@ +local cdoc_grammar = require('scripts.cdoc_grammar') +local c_grammar = require('src.nvim.generators.c_grammar') + +--- @class nvim.cdoc.parser.param +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.note +--- @field desc string + +--- @class nvim.cdoc.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.cdoc.parser.fun +--- @field name string +--- @field params nvim.cdoc.parser.param[] +--- @field returns nvim.cdoc.parser.return[] +--- @field desc string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field notes? nvim.cdoc.parser.note[] +--- @field see? nvim.cdoc.parser.note[] + +--- @class nvim.cdoc.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.cdoc.parser.obj +--- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.cdoc.parser.obj +--- | nvim.cdoc.parser.fun +--- | nvim.cdoc.parser.brief + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.cdoc.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.cdoc.parser.State +local function process_doc_line(line, state) + line = line:gsub('^%s+@', '@') + + local parsed = cdoc_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + + local kind = parsed.kind + + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + state.last_doc_item = { + name = parsed.name, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = { { + desc = parsed.desc, + } } + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[1] + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param item table +--- @param state nvim.cdoc.parser.State +local function process_proto(item, state) + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + cur_obj.name = item.name + cur_obj.params = cur_obj.params or {} + + for _, p in ipairs(item.parameters) do + local param = { name = p[2], type = p[1] } + local added = false + for _, cp in ipairs(cur_obj.params) do + if cp.name == param.name then + cp.type = param.type + added = true + break + end + end + + if not added then + table.insert(cur_obj.params, param) + end + end + + cur_obj.returns = cur_obj.returns or { {} } + cur_obj.returns[1].type = item.return_type + + for _, a in ipairs({ + 'fast', + 'remote_only', + 'lua_only', + 'textlock', + 'textlock_allow_cmdwin', + }) do + if item[a] then + cur_obj.attrs = cur_obj.attrs or {} + table.insert(cur_obj.attrs, a) + end + end + + cur_obj.deprecated_since = item.deprecated_since + + -- Remove some arguments + for i = #cur_obj.params, 1, -1 do + local p = cur_obj.params[i] + if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then + table.remove(cur_obj.params, i) + end + end +end + +local M = {} + +--- @param filename string +--- @return {} classes +--- @return nvim.cdoc.parser.fun[] funs +--- @return string[] briefs +function M.parse(filename) + local funs = {} --- @type nvim.cdoc.parser.fun[] + local briefs = {} --- @type string[] + local state = {} --- @type nvim.cdoc.parser.State + + local txt = assert(io.open(filename, 'r')):read('*all') + + local parsed = c_grammar.grammar:match(txt) + for _, item in ipairs(parsed) do + if item.comment then + process_doc_line(item.comment, state) + else + add_doc_lines_to_obj(state) + if item[1] == 'proto' then + process_proto(item, state) + table.insert(funs, state.cur_obj) + end + local cur_obj = state.cur_obj + if cur_obj and not item.static then + if cur_obj.kind == 'brief' then + table.insert(briefs, cur_obj.desc) + end + end + state = {} + end + end + + return {}, funs, briefs +end + +-- M.parse('src/nvim/api/vim.c') + +return M diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index e331dd996e..f1bba5c0a2 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -1,7 +1,8 @@ +#!/usr/bin/env -S nvim -l + -- Generator for various vimdoc and Lua type files -local DEP_API_METADATA = 'build/api_metadata.mpack' -local DEP_API_DOC = 'runtime/doc/api.mpack' +local DEP_API_METADATA = 'build/funcs_metadata.mpack' --- @class vim.api.metadata --- @field name string @@ -17,6 +18,31 @@ local DEP_API_DOC = 'runtime/doc/api.mpack' --- @field remote boolean --- @field since integer +local LUA_API_RETURN_OVERRIDES = { + nvim_buf_get_command = 'table<string,vim.api.keyset.command_info>', + nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item', + nvim_buf_get_extmarks = 'vim.api.keyset.get_extmark_item[]', + nvim_buf_get_keymap = 'vim.api.keyset.keymap[]', + nvim_get_autocmds = 'vim.api.keyset.get_autocmds.ret[]', + nvim_get_color_map = 'table<string,integer>', + nvim_get_command = 'table<string,vim.api.keyset.command_info>', + nvim_get_keymap = 'vim.api.keyset.keymap[]', + nvim_get_mark = 'vim.api.keyset.get_mark', + + -- Can also return table<string,vim.api.keyset.hl_info>, however we need to + -- pick one to get some benefit. + -- REVISIT lewrus01 (26/01/24): we can maybe add + -- @overload fun(ns: integer, {}): table<string,vim.api.keyset.hl_info> + nvim_get_hl = 'vim.api.keyset.hl_info', + + nvim_get_mode = 'vim.api.keyset.get_mode', + nvim_get_namespaces = 'table<string,integer>', + nvim_get_option_info = 'vim.api.keyset.get_option_info', + nvim_get_option_info2 = 'vim.api.keyset.get_option_info', + nvim_parse_cmd = 'vim.api.keyset.parse_cmd', + nvim_win_get_config = 'vim.api.keyset.win_config', +} + local LUA_META_HEADER = { '--- @meta _', '-- THIS FILE IS GENERATED', @@ -48,6 +74,16 @@ local LUA_OPTION_META_HEADER = { 'vim.wo = vim.wo', } +local LUA_VVAR_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", + '', + '--- @class vim.v', + 'vim.v = ...', +} + local LUA_KEYWORDS = { ['and'] = true, ['end'] = true, @@ -56,10 +92,12 @@ local LUA_KEYWORDS = { ['if'] = true, ['while'] = true, ['repeat'] = true, + ['true'] = true, + ['false'] = true, } local OPTION_TYPES = { - bool = 'boolean', + boolean = 'boolean', number = 'integer', string = 'string', } @@ -76,6 +114,7 @@ local API_TYPES = { LuaRef = 'function', Dictionary = 'table<string,any>', Float = 'number', + HLGroupID = 'number|string', void = '', } @@ -170,44 +209,65 @@ end --- @return table<string, vim.EvalFn> local function get_api_meta() - local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) - local metadata = vim.mpack.decode(mpack_f:read('*all')) --[[@as vim.api.metadata[] ]] local ret = {} --- @type table<string, vim.EvalFn> - local doc_mpack_f = assert(io.open(DEP_API_DOC, 'rb')) - local doc_metadata = vim.mpack.decode(doc_mpack_f:read('*all')) --[[@as table<string,vim.gen_vim_doc_fun>]] - - for _, fun in ipairs(metadata) do - if fun.lua then - local fdoc = doc_metadata[fun.name] - - local params = {} --- @type {[1]:string,[2]:string}[] - for _, p in ipairs(fun.parameters) do - local ptype, pname = p[1], p[2] - params[#params + 1] = { - pname, - api_type(ptype), - fdoc and fdoc.parameters_doc[pname] or nil, - } - end + local cdoc_parser = require('scripts.cdoc_parser') - local r = { - signature = 'NA', - name = fun.name, - params = params, - returns = api_type(fun.return_type), - deprecated = fun.deprecated_since ~= nil, - } + local f = 'src/nvim/api' - if fdoc then - if #fdoc.doc > 0 then - r.desc = table.concat(fdoc.doc, '\n') + local function include(fun) + if not vim.startswith(fun.name, 'nvim_') then + return false + end + if vim.tbl_contains(fun.attrs or {}, 'lua_only') then + return true + end + if vim.tbl_contains(fun.attrs or {}, 'remote_only') then + return false + end + return true + end + + --- @type table<string,nvim.cdoc.parser.fun> + local functions = {} + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + local filename = vim.fs.joinpath(f, path) + local _, funs = cdoc_parser.parse(filename) + for _, fn in ipairs(funs) do + if include(fn) then + functions[fn.name] = fn end - r.return_desc = (fdoc['return'] or {})[1] end + end + end - ret[fun.name] = r + for _, fun in pairs(functions) do + local deprecated = fun.deprecated_since ~= nil + + local params = {} --- @type {[1]:string,[2]:string}[] + for _, p in ipairs(fun.params) do + params[#params + 1] = { + p.name, + api_type(p.type), + not deprecated and p.desc or nil, + } end + + local r = { + signature = 'NA', + name = fun.name, + params = params, + returns = api_type(fun.returns[1].type), + deprecated = deprecated, + } + + if not deprecated then + r.desc = fun.desc + r.return_desc = fun.returns[1].desc + end + + ret[fun.name] = r end return ret end @@ -235,12 +295,10 @@ end --- @param fun vim.EvalFn --- @param write fun(line: string) local function render_api_meta(_f, fun, write) - if not vim.startswith(fun.name, 'nvim_') then - return - end - write('') + local text_utils = require('scripts.text_utils') + if vim.startswith(fun.name, 'nvim__') then write('--- @private') end @@ -251,10 +309,10 @@ local function render_api_meta(_f, fun, write) local desc = fun.desc if desc then + desc = text_utils.md_to_vimdoc(desc, 0, 0, 74) for _, l in ipairs(split(norm_text(desc))) do write('--- ' .. l) end - write('---') end local param_names = {} --- @type string[] @@ -263,8 +321,11 @@ local function render_api_meta(_f, fun, write) param_names[#param_names + 1] = p[1] local pdesc = p[3] if pdesc then - local pdesc_a = split(norm_text(pdesc)) - write('--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' .. pdesc_a[1]) + local s = '--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' + local indent = #('@param ' .. p[1] .. ' ') + pdesc = text_utils.md_to_vimdoc(pdesc, #s, indent, 74, true) + local pdesc_a = split(vim.trim(norm_text(pdesc))) + write(s .. pdesc_a[1]) for i = 2, #pdesc_a do if not pdesc_a[i] then break @@ -276,11 +337,10 @@ local function render_api_meta(_f, fun, write) end end if fun.returns ~= '' then - if fun.returns_desc then - write('--- @return ' .. fun.returns .. ' : ' .. fun.returns_desc) - else - write('--- @return ' .. fun.returns) - end + local ret_desc = fun.returns_desc and ' : ' .. fun.returns_desc or '' + ret_desc = text_utils.md_to_vimdoc(ret_desc, 0, 0, 74) + local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns + write('--- @return ' .. ret .. ret_desc) end local param_str = table.concat(param_names, ', ') @@ -290,8 +350,6 @@ end --- @return table<string, vim.EvalFn> local function get_api_keysets_meta() local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) - - --- @diagnostic disable-next-line:no-unknown local metadata = assert(vim.mpack.decode(mpack_f:read('*all'))) local ret = {} --- @type table<string, vim.EvalFn> @@ -302,7 +360,7 @@ local function get_api_keysets_meta() for _, k in ipairs(keysets) do local params = {} for _, key in ipairs(k.keys) do - table.insert(params, {key..'?', api_type(k.types[key] or 'any')}) + table.insert(params, { key .. '?', api_type(k.types[key] or 'any') }) end ret[k.name] = { signature = 'NA', @@ -342,50 +400,45 @@ local function render_eval_meta(f, fun, write) local params = process_params(fun.params) - if fun.signature then - write('') - if fun.deprecated then - write('--- @deprecated') - end - - local desc = fun.desc - - if desc then - --- @type string - desc = desc:gsub('\n%s*\n%s*$', '\n') - for _, l in ipairs(split(desc)) do - l = l:gsub('^ ', ''):gsub('\t', ' '):gsub('@', '\\@') - write('--- ' .. l) - end - end + write('') + if fun.deprecated then + write('--- @deprecated') + end - local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 + local desc = fun.desc - for i, param in ipairs(params) do - local pname, ptype = param[1], param[2] - local optional = (pname ~= '...' and i > req_args) and '?' or '' - write(string.format('--- @param %s%s %s', pname, optional, ptype)) + if desc then + --- @type string + desc = desc:gsub('\n%s*\n%s*$', '\n') + for _, l in ipairs(split(desc)) do + l = l:gsub('^ ', ''):gsub('\t', ' '):gsub('@', '\\@') + write('--- ' .. l) end + end - if fun.returns ~= false then - write('--- @return ' .. (fun.returns or 'any')) - end + local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 - write(render_fun_sig(funname, params)) + for i, param in ipairs(params) do + local pname, ptype = param[1], param[2] + local optional = (pname ~= '...' and i > req_args) and '?' or '' + write(string.format('--- @param %s%s %s', pname, optional, ptype)) + end - return + if fun.returns ~= false then + write('--- @return ' .. (fun.returns or 'any')) end - print('no doc for', funname) + write(render_fun_sig(funname, params)) end ---- @type table<string,true> -local rendered_tags = {} - --- @param name string --- @param fun vim.EvalFn --- @param write fun(line: string) local function render_sig_and_tag(name, fun, write) + if not fun.signature then + return + end + local tags = { '*' .. name .. '()*' } if fun.tags then @@ -396,7 +449,7 @@ local function render_sig_and_tag(name, fun, write) local tag = table.concat(tags, ' ') local siglen = #fun.signature - local conceal_offset = 2*(#tags - 1) + local conceal_offset = 2 * (#tags - 1) local tag_pad_len = math.max(1, 80 - #tag + conceal_offset) if siglen + #tag > 80 then @@ -419,24 +472,17 @@ local function render_eval_doc(f, fun, write) return end - local desc = fun.desc - - if not desc then + if f:find('__%d+$') then write(fun.signature) - return + else + render_sig_and_tag(fun.name or f, fun, write) end - local name = fun.name or f - - if rendered_tags[name] then - write(fun.signature) - else - render_sig_and_tag(name, fun, write) - rendered_tags[name] = true + if not fun.desc then + return end - desc = vim.trim(desc) - local desc_l = split(desc) + local desc_l = split(vim.trim(fun.desc)) for _, l in ipairs(desc_l) do l = l:gsub('^ ', '') if vim.startswith(l, '<') and not l:match('^<[^ \t]+>') then @@ -473,7 +519,7 @@ local function render_option_default(d, vimdoc) end end - if dt == "" or dt == nil or type(dt) == 'function' then + if dt == '' or dt == nil or type(dt) == 'function' then dt = d.meta end @@ -481,22 +527,22 @@ local function render_option_default(d, vimdoc) if not vimdoc then v = vim.inspect(dt) --[[@as string]] else - v = type(dt) == 'string' and '"'..dt..'"' or tostring(dt) + v = type(dt) == 'string' and '"' .. dt .. '"' or tostring(dt) end --- @type table<string, string|false> local envvars = { TMPDIR = false, VIMRUNTIME = false, - XDG_CONFIG_HOME = vim.env.HOME..'/.local/config', - XDG_DATA_HOME = vim.env.HOME..'/.local/share', - XDG_STATE_HOME = vim.env.HOME..'/.local/state', + XDG_CONFIG_HOME = vim.env.HOME .. '/.local/config', + XDG_DATA_HOME = vim.env.HOME .. '/.local/share', + XDG_STATE_HOME = vim.env.HOME .. '/.local/state', } for name, default in pairs(envvars) do local value = vim.env[name] or default if value then - v = v:gsub(vim.pesc(value), '$'..name) + v = v:gsub(vim.pesc(value), '$' .. name) end end @@ -509,31 +555,55 @@ end local function render_option_meta(_f, opt, write) write('') for _, l in ipairs(split(norm_text(opt.desc))) do - write('--- '..l) + write('--- ' .. l) end - write('--- @type '..OPTION_TYPES[opt.type]) - write('vim.o.'..opt.full_name..' = '..render_option_default(opt.defaults)) + write('--- @type ' .. OPTION_TYPES[opt.type]) + write('vim.o.' .. opt.full_name .. ' = ' .. render_option_default(opt.defaults)) if opt.abbreviation then - write('vim.o.'..opt.abbreviation..' = vim.o.'..opt.full_name) + write('vim.o.' .. opt.abbreviation .. ' = vim.o.' .. opt.full_name) end for _, s in pairs { - {'wo', 'window'}, - {'bo', 'buffer'}, - {'go', 'global'}, + { 'wo', 'window' }, + { 'bo', 'buffer' }, + { 'go', 'global' }, } do local id, scope = s[1], s[2] if vim.list_contains(opt.scope, scope) or (id == 'go' and #opt.scope > 1) then - local pfx = 'vim.'..id..'.' - write(pfx..opt.full_name..' = vim.o.'..opt.full_name) + local pfx = 'vim.' .. id .. '.' + write(pfx .. opt.full_name .. ' = vim.o.' .. opt.full_name) if opt.abbreviation then - write(pfx..opt.abbreviation..' = '..pfx..opt.full_name) + write(pfx .. opt.abbreviation .. ' = ' .. pfx .. opt.full_name) end end end end +--- @param _f string +--- @param opt vim.option_meta +--- @param write fun(line: string) +local function render_vvar_meta(_f, opt, write) + write('') + + local desc = split(norm_text(opt.desc)) + while desc[#desc]:match('^%s*$') do + desc[#desc] = nil + end + + for _, l in ipairs(desc) do + write('--- ' .. l) + end + + write('--- @type ' .. (opt.type or 'any')) + + if LUA_KEYWORDS[opt.full_name] then + write("vim.v['" .. opt.full_name .. "'] = ...") + else + write('vim.v.' .. opt.full_name .. ' = ...') + end +end + --- @param s string[] --- @return string local function scope_to_doc(s) @@ -541,14 +611,14 @@ local function scope_to_doc(s) global = 'global', buffer = 'local to buffer', window = 'local to window', - tab = 'local to tab page' + tab = 'local to tab page', } if #s == 1 then return m[s[1]] end assert(s[1] == 'global') - return 'global or '..m[s[2]]..' |global-local|' + return 'global or ' .. m[s[2]] .. ' |global-local|' end -- @param o vim.option_meta @@ -574,6 +644,21 @@ local function scope_more_doc(o) return '' end +--- @param x string +--- @return string +local function dedent(x) + local xs = split(x) + local leading_ws = xs[1]:match('^%s*') --[[@as string]] + local leading_ws_pat = '^' .. leading_ws + + for i in ipairs(xs) do + local strip_pat = xs[i]:match(leading_ws_pat) and leading_ws_pat or '^%s*' + xs[i] = xs[i]:gsub(strip_pat, '') + end + + return table.concat(xs, '\n') +end + --- @return table<string,vim.option_meta> local function get_option_meta() local opts = require('src/nvim/options').options @@ -596,29 +681,41 @@ local function get_option_meta() return ret end +--- @return table<string,vim.option_meta> +local function get_vvar_meta() + local info = require('src/nvim/vvars').vars + local ret = {} --- @type table<string,vim.option_meta> + for name, o in pairs(info) do + o.desc = dedent(o.desc) + o.full_name = name + ret[name] = o + end + return ret +end + --- @param opt vim.option_meta --- @return string[] local function build_option_tags(opt) --- @type string[] local tags = { opt.full_name } - tags[#tags+1] = opt.abbreviation - if opt.type == 'bool' then + tags[#tags + 1] = opt.abbreviation + if opt.type == 'boolean' then for i = 1, #tags do - tags[#tags+1] = 'no'..tags[i] + tags[#tags + 1] = 'no' .. tags[i] end end for i, t in ipairs(tags) do - tags[i] = "'"..t.."'" + tags[i] = "'" .. t .. "'" end for _, t in ipairs(opt.tags or {}) do - tags[#tags+1] = t + tags[#tags + 1] = t end for i, t in ipairs(tags) do - tags[i] = "*"..t.."*" + tags[i] = '*' .. t .. '*' end return tags @@ -630,10 +727,10 @@ end local function render_option_doc(_f, opt, write) local tags = build_option_tags(opt) local tag_str = table.concat(tags, ' ') - local conceal_offset = 2*(#tags - 1) + local conceal_offset = 2 * (#tags - 1) local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) -- local pad = string.rep(' ', 80 - #tag_str + conceal_offset) - write(tag_pad..tag_str) + write(tag_pad .. tag_str) local name_str --- @type string if opt.abbreviation then @@ -642,26 +739,65 @@ local function render_option_doc(_f, opt, write) name_str = string.format("'%s'", opt.full_name) end - local otype = opt.type == 'bool' and 'boolean' or opt.type + local otype = opt.type == 'boolean' and 'boolean' or opt.type if opt.defaults.doc or opt.defaults.if_true ~= nil or opt.defaults.meta ~= nil then local v = render_option_default(opt.defaults, true) local pad = string.rep('\t', math.max(1, math.ceil((24 - #name_str) / 8))) if opt.defaults.doc then local deflen = #string.format('%s%s%s (', name_str, pad, otype) --- @type string - v = v:gsub('\n', '\n'..string.rep(' ', deflen - 2)) + v = v:gsub('\n', '\n' .. string.rep(' ', deflen - 2)) end write(string.format('%s%s%s\t(default %s)', name_str, pad, otype, v)) else write(string.format('%s\t%s', name_str, otype)) end - write('\t\t\t'..scope_to_doc(opt.scope)..scope_more_doc(opt)) + write('\t\t\t' .. scope_to_doc(opt.scope) .. scope_more_doc(opt)) for _, l in ipairs(split(opt.desc)) do if l == '<' or l:match('^<%s') then write(l) else - write('\t'..l:gsub('\\<', '<')) + write('\t' .. l:gsub('\\<', '<')) + end + end +end + +--- @param _f string +--- @param vvar vim.option_meta +--- @param write fun(line: string) +local function render_vvar_doc(_f, vvar, write) + local name = vvar.full_name + + local tags = { 'v:' .. name, name .. '-variable' } + if vvar.tags then + vim.list_extend(tags, vvar.tags) + end + + for i, t in ipairs(tags) do + tags[i] = '*' .. t .. '*' + end + + local tag_str = table.concat(tags, ' ') + local conceal_offset = 2 * (#tags - 1) + + local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) + write(tag_pad .. tag_str) + + local desc = split(vvar.desc) + + if (#desc == 1 or #desc == 2 and desc[2]:match('^%s*$')) and #name < 10 then + -- single line + write('v:' .. name .. '\t' .. desc[1]:gsub('^%s*', '')) + write('') + else + write('v:' .. name) + for _, l in ipairs(desc) do + if l == '<' or l:match('^<%s') then + write(l) + else + write('\t\t' .. l:gsub('\\<', '<')) + end end end end @@ -751,21 +887,37 @@ local CONFIG = { header = { '' }, from = 'A jump table for the options with a short description can be found at |Q_op|.', footer = { - ' vim:tw=78:ts=8:noet:ft=help:norl:' + ' vim:tw=78:ts=8:noet:ft=help:norl:', }, funcs = get_option_meta, render = render_option_doc, - } + }, + { + path = 'runtime/lua/vim/_meta/vvars.lua', + header = LUA_VVAR_META_HEADER, + funcs = get_vvar_meta, + render = render_vvar_meta, + }, + { + path = 'runtime/doc/vvars.txt', + header = { '' }, + from = 'Type |gO| to see the table of contents.', + footer = { + ' vim:tw=78:ts=8:noet:ft=help:norl:', + }, + funcs = get_vvar_meta, + render = render_vvar_doc, + }, } --- @param elem nvim.gen_eval_files.elem local function render(elem) - print('Rendering '..elem.path) - local from_lines = {} --- @type string[] + print('Rendering ' .. elem.path) + local from_lines = {} --- @type string[] local from = elem.from if from then for line in io.lines(elem.path) do - from_lines[#from_lines+1] = line + from_lines[#from_lines + 1] = line if line:match(from) then break end diff --git a/scripts/gen_filetype.lua b/scripts/gen_filetype.lua index 42478a1082..18b53f1ea4 100644 --- a/scripts/gen_filetype.lua +++ b/scripts/gen_filetype.lua @@ -8,18 +8,18 @@ if do_not_run then return end -local filetype_vim = "runtime/filetype.vim" -local filetype_lua = "runtime/lua/vim/filetype.lua" +local filetype_vim = 'runtime/filetype.vim' +local filetype_lua = 'runtime/lua/vim/filetype.lua' local keywords = { - ["for"] = true, - ["or"] = true, - ["and"] = true, - ["end"] = true, - ["do"] = true, - ["if"] = true, - ["while"] = true, - ["repeat"] = true, + ['for'] = true, + ['or'] = true, + ['and'] = true, + ['end'] = true, + ['do'] = true, + ['if'] = true, + ['while'] = true, + ['repeat'] = true, } local sections = { @@ -28,42 +28,42 @@ local sections = { pattern = { str = {}, func = {} }, } -local specialchars = "%*%?\\%$%[%]%{%}" +local specialchars = '%*%?\\%$%[%]%{%}' local function add_pattern(pat, ft) local ok = true -- Patterns that start or end with { or } confuse splitting on commas and make parsing harder, so just skip those - if not string.find(pat, "^%{") and not string.find(pat, "%}$") then - for part in string.gmatch(pat, "[^,]+") do - if not string.find(part, "[" .. specialchars .. "]") then - if type(ft) == "string" then + if not string.find(pat, '^%{') and not string.find(pat, '%}$') then + for part in string.gmatch(pat, '[^,]+') do + if not string.find(part, '[' .. specialchars .. ']') then + if type(ft) == 'string' then sections.filename.str[part] = ft else sections.filename.func[part] = ft end - elseif string.match(part, "^%*%.[^%./" .. specialchars .. "]+$") then - if type(ft) == "string" then + elseif string.match(part, '^%*%.[^%./' .. specialchars .. ']+$') then + if type(ft) == 'string' then sections.extension.str[part:sub(3)] = ft else sections.extension.func[part:sub(3)] = ft end else - if string.match(part, "^%*/[^" .. specialchars .. "]+$") then + if string.match(part, '^%*/[^' .. specialchars .. ']+$') then -- For patterns matching */some/pattern we want to easily match files -- with path /some/pattern, so include those in filename detection - if type(ft) == "string" then + if type(ft) == 'string' then sections.filename.str[part:sub(2)] = ft else sections.filename.func[part:sub(2)] = ft end end - if string.find(part, "^[%w-_.*?%[%]/]+$") then - local p = part:gsub("%.", "%%."):gsub("%*", ".*"):gsub("%?", ".") + if string.find(part, '^[%w-_.*?%[%]/]+$') then + local p = part:gsub('%.', '%%.'):gsub('%*', '.*'):gsub('%?', '.') -- Insert into array to maintain order rather than setting -- key-value directly - if type(ft) == "string" then + if type(ft) == 'string' then sections.pattern.str[p] = ft else sections.pattern.func[p] = ft @@ -80,14 +80,16 @@ end local function parse_line(line) local pat, ft - pat, ft = line:match("^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)") + pat, ft = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)') if pat then return add_pattern(pat, ft) else local func - pat, func = line:match("^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)") + pat, func = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)') if pat then - return add_pattern(pat, function() return func end) + return add_pattern(pat, function() + return func + end) end end end @@ -95,12 +97,12 @@ end local unparsed = {} local full_line for line in io.lines(filetype_vim) do - local cont = string.match(line, "^%s*\\%s*(.*)$") + local cont = string.match(line, '^%s*\\%s*(.*)$') if cont then - full_line = full_line .. " " .. cont + full_line = full_line .. ' ' .. cont else if full_line then - if not parse_line(full_line) and string.find(full_line, "^%s*au%a* Buf") then + if not parse_line(full_line) and string.find(full_line, '^%s*au%a* Buf') then table.insert(unparsed, full_line) end end @@ -109,40 +111,46 @@ for line in io.lines(filetype_vim) do end if #unparsed > 0 then - print("Failed to parse the following patterns:") + print('Failed to parse the following patterns:') for _, v in ipairs(unparsed) do print(v) end end local function add_item(indent, key, ft) - if type(ft) == "string" then - if string.find(key, "%A") or keywords[key] then - key = string.format("[\"%s\"]", key) + if type(ft) == 'string' then + if string.find(key, '%A') or keywords[key] then + key = string.format('["%s"]', key) end return string.format([[%s%s = "%s",]], indent, key, ft) - elseif type(ft) == "function" then + elseif type(ft) == 'function' then local func = ft() - if string.find(key, "%A") or keywords[key] then - key = string.format("[\"%s\"]", key) + if string.find(key, '%A') or keywords[key] then + key = string.format('["%s"]', key) end -- Right now only a single argument is supported, which covers -- everything in filetype.vim as of this writing - local arg = string.match(func, "%((.*)%)$") - func = string.gsub(func, "%(.*$", "") - if arg == "" then + local arg = string.match(func, '%((.*)%)$') + func = string.gsub(func, '%(.*$', '') + if arg == '' then -- Function with no arguments, call the function directly return string.format([[%s%s = function() vim.fn["%s"]() end,]], indent, key, func) elseif string.match(arg, [[^(["']).*%1$]]) then -- String argument - if func == "s:StarSetf" then + if func == 's:StarSetf' then return string.format([[%s%s = starsetf(%s),]], indent, key, arg) else return string.format([[%s%s = function() vim.fn["%s"](%s) end,]], indent, key, func, arg) end - elseif string.find(arg, "%(") then + elseif string.find(arg, '%(') then -- Function argument - return string.format([[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]], indent, key, func, arg) + return string.format( + [[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]], + indent, + key, + func, + arg + ) else assert(false, arg) end @@ -153,7 +161,7 @@ do local lines = {} local start = false for line in io.lines(filetype_lua) do - if line:match("^%s+-- END [A-Z]+$") then + if line:match('^%s+-- END [A-Z]+$') then start = false end @@ -161,14 +169,14 @@ do table.insert(lines, line) end - local indent, section = line:match("^(%s+)-- BEGIN ([A-Z]+)$") + local indent, section = line:match('^(%s+)-- BEGIN ([A-Z]+)$') if section then start = true local t = sections[string.lower(section)] local sorted = {} for k, v in pairs(t.str) do - table.insert(sorted, {[k] = v}) + table.insert(sorted, { [k] = v }) end table.sort(sorted, function(a, b) @@ -182,7 +190,7 @@ do sorted = {} for k, v in pairs(t.func) do - table.insert(sorted, {[k] = v}) + table.insert(sorted, { [k] = v }) end table.sort(sorted, function(a, b) @@ -195,7 +203,7 @@ do end end end - local f = io.open(filetype_lua, "w") - f:write(table.concat(lines, "\n") .. "\n") + local f = io.open(filetype_lua, 'w') + f:write(table.concat(lines, '\n') .. '\n') f:close() end diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 633207e018..43040151eb 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -2,32 +2,38 @@ -- -- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). -- +-- USAGE (For CI/local testing purposes): Simply `make lintdoc` or `scripts/lintdoc.lua`, which +-- basically does the following: +-- 1. :helptags ALL +-- 2. nvim -V1 -es +"lua require('scripts.gen_help_html').run_validate()" +q +-- 3. nvim -V1 -es +"lua require('scripts.gen_help_html').test_gen()" +q +-- -- USAGE (GENERATE HTML): --- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). --- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" +-- 1. `:helptags ALL` first; this script depends on vim.fn.taglist(). +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./runtime/doc', 'target/dir/')" +q -- - Read the docstring at gen(). -- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 -- 4. Visit http://localhost:4000/…/help.txt.html -- -- USAGE (VALIDATE): --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate()" +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate('./runtime/doc')" +q -- - validate() is 10x faster than gen(), so it is used in CI. -- -- SELF-TEST MODE: --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +q -- -- NOTES: --- * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too --- slow (~1 min) to run in per-commit CI. +-- * gen() and validate() are the primary (programmatic) entrypoints. validate() only exists +-- because gen() is too slow (~1 min) to run in per-commit CI. -- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. -- * visit_validate() is the core function used by validate(). -- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. -local tagmap = nil -local helpfiles = nil -local invalid_links = {} -local invalid_urls = {} -local invalid_spelling = {} +local tagmap = nil ---@type table<string, string> +local helpfiles = nil ---@type string[] +local invalid_links = {} ---@type table<string, any> +local invalid_urls = {} ---@type table<string, any> +local invalid_spelling = {} ---@type table<string, table<string, string>> local spell_dict = { Neovim = 'Nvim', NeoVim = 'Nvim', @@ -36,6 +42,14 @@ local spell_dict = { VimL = 'Vimscript', vimL = 'Vimscript', viml = 'Vimscript', + ['tree-sitter'] = 'treesitter', + ['Tree-sitter'] = 'Treesitter', +} +--- specify the list of keywords to ignore (i.e. allow), or true to disable spell check completely. +--- @type table<string, true|string[]> +local spell_ignore_files = { + ['backers.txt'] = true, + ['news.txt'] = { 'tree-sitter' }, -- in news, may refer to the upstream "tree-sitter" library } local language = nil @@ -60,31 +74,33 @@ local new_layout = { -- TODO: These known invalid |links| require an update to the relevant docs. local exclude_invalid = { - ["'string'"] = "eval.txt", + ["'string'"] = 'eval.txt', Query = 'treesitter.txt', matchit = 'vim_diff.txt', - ["set!"] = "treesitter.txt", + ['set!'] = 'treesitter.txt', } -- False-positive "invalid URLs". local exclude_invalid_urls = { - ["http://"] = "usr_23.txt", - ["http://."] = "usr_23.txt", - ["http://aspell.net/man-html/Affix-Compression.html"] = "spell.txt", - ["http://aspell.net/man-html/Phonetic-Code.html"] = "spell.txt", - ["http://canna.sourceforge.jp/"] = "mbyte.txt", - ["http://gnuada.sourceforge.net"] = "ft_ada.txt", - ["http://lua-users.org/wiki/StringLibraryTutorial"] = "lua.txt", - ["http://michael.toren.net/code/"] = "pi_tar.txt", - ["http://papp.plan9.de"] = "syntax.txt", - ["http://wiki.services.openoffice.org/wiki/Dictionaries"] = "spell.txt", - ["http://www.adapower.com"] = "ft_ada.txt", - ["http://www.jclark.com/"] = "quickfix.txt", + ['http://'] = 'usr_23.txt', + ['http://.'] = 'usr_23.txt', + ['http://aspell.net/man-html/Affix-Compression.html'] = 'spell.txt', + ['http://aspell.net/man-html/Phonetic-Code.html'] = 'spell.txt', + ['http://canna.sourceforge.jp/'] = 'mbyte.txt', + ['http://gnuada.sourceforge.net'] = 'ft_ada.txt', + ['http://lua-users.org/wiki/StringLibraryTutorial'] = 'lua.txt', + ['http://michael.toren.net/code/'] = 'pi_tar.txt', + ['http://papp.plan9.de'] = 'syntax.txt', + ['http://wiki.services.openoffice.org/wiki/Dictionaries'] = 'spell.txt', + ['http://www.adapower.com'] = 'ft_ada.txt', + ['http://www.jclark.com/'] = 'quickfix.txt', + ['http://oldblog.antirez.com/post/redis-and-scripting.html'] = 'faq.txt', } -- Deprecated, brain-damaged files that I don't care about. local ignore_errors = { ['pi_netrw.txt'] = true, + ['backers.txt'] = true, } local function tofile(fname, text) @@ -97,24 +113,24 @@ local function tofile(fname, text) end end +---@type fun(s: string): string local function html_esc(s) - return s:gsub( - '&', '&'):gsub( - '<', '<'):gsub( - '>', '>') + return (s:gsub('&', '&'):gsub('<', '<'):gsub('>', '>')) end local function url_encode(s) -- Credit: tpope / vim-unimpaired -- NOTE: these chars intentionally *not* escaped: ' ( ) - return vim.fn.substitute(vim.fn.iconv(s, 'latin1', 'utf-8'), + return vim.fn.substitute( + vim.fn.iconv(s, 'latin1', 'utf-8'), [=[[^A-Za-z0-9()'_.~-]]=], [=[\="%".printf("%02X",char2nr(submatch(0)))]=], - 'g') + 'g' + ) end local function expandtabs(s) - return s:gsub('\t', (' '):rep(8)) + return s:gsub('\t', (' '):rep(8)) --[[ @as string ]] end local function to_titlecase(s) @@ -131,13 +147,14 @@ local function to_heading_tag(text) end local function basename_noext(f) - return vim.fs.basename(f:gsub('%.txt', '')) + return vim.fs.basename(f:gsub('%.txt', '')) end local function is_blank(s) return not not s:find([[^[\t ]*$]]) end +---@type fun(s: string, dir?:0|1|2): string local function trim(s, dir) return vim.fn.trim(s, '\r\t\n ', dir or 0) end @@ -146,12 +163,13 @@ end --- --- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc --- ---- @returns (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. +--- @param url string +--- @return string, string (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. local function fix_url(url) local removed_chars = '' local fixed_url = url -- Remove up to one of each char from end of the URL, in this order. - for _, c in ipairs({ '.', ')', }) do + for _, c in ipairs({ '.', ')' }) do if fixed_url:sub(-1) == c then removed_chars = c .. removed_chars fixed_url = fixed_url:sub(1, -2) @@ -162,7 +180,7 @@ end --- Checks if a given line is a "noise" line that doesn't look good in HTML form. local function is_noise(line, noise_lines) - if ( + if -- First line is always noise. (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0) or line:find('Type .*gO.* to see the table of contents') @@ -177,7 +195,7 @@ local function is_noise(line, noise_lines) or line:find('^%s*vim?%:.*ft=help') or line:find('^%s*vim?%:.*filetype=help') or line:find('[*>]local%-additions[*<]') - ) then + then -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) table.insert(noise_lines or {}, line) return true @@ -188,28 +206,32 @@ end --- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. local function get_bug_url_vimdoc(fname, to_fname, sample_text) local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ('https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' - ..vim.fs.basename(fname) - ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' - ..this_url - ..'%0D%0DContext%3A%0D%0D%60%60%60%0D' - ..url_encode(sample_text) - ..'%0D%60%60%60') + local bug_url = ( + 'https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + .. vim.fs.basename(fname) + .. '+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' + .. this_url + .. '%0D%0DContext%3A%0D%0D%60%60%60%0D' + .. url_encode(sample_text) + .. '%0D%60%60%60' + ) return bug_url end --- Creates a github issue URL at neovim/neovim with prefilled content. local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ('https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' - ..vim.fs.basename(fname) - ..'+&body=%60gen_help_html.lua%60+problem+at%3A+' - ..this_url - ..'%0D' - ..(token_name and '+unhandled+token%3A+%60'..token_name..'%60' or '') - ..'%0DContext%3A%0D%0D%60%60%60%0D' - ..url_encode(sample_text) - ..'%0D%60%60%60') + local bug_url = ( + 'https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' + .. vim.fs.basename(fname) + .. '+&body=%60gen_help_html.lua%60+problem+at%3A+' + .. this_url + .. '%0D' + .. (token_name and '+unhandled+token%3A+%60' .. token_name .. '%60' or '') + .. '%0DContext%3A%0D%0D%60%60%60%0D' + .. url_encode(sample_text) + .. '%0D%60%60%60' + ) return bug_url end @@ -254,6 +276,9 @@ local function trim_indent(s) end --- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. +---@param node TSNode +---@param bufnr integer +---@param offset integer local function getbuflinestr(node, bufnr, offset) local line1, _, line2, _ = node:range() line1 = line1 - offset @@ -264,8 +289,12 @@ end --- Gets the whitespace just before `node` from the raw buffer text. --- Needed for preformatted `old` lines. +---@param node TSNode +---@param bufnr integer +---@return string local function getws(node, bufnr) local line1, c1, line2, _ = node:range() + ---@type string local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] local text_before = raw:sub(1, c1) local leading_ws = text_before:match('%s+$') or '' @@ -274,9 +303,11 @@ end local function get_tagname(node, bufnr) local text = vim.treesitter.get_node_text(node, bufnr) - local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(text) or text - local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" - local helppage = get_helppage(helpfile) -- "api.html" + local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') + and ("'%s'"):format(text) + or text + local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" return helppage, tag end @@ -295,16 +326,15 @@ local function ignore_parse_error(fname, s) if ignore_errors[vim.fs.basename(fname)] then return true end - return ( - -- Ignore parse errors for unclosed tag. - -- This is common in vimdocs and is treated as plaintext by :help. - s:find("^[`'|*]") - ) + -- Ignore parse errors for unclosed tag. + -- This is common in vimdocs and is treated as plaintext by :help. + return s:find("^[`'|*]") end +---@param node TSNode local function has_ancestor(node, ancestor_name) - local p = node - while true do + local p = node ---@type TSNode? + while p do p = p:parent() if not p or p:type() == 'help_file' then break @@ -316,6 +346,7 @@ local function has_ancestor(node, ancestor_name) end --- Gets the first matching child node matching `name`. +---@param node TSNode local function first(node, name) for c, _ in node:iter_children() do if c:named() and c:type() == name then @@ -349,6 +380,11 @@ local function validate_url(text, fname) end --- Traverses the tree at `root` and checks that |tag| links point to valid helptags. +---@param root TSNode +---@param level integer +---@param lang_tree TSTree +---@param opt table +---@param stats table local function visit_validate(root, level, lang_tree, opt, stats) level = level or 0 local node_name = (root.named and root:named()) and root:type() or nil @@ -377,13 +413,24 @@ local function visit_validate(root, level, lang_tree, opt, stats) -- Flatten the sample text to a single, truncated line. sample_text = vim.trim(sample_text):gsub('[\t\n]', ' '):sub(1, 80) table.insert(stats.parse_errors, sample_text) - elseif (node_name == 'word' or node_name == 'uppercase_name') - and (not vim.tbl_contains({'codespan', 'taglink', 'tag'}, parent)) + elseif + (node_name == 'word' or node_name == 'uppercase_name') + and (not vim.tbl_contains({ 'codespan', 'taglink', 'tag' }, parent)) then - local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. + local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. + local fname_basename = assert(vim.fs.basename(opt.fname)) if spell_dict[text_nopunct] then - invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} - invalid_spelling[text_nopunct][vim.fs.basename(opt.fname)] = node_text(root:parent()) + local should_ignore = ( + spell_ignore_files[fname_basename] == true + or vim.tbl_contains( + (spell_ignore_files[fname_basename] or {}) --[[ @as string[] ]], + text_nopunct + ) + ) + if not should_ignore then + invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} + invalid_spelling[text_nopunct][fname_basename] = node_text(root:parent()) + end end elseif node_name == 'url' then local fixed_url, _ = fix_url(trim(text)) @@ -395,29 +442,47 @@ end -- Fix tab alignment issues caused by concealed characters like |, `, * in tags -- and code blocks. +---@param text string +---@param next_node_text string local function fix_tab_after_conceal(text, next_node_text) -- Vim tabs take into account the two concealed characters even though they -- are invisible, so we need to add back in the two spaces if this is -- followed by a tab to make the tab alignment to match Vim's behavior. - if string.sub(next_node_text,1,1) == '\t' then + if string.sub(next_node_text, 1, 1) == '\t' then text = text .. ' ' end return text end +---@class (exact) nvim.gen_help_html.heading +---@field name string +---@field subheadings nvim.gen_help_html.heading[] +---@field tag string + -- Generates HTML from node `root` recursively. +---@param root TSNode +---@param level integer +---@param lang_tree TSTree +---@param headings nvim.gen_help_html.heading[] +---@param opt table +---@param stats table local function visit_node(root, level, lang_tree, headings, opt, stats) level = level or 0 local node_name = (root.named and root:named()) and root:type() or nil -- Previous sibling kind (string). - local prev = root:prev_sibling() and (root:prev_sibling().named and root:prev_sibling():named()) and root:prev_sibling():type() or nil + local prev = root:prev_sibling() + and (root:prev_sibling().named and root:prev_sibling():named()) + and root:prev_sibling():type() + or nil -- Next sibling kind (string). - local next_ = root:next_sibling() and (root:next_sibling().named and root:next_sibling():named()) and root:next_sibling():type() or nil + local next_ = root:next_sibling() + and (root:next_sibling().named and root:next_sibling():named()) + and root:next_sibling():type() + or nil -- Parent kind (string). local parent = root:parent() and root:parent():type() or nil local text = '' - local trimmed -- Gets leading whitespace of `node`. local function ws(node) node = node or root @@ -435,6 +500,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) end + local trimmed ---@type string if root:named_child_count() == 0 or node_name == 'ERROR' then text = node_text() trimmed = html_esc(trim(text)) @@ -450,7 +516,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) trimmed = trim(text) end - if node_name == 'help_file' then -- root node + if node_name == 'help_file' then -- root node return text elseif node_name == 'url' then local fixed_url, removed_chars = fix_url(trimmed) @@ -459,18 +525,24 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return text elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then if is_noise(text, stats.noise_lines) then - return '' -- Discard common "noise" lines. + return '' -- Discard common "noise" lines. end -- Remove "===" and tags from ToC text. local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) -- Use the first *tag* node as the heading anchor, if any. local tagnode = first(root, 'tag') -- Use the *tag* as the heading anchor id, if possible. - local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname) + local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) + or to_heading_tag(hname) if node_name == 'h1' or #headings == 0 then - table.insert(headings, { name = hname, subheadings = {}, tag = tagname }) + ---@type nvim.gen_help_html.heading + local heading = { name = hname, subheadings = {}, tag = tagname } + headings[#headings + 1] = heading else - table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, tag = tagname }) + table.insert( + headings[#headings].subheadings, + { name = hname, subheadings = {}, tag = tagname } + ) end local el = node_name == 'h1' and 'h2' or 'h3' return ('<%s id="%s" class="help-heading">%s</%s>\n'):format(el, tagname, text, el) @@ -490,11 +562,16 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end return string.format('<div class="help-para">\n%s\n</div>\n', text) elseif node_name == 'line' then - if (parent ~= 'codeblock' or parent ~= 'code') and (is_blank(text) or is_noise(text, stats.noise_lines)) then - return '' -- Discard common "noise" lines. + if + (parent ~= 'codeblock' or parent ~= 'code') + and (is_blank(text) or is_noise(text, stats.noise_lines)) + then + return '' -- Discard common "noise" lines. end -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. - local div = opt.old and root:child(0) and vim.list_contains({'column_heading', 'h1', 'h2', 'h3'}, root:child(0):type()) + local div = opt.old + and root:child(0) + and vim.list_contains({ 'column_heading', 'h1', 'h2', 'h3' }, root:child(0):type()) return string.format('%s%s', div and trim(text) or text, div and '' or '\n') elseif node_name == 'line_li' then local sib = root:prev_sibling() @@ -520,12 +597,17 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if ignored then return text end - local s = ('%s<a href="%s#%s">%s</a>'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) + local s = ('%s<a href="%s#%s">%s</a>'):format( + ws(), + helppage, + url_encode(tagname), + html_esc(tagname) + ) if opt.old and node_name == 'taglink' then s = fix_tab_after_conceal(s, node_text(root:next_sibling())) end return s - elseif vim.list_contains({'codespan', 'keycode'}, node_name) then + elseif vim.list_contains({ 'codespan', 'keycode' }, node_name) then if root:has_error() then return text end @@ -541,24 +623,28 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) elseif node_name == 'language' then language = node_text(root) return '' - elseif node_name == 'code' then -- Highlighted codeblock (child). + elseif node_name == 'code' then -- Highlighted codeblock (child). if is_blank(text) then return '' end - local code + local code ---@type string if language then - code = ('<pre><code class="language-%s">%s</code></pre>'):format(language,trim(trim_indent(text), 2)) + code = ('<pre><code class="language-%s">%s</code></pre>'):format( + language, + trim(trim_indent(text), 2) + ) language = nil else code = ('<pre>%s</pre>'):format(trim(trim_indent(text), 2)) end return code - elseif node_name == 'tag' then -- anchor + elseif node_name == 'tag' then -- anchor if root:has_error() then return text end - local in_heading = vim.list_contains({'h1', 'h2', 'h3'}, parent) - local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' + local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent) + local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' + or 'help-tag' local tagname = node_text(root:child(1), false) if vim.tbl_count(stats.first_tags) < 2 then -- Force the first 2 tags in the doc to be anchored at the main heading. @@ -567,14 +653,29 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end local el = in_heading and 'span' or 'code' local encoded_tagname = url_encode(tagname) - local s = ('%s<%s id="%s" class="%s"><a href="#%s">%s</a></%s>'):format(ws(), el, encoded_tagname, cssclass, encoded_tagname, trimmed, el) + local s = ('%s<%s id="%s" class="%s"><a href="#%s">%s</a></%s>'):format( + ws(), + el, + encoded_tagname, + cssclass, + encoded_tagname, + trimmed, + el + ) if opt.old then - s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) end if in_heading and prev ~= 'tag' then -- Don't set "id", let the heading use the tag as its "id" (used by search engines). - s = ('%s<%s class="%s"><a href="#%s">%s</a></%s>'):format(ws(), el, cssclass, encoded_tagname, trimmed, el) + s = ('%s<%s class="%s"><a href="#%s">%s</a></%s>'):format( + ws(), + el, + cssclass, + encoded_tagname, + trimmed, + el + ) -- Start the <span> container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. -- <h2>foo bar<span>tag1 tag2</span></h2> @@ -593,21 +694,31 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) return ('<a class="parse-error" target="_blank" title="Report bug... (parse error)" href="%s">%s</a>'):format( - get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) - else -- Unknown token. + get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), + trimmed + ) + else -- Unknown token. local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' return ('<a class="unknown-token" target="_blank" title="Report bug... (unhandled token "%s")" href="%s">%s</a>'):format( - node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) + node_name, + get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), + trimmed + ), + ('unknown-token:"%s"'):format(node_name) end end -local function get_helpfiles(include) - local dir = './build/runtime/doc' +--- @param dir string e.g. '$VIMRUNTIME/doc' +--- @param include string[]|nil +--- @return string[] +local function get_helpfiles(dir, include) local rv = {} for f, type in vim.fs.dir(dir) do - if (vim.endswith(f, '.txt') - and type == 'file' - and (not include or vim.list_contains(include, f))) then + if + vim.endswith(f, '.txt') + and type == 'file' + and (not include or vim.list_contains(include, f)) + then local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') table.insert(rv, fullpath) end @@ -633,7 +744,7 @@ end --- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system. local function ensure_runtimepath() if not vim.o.runtimepath:find('build/lib/nvim/') then - vim.cmd[[set runtimepath^=./build/lib/nvim/]] + vim.cmd [[set runtimepath^=./build/lib/nvim/]] end end @@ -641,15 +752,17 @@ end --- --- @param fname string help file to parse --- @param parser_path string? path to non-default vimdoc.so ---- @returns lang_tree, bufnr +--- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr) local function parse_buf(fname, parser_path) - local buf + local buf ---@type integer if type(fname) == 'string' then - vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. + vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename. buf = vim.api.nvim_get_current_buf() else + -- Left for debugging + ---@diagnostic disable-next-line: no-unknown buf = fname - vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. + vim.cmd('sbuffer ' .. tostring(fname)) -- Buffer number. end if parser_path then vim.treesitter.language.add('vimdoc', { path = parser_path }) @@ -664,14 +777,14 @@ end --- --- @param fname string help file to validate --- @param parser_path string? path to non-default vimdoc.so ---- @returns { invalid_links: number, parse_errors: string[] } +--- @return { invalid_links: number, parse_errors: string[] } local function validate_one(fname, parser_path) local stats = { parse_errors = {}, } local lang_tree, buf = parse_buf(fname, parser_path) for _, tree in ipairs(lang_tree:trees()) do - visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname, }, stats) + visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats) end lang_tree:destroy() vim.cmd.close() @@ -685,15 +798,17 @@ end --- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) --- @param parser_path string? path to non-default vimdoc.so --- ---- @returns html, stats +--- @return string html +--- @return table stats local function gen_one(fname, to_fname, old, commit, parser_path) local stats = { noise_lines = {}, parse_errors = {}, - first_tags = {}, -- Track the first few tags in doc. + first_tags = {}, -- Track the first few tags in doc. } local lang_tree, buf = parse_buf(fname, parser_path) - local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. + ---@type nvim.gen_help_html.heading[] + local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. local title = to_titlecase(basename_noext(fname)) local html = ([[ @@ -777,9 +892,17 @@ local function gen_one(fname, to_fname, old, commit, parser_path) local main = '' for _, tree in ipairs(lang_tree:trees()) do - main = main .. (visit_node(tree:root(), 0, tree, headings, - { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1, }, - stats)) + main = main + .. ( + visit_node( + tree:root(), + 0, + tree, + headings, + { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1 }, + stats + ) + ) end main = ([[ @@ -809,8 +932,16 @@ local function gen_one(fname, to_fname, old, commit, parser_path) <hr/> %s </div> - ]]):format(logo_svg, stats.first_tags[2] or '', stats.first_tags[1] or '', title, vim.fs.basename(fname), main) + ]]):format( + logo_svg, + stats.first_tags[2] or '', + stats.first_tags[1] or '', + title, + vim.fs.basename(fname), + main + ) + ---@type string local toc = [[ <div class="col-narrow toc"> <div><a href="index.html">Main</a></div> @@ -819,13 +950,17 @@ local function gen_one(fname, to_fname, old, commit, parser_path) <hr/> ]] - local n = 0 -- Count of all headings + subheadings. - for _, h1 in ipairs(headings) do n = n + 1 + #h1.subheadings end + local n = 0 -- Count of all headings + subheadings. for _, h1 in ipairs(headings) do + n = n + 1 + #h1.subheadings + end + for _, h1 in ipairs(headings) do + ---@type string toc = toc .. ('<div class="help-toc-h1"><a href="#%s">%s</a>\n'):format(h1.tag, h1.name) - if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. + if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. for _, h2 in ipairs(h1.subheadings) do - toc = toc .. ('<div class="help-toc-h2"><a href="#%s">%s</a></div>\n'):format(h2.tag, h2.name) + toc = toc + .. ('<div class="help-toc-h2"><a href="#%s">%s</a></div>\n'):format(h2.tag, h2.name) end end toc = toc .. '</div>' @@ -859,11 +994,16 @@ local function gen_one(fname, to_fname, old, commit, parser_path) </footer> ]]):format( - os.date('%Y-%m-%d %H:%M'), commit, commit:sub(1, 7), #stats.parse_errors, bug_link, - html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines) + os.date('%Y-%m-%d %H:%M'), + commit, + commit:sub(1, 7), + #stats.parse_errors, + bug_link, + html_esc(table.concat(stats.noise_lines, '\n')), + #stats.noise_lines + ) - html = ('%s%s%s</div>\n%s</body>\n</html>\n'):format( - html, main, toc, footer) + html = ('%s%s%s</div>\n%s</body>\n</html>\n'):format(html, main, toc, footer) vim.cmd('q!') lang_tree:destroy() return html, stats @@ -1033,24 +1173,41 @@ local function gen_css(fname) tofile(fname, css) end -function M._test() - tagmap = get_helptags('./build/runtime/doc') - helpfiles = get_helpfiles() +-- Testing - local function ok(cond, expected, actual) - assert((not expected and not actual) or (expected and actual), 'if "expected" is given, "actual" is also required') - if expected then - return assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual))) - else - return assert(cond) - end - end - local function eq(expected, actual) - return ok(expected == actual, expected, actual) +local function ok(cond, expected, actual, message) + assert( + (not expected and not actual) or (expected and actual), + 'if "expected" is given, "actual" is also required' + ) + if expected then + assert( + cond, + ('%sexpected %s, got: %s'):format( + message and (message .. '\n') or '', + vim.inspect(expected), + vim.inspect(actual) + ) + ) + return cond + else + return assert(cond) end +end +local function eq(expected, actual, message) + return ok(vim.deep_equal(expected, actual), expected, actual, message) +end + +function M._test() + tagmap = get_helptags('$VIMRUNTIME/doc') + helpfiles = get_helpfiles(vim.fn.expand('$VIMRUNTIME/doc')) ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) - ok(vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), tagmap['vim.diagnostic.set()'], 'diagnostic.txt') + ok( + vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), + tagmap['vim.diagnostic.set()'], + 'diagnostic.txt' + ) ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt') ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]])) ok(is_noise([[ NVIM REFERENCE MANUAL by Thiago de Arruda ]])) @@ -1060,7 +1217,10 @@ function M._test() eq(1, get_indent(' a')) eq(2, get_indent(' a\n b\n c\n')) eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) - eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + eq( + 'a\n \n b\n c\n d\n e\n', + trim_indent(' a\n \n b\n c\n d\n e\n') + ) local fixed_url, removed_chars = fix_url('https://example.com).') eq('https://example.com', fixed_url) @@ -1078,33 +1238,50 @@ function M._test() eq('https://example.com', fixed_url) eq('', removed_chars) - print('all tests passed') + print('all tests passed.\n') end +--- @class nvim.gen_help_html.gen_result +--- @field helpfiles string[] list of generated HTML files, from the source docs {include} +--- @field err_count integer number of parse errors in :help docs +--- @field invalid_links table<string, any> + --- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`. --- --- Example: --- ---- gen('./build/runtime/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) +--- gen('$VIMRUNTIME/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) --- --- @param help_dir string Source directory containing the :help files. Must run `make helptags` first. --- @param to_dir string Target directory where the .html files will be written. ---- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} +--- @param include string[]|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} --- ---- @returns info dict +--- @return nvim.gen_help_html.gen_result result function M.gen(help_dir, to_dir, include, commit, parser_path) - vim.validate{ - help_dir={help_dir, function(d) return vim.fn.isdirectory(vim.fn.expand(d)) == 1 end, 'valid directory'}, - to_dir={to_dir, 's'}, - include={include, 't', true}, - commit={commit, 's', true}, - parser_path={parser_path, function(f) return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 end, 'valid vimdoc.{so,dll} filepath'}, + vim.validate { + help_dir = { + help_dir, + function(d) + return vim.fn.isdirectory(vim.fn.expand(d)) == 1 + end, + 'valid directory', + }, + to_dir = { to_dir, 's' }, + include = { include, 't', true }, + commit = { commit, 's', true }, + parser_path = { + parser_path, + function(f) + return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 + end, + 'valid vimdoc.{so,dll} filepath', + }, } local err_count = 0 ensure_runtimepath() tagmap = get_helptags(vim.fn.expand(help_dir)) - helpfiles = get_helpfiles(include) + helpfiles = get_helpfiles(help_dir, include) to_dir = vim.fn.expand(to_dir) parser_path = parser_path and vim.fn.expand(parser_path) or nil @@ -1117,13 +1294,20 @@ function M.gen(help_dir, to_dir, include, commit, parser_path) local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?', parser_path) tofile(to_fname, html) - print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) + print( + ('generated (%-4s errors): %-15s => %s'):format( + #stats.parse_errors, + helpfile, + vim.fs.basename(to_fname) + ) + ) err_count = err_count + #stats.parse_errors end print(('generated %d html pages'):format(#helpfiles)) print(('total errors: %d'):format(err_count)) print(('invalid tags:\n%s'):format(vim.inspect(invalid_links))) + --- @type nvim.gen_help_html.gen_result return { helpfiles = helpfiles, err_count = err_count, @@ -1131,45 +1315,118 @@ function M.gen(help_dir, to_dir, include, commit, parser_path) } end --- Validates all :help files found in `help_dir`: --- - checks that |tag| links point to valid helptags. --- - recursively counts parse errors ("ERROR" nodes) --- --- This is 10x faster than gen(), for use in CI. --- --- @returns results dict +--- @class nvim.gen_help_html.validate_result +--- @field helpfiles integer number of generated helpfiles +--- @field err_count integer number of parse errors +--- @field parse_errors table<string, string[]> +--- @field invalid_links table<string, any> invalid tags in :help docs +--- @field invalid_urls table<string, any> invalid URLs in :help docs +--- @field invalid_spelling table<string, table<string, string>> invalid spelling in :help docs + +--- Validates all :help files found in `help_dir`: +--- - checks that |tag| links point to valid helptags. +--- - recursively counts parse errors ("ERROR" nodes) +--- +--- This is 10x faster than gen(), for use in CI. +--- +--- @return nvim.gen_help_html.validate_result result function M.validate(help_dir, include, parser_path) - vim.validate{ - help_dir={help_dir, function(d) return vim.fn.isdirectory(vim.fn.expand(d)) == 1 end, 'valid directory'}, - include={include, 't', true}, - parser_path={parser_path, function(f) return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 end, 'valid vimdoc.{so,dll} filepath'}, + vim.validate { + help_dir = { + help_dir, + function(d) + return vim.fn.isdirectory(vim.fn.expand(d)) == 1 + end, + 'valid directory', + }, + include = { include, 't', true }, + parser_path = { + parser_path, + function(f) + return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 + end, + 'valid vimdoc.{so,dll} filepath', + }, } - local err_count = 0 - local files_to_errors = {} + local err_count = 0 ---@type integer + local files_to_errors = {} ---@type table<string, string[]> ensure_runtimepath() tagmap = get_helptags(vim.fn.expand(help_dir)) - helpfiles = get_helpfiles(include) + helpfiles = get_helpfiles(help_dir, include) parser_path = parser_path and vim.fn.expand(parser_path) or nil for _, f in ipairs(helpfiles) do - local helpfile = vim.fs.basename(f) + local helpfile = assert(vim.fs.basename(f)) local rv = validate_one(f, parser_path) print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) if #rv.parse_errors > 0 then files_to_errors[helpfile] = rv.parse_errors - vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v) return s..'\n '..v end))) + vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v) + return s .. '\n ' .. v + end))) end err_count = err_count + #rv.parse_errors end + ---@type nvim.gen_help_html.validate_result return { helpfiles = #helpfiles, err_count = err_count, + parse_errors = files_to_errors, invalid_links = invalid_links, invalid_urls = invalid_urls, invalid_spelling = invalid_spelling, - parse_errors = files_to_errors, } end +--- Validates vimdoc files on $VIMRUNTIME. and print human-readable error messages if fails. +--- +--- If this fails, try these steps (in order): +--- 1. Fix/cleanup the :help docs. +--- 2. Fix the parser: https://github.com/neovim/tree-sitter-vimdoc +--- 3. File a parser bug, and adjust the tolerance of this test in the meantime. +--- +--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc' +function M.run_validate(help_dir) + help_dir = vim.fn.expand(help_dir or '$VIMRUNTIME/doc') + print('doc path = ' .. vim.uv.fs_realpath(help_dir)) + + local rv = M.validate(help_dir) + + -- Check that we actually found helpfiles. + ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles) + + eq({}, rv.parse_errors, 'no parse errors') + eq(0, rv.err_count, 'no parse errors') + eq({}, rv.invalid_links, 'invalid tags in :help docs') + eq({}, rv.invalid_urls, 'invalid URLs in :help docs') + eq( + {}, + rv.invalid_spelling, + 'invalid spelling in :help docs (see spell_dict in scripts/gen_help_html.lua)' + ) +end + +--- Test-generates HTML from docs. +--- +--- 1. Test that gen_help_html.lua actually works. +--- 2. Test that parse errors did not increase wildly. Because we explicitly test only a few +--- :help files, we can be precise about the tolerances here. +--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc' +function M.test_gen(help_dir) + local tmpdir = assert(vim.fs.dirname(vim.fn.tempname())) + help_dir = vim.fn.expand(help_dir or '$VIMRUNTIME/doc') + print('doc path = ' .. vim.uv.fs_realpath(help_dir)) + + local rv = M.gen( + help_dir, + tmpdir, + -- Because gen() is slow (~30s), this test is limited to a few files. + { 'pi_health.txt', 'help.txt', 'index.txt', 'nvim.txt' } + ) + eq(4, #rv.helpfiles) + eq(0, rv.err_count, 'parse errors in :help docs') + eq({}, rv.invalid_links, 'invalid tags in :help docs') +end + return M diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua index 6ff8dcb3f4..19fad7bab4 100644 --- a/scripts/gen_lsp.lua +++ b/scripts/gen_lsp.lua @@ -1,11 +1,15 @@ ---[[ -Generates lua-ls annotations for lsp +-- Generates lua-ls annotations for lsp. + +local USAGE = [[ +Generates lua-ls annotations for lsp. + USAGE: -nvim -l scripts/gen_lsp.lua gen # this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua -nvim -l scripts/gen_lsp.lua gen --version 3.18 --build/new_lsp_types.lua +nvim -l scripts/gen_lsp.lua gen # by default, this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua nvim -l scripts/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua nvim -l scripts/gen_lsp.lua gen --version 3.18 --methods ---]] +]] + +local DEFAULT_LSP_VERSION = '3.18' local M = {} @@ -14,15 +18,28 @@ local function tofile(fname, text) if not f then error(('failed to write: %s'):format(f)) else + print(('Written to: %s'):format(fname)) f:write(text) f:close() end end +--- The LSP protocol JSON data (it's partial, non-exhaustive). +--- https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/3.18/metaModel/metaModel.schema.json +--- @class vim._gen_lsp.Protocol +--- @field requests vim._gen_lsp.Request[] +--- @field notifications vim._gen_lsp.Notification[] +--- @field structures vim._gen_lsp.Structure[] +--- @field enumerations vim._gen_lsp.Enumeration[] +--- @field typeAliases vim._gen_lsp.TypeAlias[] + +---@param opt vim._gen_lsp.opt +---@return vim._gen_lsp.Protocol local function read_json(opt) local uri = 'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/' .. opt.version .. '/metaModel/metaModel.json' + print('Reading ' .. uri) local res = vim.system({ 'curl', '--no-progress-meter', uri, '-o', '-' }):wait() if res.code ~= 0 or (res.stdout or ''):len() < 999 then @@ -34,11 +51,12 @@ local function read_json(opt) end -- Gets the Lua symbol for a given fully-qualified LSP method name. -local function name(s) +local function to_luaname(s) -- "$/" prefix is special: https://microsoft.github.io/language-server-protocol/specification/#dollarRequests return s:gsub('^%$', 'dollar'):gsub('/', '_') end +---@param protocol vim._gen_lsp.Protocol local function gen_methods(protocol) local output = { '-- Generated by gen_lsp.lua, keep at end of file.', @@ -49,9 +67,35 @@ local function gen_methods(protocol) } local indent = (' '):rep(2) + --- @class vim._gen_lsp.Request + --- @field deprecated? string + --- @field documentation? string + --- @field messageDirection string + --- @field method string + --- @field params? any + --- @field proposed? boolean + --- @field registrationMethod? string + --- @field registrationOptions? any + --- @field since? string + + --- @class vim._gen_lsp.Notification + --- @field deprecated? string + --- @field documentation? string + --- @field errorData? any + --- @field messageDirection string + --- @field method string + --- @field params? any[] + --- @field partialResult? any + --- @field proposed? boolean + --- @field registrationMethod? string + --- @field registrationOptions? any + --- @field result any + --- @field since? string + + ---@type (vim._gen_lsp.Request|vim._gen_lsp.Notification)[] local all = vim.list_extend(protocol.requests, protocol.notifications) table.sort(all, function(a, b) - return name(a.method) < name(b.method) + return to_luaname(a.method) < to_luaname(b.method) end) for _, item in ipairs(all) do if item.method then @@ -61,7 +105,7 @@ local function gen_methods(protocol) output[#output + 1] = indent .. '--- ' .. docstring end end - output[#output + 1] = ("%s%s = '%s',"):format(indent, name(item.method), item.method) + output[#output + 1] = ("%s%s = '%s',"):format(indent, to_luaname(item.method), item.method) end end output[#output + 1] = '}' @@ -99,7 +143,14 @@ return protocol vim.cmd.write() end +---@class vim._gen_lsp.opt +---@field output_file string +---@field version string +---@field methods boolean + +---@param opt vim._gen_lsp.opt function M.gen(opt) + --- @type vim._gen_lsp.Protocol local protocol = read_json(opt) if opt.methods then @@ -107,25 +158,30 @@ function M.gen(opt) end local output = { - '--[[', - 'This file is autogenerated from scripts/gen_lsp.lua', + '--' .. '[[', + 'THIS FILE IS GENERATED by scripts/gen_lsp.lua', + 'DO NOT EDIT MANUALLY', + '', + 'Based on LSP protocol ' .. opt.version, + '', 'Regenerate:', - [=[nvim -l scripts/gen_lsp.lua gen --version 3.18 --runtime/lua/vim/lsp/_meta/protocol.lua]=], - '--]]', + ([=[nvim -l scripts/gen_lsp.lua gen --version %s]=]):format(DEFAULT_LSP_VERSION), + '--' .. ']]', + '', + '---@meta', + "error('Cannot require a meta file')", '', '---@alias lsp.null nil', '---@alias uinteger integer', - '---@alias lsp.decimal number', + '---@alias decimal number', '---@alias lsp.DocumentUri string', '---@alias lsp.URI string', - '---@alias lsp.LSPObject table<string, lsp.LSPAny>', - '---@alias lsp.LSPArray lsp.LSPAny[]', - '---@alias lsp.LSPAny lsp.LSPObject|lsp.LSPArray|string|number|boolean|nil', '', } local anonymous_num = 0 + ---@type string[] local anonym_classes = {} local simple_types = { @@ -136,95 +192,181 @@ function M.gen(opt) 'decimal', } - local function parse_type(type) + ---@param documentation string + local _process_documentation = function(documentation) + documentation = documentation:gsub('\n', '\n---') + -- Remove <200b> (zero-width space) unicode characters: e.g., `**/<200b>*` + documentation = documentation:gsub('\226\128\139', '') + -- Escape annotations that are not recognized by lua-ls + documentation = documentation:gsub('%^---@sample', '---\\@sample') + return '---' .. documentation + end + + --- @class vim._gen_lsp.Type + --- @field kind string a common field for all Types. + --- @field name? string for ReferenceType, BaseType + --- @field element? any for ArrayType + --- @field items? vim._gen_lsp.Type[] for OrType, AndType + --- @field key? vim._gen_lsp.Type for MapType + --- @field value? string|vim._gen_lsp.Type for StringLiteralType, MapType, StructureLiteralType + + ---@param type vim._gen_lsp.Type + ---@param prefix? string Optional prefix associated with the this type, made of (nested) field name. + --- Used to generate class name for structure literal types. + ---@return string + local function parse_type(type, prefix) + -- ReferenceType | BaseType if type.kind == 'reference' or type.kind == 'base' then if vim.tbl_contains(simple_types, type.name) then return type.name end return 'lsp.' .. type.name + + -- ArrayType elseif type.kind == 'array' then - return parse_type(type.element) .. '[]' + local parsed_items = parse_type(type.element, prefix) + if type.element.items and #type.element.items > 1 then + parsed_items = '(' .. parsed_items .. ')' + end + return parsed_items .. '[]' + + -- OrType elseif type.kind == 'or' then local val = '' for _, item in ipairs(type.items) do - val = val .. parse_type(item) .. '|' + val = val .. parse_type(item, prefix) .. '|' --[[ @as string ]] end val = val:sub(0, -2) return val + + -- StringLiteralType elseif type.kind == 'stringLiteral' then return '"' .. type.value .. '"' + + -- MapType elseif type.kind == 'map' then - return 'table<' .. parse_type(type.key) .. ', ' .. parse_type(type.value) .. '>' + local key = assert(type.key) + local value = type.value --[[ @as vim._gen_lsp.Type ]] + return 'table<' .. parse_type(key, prefix) .. ', ' .. parse_type(value, prefix) .. '>' + + -- StructureLiteralType elseif type.kind == 'literal' then -- can I use ---@param disabled? {reason: string} -- use | to continue the inline class to be able to add docs -- https://github.com/LuaLS/lua-language-server/issues/2128 anonymous_num = anonymous_num + 1 - local anonym = { '---@class anonym' .. anonymous_num } - for _, field in ipairs(type.value.properties) do + local anonymous_classname = 'lsp._anonym' .. anonymous_num + if prefix then + anonymous_classname = anonymous_classname .. '.' .. prefix + end + local anonym = vim.tbl_flatten { -- remove nil + anonymous_num > 1 and '' or nil, + '---@class ' .. anonymous_classname, + } + + --- @class vim._gen_lsp.StructureLiteral translated to anonymous @class. + --- @field deprecated? string + --- @field description? string + --- @field properties vim._gen_lsp.Property[] + --- @field proposed? boolean + --- @field since? string + + ---@type vim._gen_lsp.StructureLiteral + local structural_literal = assert(type.value) --[[ @as vim._gen_lsp.StructureLiteral ]] + for _, field in ipairs(structural_literal.properties) do + anonym[#anonym + 1] = '---' if field.documentation then - field.documentation = field.documentation:gsub('\n', '\n---') - anonym[#anonym + 1] = '---' .. field.documentation + anonym[#anonym + 1] = _process_documentation(field.documentation) end anonym[#anonym + 1] = '---@field ' .. field.name .. (field.optional and '?' or '') .. ' ' - .. parse_type(field.type) + .. parse_type(field.type, prefix .. '.' .. field.name) end - anonym[#anonym + 1] = '' + -- anonym[#anonym + 1] = '' for _, line in ipairs(anonym) do - anonym_classes[#anonym_classes + 1] = line + if line then + anonym_classes[#anonym_classes + 1] = line + end end - return 'anonym' .. anonymous_num + return anonymous_classname + + -- TupleType elseif type.kind == 'tuple' then local tuple = '{ ' for i, value in ipairs(type.items) do - tuple = tuple .. '[' .. i .. ']: ' .. parse_type(value) .. ', ' + tuple = tuple .. '[' .. i .. ']: ' .. parse_type(value, prefix) .. ', ' end -- remove , at the end tuple = tuple:sub(0, -3) return tuple .. ' }' end - vim.print(type) + + vim.print('WARNING: Unknown type ', type) return '' end + --- @class vim._gen_lsp.Structure translated to @class + --- @field deprecated? string + --- @field documentation? string + --- @field extends? { kind: string, name: string }[] + --- @field mixins? { kind: string, name: string }[] + --- @field name string + --- @field properties? vim._gen_lsp.Property[] members, translated to @field + --- @field proposed? boolean + --- @field since? string for _, structure in ipairs(protocol.structures) do + -- output[#output + 1] = '' if structure.documentation then - structure.documentation = structure.documentation:gsub('\n', '\n---') - output[#output + 1] = '---' .. structure.documentation + output[#output + 1] = _process_documentation(structure.documentation) end - if structure.extends then - local class_string = '---@class lsp.' - .. structure.name - .. ': ' - .. parse_type(structure.extends[1]) - for _, mixin in ipairs(structure.mixins or {}) do - class_string = class_string .. ', ' .. parse_type(mixin) - end - output[#output + 1] = class_string - else - output[#output + 1] = '---@class lsp.' .. structure.name + local class_string = ('---@class lsp.%s'):format(structure.name) + if structure.extends or structure.mixins then + local inherits_from = table.concat( + vim.list_extend( + vim.tbl_map(parse_type, structure.extends or {}), + vim.tbl_map(parse_type, structure.mixins or {}) + ), + ', ' + ) + class_string = class_string .. ': ' .. inherits_from end + output[#output + 1] = class_string + + --- @class vim._gen_lsp.Property translated to @field + --- @field deprecated? string + --- @field documentation? string + --- @field name string + --- @field optional? boolean + --- @field proposed? boolean + --- @field since? string + --- @field type { kind: string, name: string } for _, field in ipairs(structure.properties or {}) do + output[#output + 1] = '---' -- Insert a single newline between @fields (and after @class) if field.documentation then - field.documentation = field.documentation:gsub('\n', '\n---') - output[#output + 1] = '---' .. field.documentation + output[#output + 1] = _process_documentation(field.documentation) end output[#output + 1] = '---@field ' .. field.name .. (field.optional and '?' or '') .. ' ' - .. parse_type(field.type) + .. parse_type(field.type, field.name) end output[#output + 1] = '' end + --- @class vim._gen_lsp.Enumeration translated to @enum + --- @field deprecated string? + --- @field documentation string? + --- @field name string? + --- @field proposed boolean? + --- @field since string? + --- @field suportsCustomValues boolean? + --- @field values { name: string, value: string, documentation?: string, since?: string }[] for _, enum in ipairs(protocol.enumerations) do if enum.documentation then - enum.documentation = enum.documentation:gsub('\n', '\n---') - output[#output + 1] = '---' .. enum.documentation + output[#output + 1] = _process_documentation(enum.documentation) end local enum_type = '---@alias lsp.' .. enum.name for _, value in ipairs(enum.values) do @@ -238,53 +380,77 @@ function M.gen(opt) output[#output + 1] = '' end + --- @class vim._gen_lsp.TypeAlias translated to @alias + --- @field deprecated? string? + --- @field documentation? string + --- @field name string + --- @field proposed? boolean + --- @field since? string + --- @field type vim._gen_lsp.Type for _, alias in ipairs(protocol.typeAliases) do if alias.documentation then - alias.documentation = alias.documentation:gsub('\n', '\n---') - output[#output + 1] = '---' .. alias.documentation + output[#output + 1] = _process_documentation(alias.documentation) end if alias.type.kind == 'or' then local alias_type = '---@alias lsp.' .. alias.name .. ' ' for _, item in ipairs(alias.type.items) do - alias_type = alias_type .. parse_type(item) .. '|' + alias_type = alias_type .. parse_type(item, alias.name) .. '|' end alias_type = alias_type:sub(0, -2) output[#output + 1] = alias_type else - output[#output + 1] = '---@alias lsp.' .. alias.name .. ' ' .. parse_type(alias.type) + output[#output + 1] = '---@alias lsp.' + .. alias.name + .. ' ' + .. parse_type(alias.type, alias.name) end output[#output + 1] = '' end + -- anonymous classes for _, line in ipairs(anonym_classes) do output[#output + 1] = line end - tofile(opt.output_file, table.concat(output, '\n')) + tofile(opt.output_file, table.concat(output, '\n') .. '\n') end +---@type vim._gen_lsp.opt local opt = { output_file = 'runtime/lua/vim/lsp/_meta/protocol.lua', - version = nil, - methods = nil, + version = DEFAULT_LSP_VERSION, + methods = false, } -for i = 1, #_G.arg do +local command = nil +local i = 1 +while i <= #_G.arg do if _G.arg[i] == '--out' then - opt.output_file = _G.arg[i + 1] + opt.output_file = assert(_G.arg[i + 1], '--out <outfile> needed') + i = i + 1 elseif _G.arg[i] == '--version' then - opt.version = _G.arg[i + 1] + opt.version = assert(_G.arg[i + 1], '--version <version> needed') + i = i + 1 elseif _G.arg[i] == '--methods' then opt.methods = true - elseif vim.startswith(_G.arg[i], '--') then - opt.output_file = _G.arg[i]:sub(3) + elseif vim.startswith(_G.arg[i], '-') then + error('Unrecognized args: ' .. _G.arg[i]) + else + if command then + error('More than one command was given: ' .. _G.arg[i]) + else + command = _G.arg[i] + end end + i = i + 1 end -for _, a in ipairs(arg) do - if M[a] then - M[a](opt) - end +if not command then + print(USAGE) +elseif M[command] then + M[command](opt) -- see M.gen() +else + error('Unknown command: ' .. command) end return M diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua new file mode 100755 index 0000000000..22df411a35 --- /dev/null +++ b/scripts/gen_vimdoc.lua @@ -0,0 +1,953 @@ +#!/usr/bin/env -S nvim -l +--- Generates Nvim :help docs from Lua/C docstrings +--- +--- The generated :help text for each function is formatted as follows: +--- - Max width of 78 columns (`TEXT_WIDTH`). +--- - Indent with spaces (not tabs). +--- - Indent of 4 columns for body text (`INDENTATION`). +--- - Function signature and helptag (right-aligned) on the same line. +--- - Signature and helptag must have a minimum of 8 spaces between them. +--- - If the signature is too long, it is placed on the line after the helptag. +--- Signature wraps with subsequent lines indented to the open parenthesis. +--- - Subsection bodies are indented an additional 4 spaces. +--- - Body consists of function description, parameters, return description, and +--- C declaration (`INCLUDE_C_DECL`). +--- - Parameters are omitted for the `void` and `Error *` types, or if the +--- parameter is marked as [out]. +--- - Each function documentation is separated by a single line. + +local luacats_parser = require('scripts.luacats_parser') +local cdoc_parser = require('scripts.cdoc_parser') +local text_utils = require('scripts.text_utils') + +local fmt = string.format + +local wrap = text_utils.wrap +local md_to_vimdoc = text_utils.md_to_vimdoc + +local TEXT_WIDTH = 78 +local INDENTATION = 4 + +--- @class (exact) nvim.gen_vimdoc.Config +--- +--- Generated documentation target, e.g. api.txt +--- @field filename string +--- +--- @field section_order string[] +--- +--- List of files/directories for doxygen to read, relative to `base_dir`. +--- @field files string[] +--- +--- @field exclude_types? true +--- +--- Section name overrides. Key: filename (e.g., vim.c) +--- @field section_name? table<string,string> +--- +--- @field fn_name_pat? string +--- +--- @field fn_xform? fun(fun: nvim.luacats.parser.fun) +--- +--- For generated section names. +--- @field section_fmt fun(name: string): string +--- +--- @field helptag_fmt fun(name: string): string +--- +--- Per-function helptag. +--- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string +--- +--- @field append_only? string[] + +local function contains(t, xs) + return vim.tbl_contains(xs, t) +end + +--- @type {level:integer, prerelease:boolean}? +local nvim_api_info_ + +--- @return {level: integer, prerelease:boolean} +local function nvim_api_info() + if not nvim_api_info_ then + --- @type integer?, boolean? + local level, prerelease + for l in io.lines('CMakeLists.txt') do + --- @cast l string + if level and prerelease then + break + end + local m1 = l:match('^set%(NVIM_API_LEVEL%s+(%d+)%)') + if m1 then + level = tonumber(m1) --[[@as integer]] + end + local m2 = l:match('^set%(NVIM_API_PRERELEASE%s+(%w+)%)') + if m2 then + prerelease = m2 == 'true' + end + end + nvim_api_info_ = { level = level, prerelease = prerelease } + end + + return nvim_api_info_ +end + +--- @param fun nvim.luacats.parser.fun +--- @return string +local function fn_helptag_fmt_common(fun) + local fn_sfx = fun.table and '' or '()' + if fun.classvar then + return fmt('*%s:%s%s*', fun.classvar, fun.name, fn_sfx) + end + if fun.module then + return fmt('*%s.%s%s*', fun.module, fun.name, fn_sfx) + end + return fmt('*%s%s*', fun.name, fn_sfx) +end + +--- @type table<string,nvim.gen_vimdoc.Config> +local config = { + api = { + filename = 'api.txt', + section_order = { + 'vim.c', + 'vimscript.c', + 'command.c', + 'options.c', + 'buffer.c', + 'extmark.c', + 'window.c', + 'win_config.c', + 'tabpage.c', + 'autocmd.c', + 'ui.c', + }, + exclude_types = true, + fn_name_pat = 'nvim_.*', + files = { 'src/nvim/api' }, + section_name = { + ['vim.c'] = 'Global', + }, + section_fmt = function(name) + return name .. ' Functions' + end, + helptag_fmt = function(name) + return fmt('*api-%s*', name:lower()) + end, + }, + lua = { + filename = 'lua.txt', + section_order = { + 'highlight.lua', + 'diff.lua', + 'mpack.lua', + 'json.lua', + 'base64.lua', + 'spell.lua', + 'builtin.lua', + '_options.lua', + '_editor.lua', + '_inspector.lua', + 'shared.lua', + 'loader.lua', + 'uri.lua', + 'ui.lua', + 'filetype.lua', + 'keymap.lua', + 'fs.lua', + 'glob.lua', + 'lpeg.lua', + 're.lua', + 'regex.lua', + 'secure.lua', + 'version.lua', + 'iter.lua', + 'snippet.lua', + 'text.lua', + 'tohtml.lua', + }, + files = { + 'runtime/lua/vim/iter.lua', + 'runtime/lua/vim/_editor.lua', + 'runtime/lua/vim/_options.lua', + 'runtime/lua/vim/shared.lua', + 'runtime/lua/vim/loader.lua', + 'runtime/lua/vim/uri.lua', + 'runtime/lua/vim/ui.lua', + 'runtime/lua/vim/filetype.lua', + 'runtime/lua/vim/keymap.lua', + 'runtime/lua/vim/fs.lua', + 'runtime/lua/vim/highlight.lua', + 'runtime/lua/vim/secure.lua', + 'runtime/lua/vim/version.lua', + 'runtime/lua/vim/_inspector.lua', + 'runtime/lua/vim/snippet.lua', + 'runtime/lua/vim/text.lua', + 'runtime/lua/vim/glob.lua', + 'runtime/lua/vim/_meta/builtin.lua', + 'runtime/lua/vim/_meta/diff.lua', + 'runtime/lua/vim/_meta/mpack.lua', + 'runtime/lua/vim/_meta/json.lua', + 'runtime/lua/vim/_meta/base64.lua', + 'runtime/lua/vim/_meta/regex.lua', + 'runtime/lua/vim/_meta/lpeg.lua', + 'runtime/lua/vim/_meta/re.lua', + 'runtime/lua/vim/_meta/spell.lua', + 'runtime/lua/tohtml.lua', + }, + fn_xform = function(fun) + if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then + fun.module = 'vim' + end + + if fun.module == 'vim' and contains(fun.name, { 'cmd', 'inspect' }) then + fun.table = nil + end + + if fun.classvar or vim.startswith(fun.name, 'vim.') or fun.module == 'vim.iter' then + return + end + + fun.name = fmt('%s.%s', fun.module, fun.name) + end, + section_name = { + ['_inspector.lua'] = 'inspector', + }, + section_fmt = function(name) + name = name:lower() + if name == '_editor' then + return 'Lua module: vim' + elseif name == '_options' then + return 'LUA-VIMSCRIPT BRIDGE' + elseif name == 'builtin' then + return 'VIM' + end + if + contains(name, { + 'highlight', + 'mpack', + 'json', + 'base64', + 'diff', + 'spell', + 'regex', + 'lpeg', + 're', + }) + then + return 'VIM.' .. name:upper() + end + if name == 'tohtml' then + return 'Lua module: tohtml' + end + return 'Lua module: vim.' .. name + end, + helptag_fmt = function(name) + if name == '_editor' then + return '*lua-vim*' + elseif name == '_options' then + return '*lua-vimscript*' + elseif name == 'tohtml' then + return '*tohtml*' + end + return '*vim.' .. name:lower() .. '*' + end, + fn_helptag_fmt = function(fun) + local name = fun.name + + if vim.startswith(name, 'vim.') then + local fn_sfx = fun.table and '' or '()' + return fmt('*%s%s*', name, fn_sfx) + elseif fun.classvar == 'Option' then + return fmt('*vim.opt:%s()*', name) + end + + return fn_helptag_fmt_common(fun) + end, + append_only = { + 'shared.lua', + }, + }, + lsp = { + filename = 'lsp.txt', + section_order = { + 'lsp.lua', + 'client.lua', + 'buf.lua', + 'diagnostic.lua', + 'codelens.lua', + 'inlay_hint.lua', + 'tagfunc.lua', + 'semantic_tokens.lua', + 'handlers.lua', + 'util.lua', + 'log.lua', + 'rpc.lua', + 'protocol.lua', + }, + files = { + 'runtime/lua/vim/lsp', + 'runtime/lua/vim/lsp.lua', + }, + fn_xform = function(fun) + fun.name = fun.name:gsub('result%.', '') + end, + section_fmt = function(name) + if name:lower() == 'lsp' then + return 'Lua module: vim.lsp' + end + return 'Lua module: vim.lsp.' .. name:lower() + end, + helptag_fmt = function(name) + if name:lower() == 'lsp' then + return '*lsp-core*' + end + return fmt('*lsp-%s*', name:lower()) + end, + }, + diagnostic = { + filename = 'diagnostic.txt', + section_order = { + 'diagnostic.lua', + }, + files = { 'runtime/lua/vim/diagnostic.lua' }, + section_fmt = function() + return 'Lua module: vim.diagnostic' + end, + helptag_fmt = function() + return '*diagnostic-api*' + end, + }, + treesitter = { + filename = 'treesitter.txt', + section_order = { + 'treesitter.lua', + 'language.lua', + 'query.lua', + 'highlighter.lua', + 'languagetree.lua', + 'dev.lua', + }, + files = { + 'runtime/lua/vim/treesitter.lua', + 'runtime/lua/vim/treesitter/', + }, + section_fmt = function(name) + if name:lower() == 'treesitter' then + return 'Lua module: vim.treesitter' + end + return 'Lua module: vim.treesitter.' .. name:lower() + end, + helptag_fmt = function(name) + if name:lower() == 'treesitter' then + return '*lua-treesitter-core*' + end + return '*lua-treesitter-' .. name:lower() .. '*' + end, + }, +} + +--- @param ty string +--- @param generics table<string,string> +--- @return string +local function replace_generics(ty, generics) + if ty:sub(-2) == '[]' then + local ty0 = ty:sub(1, -3) + if generics[ty0] then + return generics[ty0] .. '[]' + end + elseif ty:sub(-1) == '?' then + local ty0 = ty:sub(1, -2) + if generics[ty0] then + return generics[ty0] .. '?' + end + end + + return generics[ty] or ty +end + +--- @param name string +local function fmt_field_name(name) + local name0, opt = name:match('^([^?]*)(%??)$') + return fmt('{%s}%s', name0, opt) +end + +--- @param ty string +--- @param generics? table<string,string> +--- @param default? string +local function render_type(ty, generics, default) + if generics then + ty = replace_generics(ty, generics) + end + ty = ty:gsub('%s*|%s*nil', '?') + ty = ty:gsub('nil%s*|%s*(.*)', '%1?') + ty = ty:gsub('%s*|%s*', '|') + if default then + return fmt('(`%s`, default: %s)', ty, default) + end + return fmt('(`%s`)', ty) +end + +--- @param p nvim.luacats.parser.param|nvim.luacats.parser.field +local function should_render_param(p) + return not p.access and not contains(p.name, { '_', 'self' }) +end + +--- @param desc? string +--- @return string?, string? +local function get_default(desc) + if not desc then + return + end + + local default = desc:match('\n%s*%([dD]efault: ([^)]+)%)') + if default then + desc = desc:gsub('\n%s*%([dD]efault: [^)]+%)', '') + end + + return desc, default +end + +--- @param ty string +--- @param classes? table<string,nvim.luacats.parser.class> +--- @return nvim.luacats.parser.class? +local function get_class(ty, classes) + if not classes then + return + end + + local cty = ty:gsub('%s*|%s*nil', '?'):gsub('?$', ''):gsub('%[%]$', '') + + return classes[cty] +end + +--- @param obj nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.field +--- @param classes? table<string,nvim.luacats.parser.class> +local function inline_type(obj, classes) + local ty = obj.type + if not ty then + return + end + + local cls = get_class(ty, classes) + + if not cls or cls.nodoc then + return + end + + if not cls.inlinedoc then + -- Not inlining so just add a: "See |tag|." + local tag = fmt('|%s|', cls.name) + if obj.desc and obj.desc:find(tag) then + -- Tag already there + return + end + + -- TODO(lewis6991): Aim to remove this. Need this to prevent dead + -- references to types defined in runtime/lua/vim/lsp/_meta/protocol.lua + if not vim.startswith(cls.name, 'vim.') then + return + end + + obj.desc = obj.desc or '' + local period = (obj.desc == '' or vim.endswith(obj.desc, '.')) and '' or '.' + obj.desc = obj.desc .. fmt('%s See %s.', period, tag) + return + end + + local ty_isopt = (ty:match('%?$') or ty:match('%s*|%s*nil')) ~= nil + local ty_islist = (ty:match('%[%]$')) ~= nil + ty = ty_isopt and 'table?' or ty_islist and 'table[]' or 'table' + + local desc = obj.desc or '' + if cls.desc then + desc = desc .. cls.desc + elseif desc == '' then + if ty_islist then + desc = desc .. 'A list of objects with the following fields:' + else + desc = desc .. 'A table with the following fields:' + end + end + + local desc_append = {} + for _, f in ipairs(cls.fields) do + local fdesc, default = get_default(f.desc) + local fty = render_type(f.type, nil, default) + local fnm = fmt_field_name(f.name) + table.insert(desc_append, table.concat({ '-', fnm, fty, fdesc }, ' ')) + end + + desc = desc .. '\n' .. table.concat(desc_append, '\n') + obj.type = ty + obj.desc = desc +end + +--- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[] +--- @param generics? table<string,string> +--- @param classes? table<string,nvim.luacats.parser.class> +--- @param exclude_types? true +local function render_fields_or_params(xs, generics, classes, exclude_types) + local ret = {} --- @type string[] + + xs = vim.tbl_filter(should_render_param, xs) + + local indent = 0 + for _, p in ipairs(xs) do + if p.type or p.desc then + indent = math.max(indent, #p.name + 3) + end + if exclude_types then + p.type = nil + end + end + + for _, p in ipairs(xs) do + local pdesc, default = get_default(p.desc) + p.desc = pdesc + + inline_type(p, classes) + local nm, ty, desc = p.name, p.type, p.desc + + local fnm = p.kind == 'operator' and fmt('op(%s)', nm) or fmt_field_name(nm) + local pnm = fmt(' • %-' .. indent .. 's', fnm) + + if ty then + local pty = render_type(ty, generics, default) + + if desc then + table.insert(ret, pnm) + if #pty > TEXT_WIDTH - indent then + vim.list_extend(ret, { ' ', pty, '\n' }) + table.insert(ret, md_to_vimdoc(desc, 9 + indent, 9 + indent, TEXT_WIDTH, true)) + else + desc = fmt('%s %s', pty, desc) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + end + else + table.insert(ret, fmt('%s %s\n', pnm, pty)) + end + else + if desc then + table.insert(ret, pnm) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + end + end + end + + return table.concat(ret) +end + +--- @param class nvim.luacats.parser.class +--- @param classes table<string,nvim.luacats.parser.class> +local function render_class(class, classes) + if class.access or class.nodoc or class.inlinedoc then + return + end + + local ret = {} --- @type string[] + + table.insert(ret, fmt('*%s*\n', class.name)) + + if class.parent then + local txt = fmt('Extends: |%s|', class.parent) + table.insert(ret, md_to_vimdoc(txt, INDENTATION, INDENTATION, TEXT_WIDTH)) + table.insert(ret, '\n') + end + + if class.desc then + table.insert(ret, md_to_vimdoc(class.desc, INDENTATION, INDENTATION, TEXT_WIDTH)) + end + + local fields_txt = render_fields_or_params(class.fields, nil, classes) + if not fields_txt:match('^%s*$') then + table.insert(ret, '\n Fields: ~\n') + table.insert(ret, fields_txt) + end + table.insert(ret, '\n') + + return table.concat(ret) +end + +--- @param classes table<string,nvim.luacats.parser.class> +local function render_classes(classes) + local ret = {} --- @type string[] + + for _, class in vim.spairs(classes) do + ret[#ret + 1] = render_class(class, classes) + end + + return table.concat(ret) +end + +--- @param fun nvim.luacats.parser.fun +--- @param cfg nvim.gen_vimdoc.Config +local function render_fun_header(fun, cfg) + local ret = {} --- @type string[] + + local args = {} --- @type string[] + for _, p in ipairs(fun.params or {}) do + if p.name ~= 'self' then + args[#args + 1] = fmt_field_name(p.name) + end + end + + local nm = fun.name + if fun.classvar then + nm = fmt('%s:%s', fun.classvar, nm) + end + + local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')' + + if not cfg.fn_helptag_fmt then + cfg.fn_helptag_fmt = fn_helptag_fmt_common + end + + local tag = cfg.fn_helptag_fmt(fun) + + if #proto + #tag > TEXT_WIDTH - 8 then + table.insert(ret, fmt('%78s\n', tag)) + local name, pargs = proto:match('([^(]+%()(.*)') + table.insert(ret, name) + table.insert(ret, wrap(pargs, 0, #name, TEXT_WIDTH)) + else + local pad = TEXT_WIDTH - #proto - #tag + table.insert(ret, proto .. string.rep(' ', pad) .. tag) + end + + return table.concat(ret) +end + +--- @param returns nvim.luacats.parser.return[] +--- @param generics? table<string,string> +--- @param classes? table<string,nvim.luacats.parser.class> +--- @param exclude_types boolean +local function render_returns(returns, generics, classes, exclude_types) + local ret = {} --- @type string[] + + returns = vim.deepcopy(returns) + if exclude_types then + for _, r in ipairs(returns) do + r.type = nil + end + end + + if #returns > 1 then + table.insert(ret, ' Return (multiple): ~\n') + elseif #returns == 1 and next(returns[1]) then + table.insert(ret, ' Return: ~\n') + end + + for _, p in ipairs(returns) do + inline_type(p, classes) + local rnm, ty, desc = p.name, p.type, p.desc + + local blk = {} --- @type string[] + if ty then + blk[#blk + 1] = render_type(ty, generics) + end + blk[#blk + 1] = rnm + blk[#blk + 1] = desc + + table.insert(ret, md_to_vimdoc(table.concat(blk, ' '), 8, 8, TEXT_WIDTH, true)) + end + + return table.concat(ret) +end + +--- @param fun nvim.luacats.parser.fun +--- @param classes table<string,nvim.luacats.parser.class> +--- @param cfg nvim.gen_vimdoc.Config +local function render_fun(fun, classes, cfg) + if fun.access or fun.deprecated or fun.nodoc then + return + end + + if cfg.fn_name_pat and not fun.name:match(cfg.fn_name_pat) then + return + end + + if vim.startswith(fun.name, '_') or fun.name:find('[:.]_') then + return + end + + local ret = {} --- @type string[] + + table.insert(ret, render_fun_header(fun, cfg)) + table.insert(ret, '\n') + + if fun.desc then + table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH)) + end + + if fun.since then + local since = tonumber(fun.since) + local info = nvim_api_info() + if since and (since > info.level or since == info.level and info.prerelease) then + fun.notes = fun.notes or {} + table.insert(fun.notes, { desc = 'This API is pre-release (unstable).' }) + end + end + + if fun.notes then + table.insert(ret, '\n Note: ~\n') + for _, p in ipairs(fun.notes) do + table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true)) + end + end + + if fun.attrs then + table.insert(ret, '\n Attributes: ~\n') + for _, attr in ipairs(fun.attrs) do + local attr_str = ({ + textlock = 'not allowed when |textlock| is active or in the |cmdwin|', + textlock_allow_cmdwin = 'not allowed when |textlock| is active', + fast = '|api-fast|', + remote_only = '|RPC| only', + lua_only = 'Lua |vim.api| only', + })[attr] or attr + table.insert(ret, fmt(' %s\n', attr_str)) + end + end + + if fun.params and #fun.params > 0 then + local param_txt = render_fields_or_params(fun.params, fun.generics, classes, cfg.exclude_types) + if not param_txt:match('^%s*$') then + table.insert(ret, '\n Parameters: ~\n') + ret[#ret + 1] = param_txt + end + end + + if fun.returns then + local txt = render_returns(fun.returns, fun.generics, classes, cfg.exclude_types) + if not txt:match('^%s*$') then + table.insert(ret, '\n') + ret[#ret + 1] = txt + end + end + + if fun.see then + table.insert(ret, '\n See also: ~\n') + for _, p in ipairs(fun.see) do + table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true)) + end + end + + table.insert(ret, '\n') + return table.concat(ret) +end + +--- @param funs nvim.luacats.parser.fun[] +--- @param classes table<string,nvim.luacats.parser.class> +--- @param cfg nvim.gen_vimdoc.Config +local function render_funs(funs, classes, cfg) + local ret = {} --- @type string[] + + for _, f in ipairs(funs) do + if cfg.fn_xform then + cfg.fn_xform(f) + end + ret[#ret + 1] = render_fun(f, classes, cfg) + end + + -- Sort via prototype + table.sort(ret, function(a, b) + local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n') + local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n') + return a1:lower() < b1:lower() + end) + + return table.concat(ret) +end + +--- @return string +local function get_script_path() + local str = debug.getinfo(2, 'S').source:sub(2) + return str:match('(.*[/\\])') or './' +end + +local script_path = get_script_path() +local base_dir = vim.fs.dirname(assert(vim.fs.dirname(script_path))) + +local function delete_lines_below(doc_file, tokenstr) + local lines = {} --- @type string[] + local found = false + for line in io.lines(doc_file) do + if line:find(vim.pesc(tokenstr)) then + found = true + break + end + lines[#lines + 1] = line + end + if not found then + error(fmt('not found: %s in %s', tokenstr, doc_file)) + end + lines[#lines] = nil + local fp = assert(io.open(doc_file, 'w')) + fp:write(table.concat(lines, '\n')) + fp:write('\n') + fp:close() +end + +--- @param x string +local function mktitle(x) + if x == 'ui' then + return 'UI' + end + return x:sub(1, 1):upper() .. x:sub(2) +end + +--- @class nvim.gen_vimdoc.Section +--- @field name string +--- @field title string +--- @field help_tag string +--- @field funs_txt string +--- @field doc? string[] + +--- @param filename string +--- @param cfg nvim.gen_vimdoc.Config +--- @param section_docs table<string,nvim.gen_vimdoc.Section> +--- @param funs_txt string +--- @return nvim.gen_vimdoc.Section? +local function make_section(filename, cfg, section_docs, funs_txt) + -- filename: e.g., 'autocmd.c' + -- name: e.g. 'autocmd' + local name = filename:match('(.*)%.[a-z]+') + + -- Formatted (this is what's going to be written in the vimdoc) + -- e.g., "Autocmd Functions" + local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name) + + -- section tag: e.g., "*api-autocmd*" + local help_tag = cfg.helptag_fmt(sectname) + + if funs_txt == '' and #section_docs == 0 then + return + end + + return { + name = sectname, + title = cfg.section_fmt(sectname), + help_tag = help_tag, + funs_txt = funs_txt, + doc = section_docs, + } +end + +--- @param section nvim.gen_vimdoc.Section +--- @param add_header? boolean +local function render_section(section, add_header) + local doc = {} --- @type string[] + + if add_header ~= false then + vim.list_extend(doc, { + string.rep('=', TEXT_WIDTH), + '\n', + section.title, + fmt('%' .. (TEXT_WIDTH - section.title:len()) .. 's', section.help_tag), + }) + end + + if section.doc and #section.doc > 0 then + table.insert(doc, '\n\n') + vim.list_extend(doc, section.doc) + end + + if section.funs_txt then + table.insert(doc, '\n\n') + table.insert(doc, section.funs_txt) + end + + return table.concat(doc) +end + +local parsers = { + lua = luacats_parser.parse, + c = cdoc_parser.parse, + h = cdoc_parser.parse, +} + +--- @param files string[] +local function expand_files(files) + for k, f in pairs(files) do + if vim.fn.isdirectory(f) == 1 then + table.remove(files, k) + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + table.insert(files, vim.fs.joinpath(f, path)) + end + end + end + end +end + +--- @param cfg nvim.gen_vimdoc.Config +local function gen_target(cfg) + local sections = {} --- @type table<string,nvim.gen_vimdoc.Section> + + expand_files(cfg.files) + + --- @type table<string,{[1]:table<string,nvim.luacats.parser.class>, [2]: nvim.luacats.parser.fun[], [3]: string[]}> + local file_results = {} + + --- @type table<string,nvim.luacats.parser.class> + local all_classes = {} + + --- First pass so we can collect all classes + for _, f in pairs(cfg.files) do + local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']] + local parser = assert(parsers[ext]) + local classes, funs, briefs = parser(f) + file_results[f] = { classes, funs, briefs } + all_classes = vim.tbl_extend('error', all_classes, classes) + end + + for f, r in pairs(file_results) do + local classes, funs, briefs = r[1], r[2], r[3] + + local briefs_txt = {} --- @type string[] + for _, b in ipairs(briefs) do + briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH) + end + local funs_txt = render_funs(funs, all_classes, cfg) + if next(classes) then + local classes_txt = render_classes(classes) + if vim.trim(classes_txt) ~= '' then + funs_txt = classes_txt .. '\n' .. funs_txt + end + end + -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua` + local f_base = assert(vim.fs.basename(f)) + sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt) + end + + local first_section_tag = sections[cfg.section_order[1]].help_tag + local docs = {} --- @type string[] + for _, f in ipairs(cfg.section_order) do + local section = sections[f] + if section then + local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f) + table.insert(docs, render_section(section, add_sep_and_header)) + end + end + + table.insert( + docs, + fmt(' vim:tw=78:ts=8:sw=%d:sts=%d:et:ft=help:norl:\n', INDENTATION, INDENTATION) + ) + + local doc_file = vim.fs.joinpath(base_dir, 'runtime', 'doc', cfg.filename) + + if vim.uv.fs_stat(doc_file) then + delete_lines_below(doc_file, first_section_tag) + end + + local fp = assert(io.open(doc_file, 'a')) + fp:write(table.concat(docs, '\n')) + fp:close() +end + +local function run() + for _, cfg in pairs(config) do + gen_target(cfg) + end +end + +run() diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py deleted file mode 100755 index 8ed88cb8f5..0000000000 --- a/scripts/gen_vimdoc.py +++ /dev/null @@ -1,1447 +0,0 @@ -#!/usr/bin/env python3 -"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. - -Also generates *.mpack files. To inspect the *.mpack structure: - :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B'))) - -Flow: - main - extract_from_xml - fmt_node_as_vimhelp \ - para_as_map } recursive - update_params_map / - render_node - -TODO: eliminate this script and use Lua+treesitter (requires parsers for C and -Lua markdown-style docstrings). - -The generated :help text for each function is formatted as follows: - - - Max width of 78 columns (`text_width`). - - Indent with spaces (not tabs). - - Indent of 4 columns for body text (`indentation`). - - Function signature and helptag (right-aligned) on the same line. - - Signature and helptag must have a minimum of 8 spaces between them. - - If the signature is too long, it is placed on the line after the helptag. - Signature wraps at `text_width - 8` characters with subsequent - lines indented to the open parenthesis. - - Subsection bodies are indented an additional 4 spaces. - - Body consists of function description, parameters, return description, and - C declaration (`INCLUDE_C_DECL`). - - Parameters are omitted for the `void` and `Error *` types, or if the - parameter is marked as [out]. - - Each function documentation is separated by a single line. -""" -import argparse -import os -import re -import sys -import shutil -import textwrap -import subprocess -import collections -import msgpack -import logging -from typing import Tuple -from pathlib import Path - -from xml.dom import minidom -Element = minidom.Element -Document = minidom.Document - -MIN_PYTHON_VERSION = (3, 6) -MIN_DOXYGEN_VERSION = (1, 9, 0) - -if sys.version_info < MIN_PYTHON_VERSION: - print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION)) - sys.exit(1) - -doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"], - universal_newlines=True).split()[0].split('.'))) - -if doxygen_version < MIN_DOXYGEN_VERSION: - print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION)) - print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version)) - sys.exit(1) - - -# Need a `nvim` that supports `-l`, try the local build -nvim_path = Path(__file__).parent / "../build/bin/nvim" -if nvim_path.exists(): - nvim = str(nvim_path) -else: - # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works. - nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True) - nvim_version = [line for line in nvim_out.split('\n') - if '-l ' in line] - if len(nvim_version) == 0: - print(( - "\nYou need to have a local Neovim build or a `nvim` version 0.9 for `-l` " - "support to build the documentation.")) - sys.exit(1) - nvim = 'nvim' - - -# DEBUG = ('DEBUG' in os.environ) -INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) -INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) - -log = logging.getLogger(__name__) - -LOG_LEVELS = { - logging.getLevelName(level): level for level in [ - logging.DEBUG, logging.INFO, logging.ERROR - ] -} - -text_width = 78 -indentation = 4 -script_path = os.path.abspath(__file__) -base_dir = os.path.dirname(os.path.dirname(script_path)) -out_dir = os.path.join(base_dir, 'tmp-{target}-doc') -filter_cmd = '%s %s' % (sys.executable, script_path) -msgs = [] # Messages to show on exit. -lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') - -CONFIG = { - 'api': { - 'mode': 'c', - 'filename': 'api.txt', - # Section ordering. - 'section_order': [ - 'vim.c', - 'vimscript.c', - 'command.c', - 'options.c', - 'buffer.c', - 'extmark.c', - 'window.c', - 'win_config.c', - 'tabpage.c', - 'autocmd.c', - 'ui.c', - ], - # List of files/directories for doxygen to read, relative to `base_dir` - 'files': ['src/nvim/api'], - # file patterns used by doxygen - 'file_patterns': '*.h *.c', - # Only function with this prefix are considered - 'fn_name_prefix': 'nvim_', - # Section name overrides. - 'section_name': { - 'vim.c': 'Global', - }, - # For generated section names. - 'section_fmt': lambda name: f'{name} Functions', - # Section helptag. - 'helptag_fmt': lambda name: f'*api-{name.lower()}*', - # Per-function helptag. - 'fn_helptag_fmt': lambda fstem, name, istbl: f'*{name}()*', - # Module name overrides (for Lua). - 'module_override': {}, - # Append the docs for these modules, do not start a new section. - 'append_only': [], - }, - 'lua': { - 'mode': 'lua', - 'filename': 'lua.txt', - 'section_order': [ - 'highlight.lua', - 'regex.lua', - 'diff.lua', - 'mpack.lua', - 'json.lua', - 'base64.lua', - 'spell.lua', - 'builtin.lua', - '_options.lua', - '_editor.lua', - '_inspector.lua', - 'shared.lua', - 'loader.lua', - 'uri.lua', - 'ui.lua', - 'filetype.lua', - 'keymap.lua', - 'fs.lua', - 'secure.lua', - 'version.lua', - 'iter.lua', - 'snippet.lua', - 'text.lua', - ], - 'files': [ - 'runtime/lua/vim/iter.lua', - 'runtime/lua/vim/_editor.lua', - 'runtime/lua/vim/_options.lua', - 'runtime/lua/vim/shared.lua', - 'runtime/lua/vim/loader.lua', - 'runtime/lua/vim/uri.lua', - 'runtime/lua/vim/ui.lua', - 'runtime/lua/vim/filetype.lua', - 'runtime/lua/vim/keymap.lua', - 'runtime/lua/vim/fs.lua', - 'runtime/lua/vim/highlight.lua', - 'runtime/lua/vim/secure.lua', - 'runtime/lua/vim/version.lua', - 'runtime/lua/vim/_inspector.lua', - 'runtime/lua/vim/snippet.lua', - 'runtime/lua/vim/text.lua', - 'runtime/lua/vim/_meta/builtin.lua', - 'runtime/lua/vim/_meta/diff.lua', - 'runtime/lua/vim/_meta/mpack.lua', - 'runtime/lua/vim/_meta/json.lua', - 'runtime/lua/vim/_meta/base64.lua', - 'runtime/lua/vim/_meta/regex.lua', - 'runtime/lua/vim/_meta/spell.lua', - ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'fn_name_fmt': lambda fstem, name: ( - name if fstem in [ 'vim.iter' ] else - f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else - f'vim.{name}' if fstem == '_options' and not name[0].isupper() else - f'{fstem}.{name}' if fstem.startswith('vim') else - name - ), - 'section_name': { - 'lsp.lua': 'core', - '_inspector.lua': 'inspector', - }, - 'section_fmt': lambda name: ( - 'Lua module: vim' if name.lower() == '_editor' else - 'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else - f'VIM.{name.upper()}' if name.lower() in [ 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', 'regex' ] else - 'VIM' if name.lower() == 'builtin' else - f'Lua module: vim.{name.lower()}'), - 'helptag_fmt': lambda name: ( - '*lua-vim*' if name.lower() == '_editor' else - '*lua-vimscript*' if name.lower() == '_options' else - f'*vim.{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( - f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else - # Exclude fstem for methods - f'*{name}()*' if ':' in name else - f'*vim.{name}()*' if fstem.lower() == '_editor' else - f'*vim.{name}*' if fstem.lower() == '_options' and istbl else - # Prevents vim.regex.regex - f'*{fstem}()*' if fstem.endswith('.' + name) else - f'*{fstem}.{name}{"" if istbl else "()"}*' - ), - 'module_override': { - # `shared` functions are exposed on the `vim` module. - 'shared': 'vim', - '_inspector': 'vim', - 'uri': 'vim', - 'ui': 'vim.ui', - 'loader': 'vim.loader', - 'filetype': 'vim.filetype', - 'keymap': 'vim.keymap', - 'fs': 'vim.fs', - 'highlight': 'vim.highlight', - 'secure': 'vim.secure', - 'version': 'vim.version', - 'iter': 'vim.iter', - 'diff': 'vim', - 'builtin': 'vim', - 'mpack': 'vim.mpack', - 'json': 'vim.json', - 'base64': 'vim.base64', - 'regex': 'vim.regex', - 'spell': 'vim.spell', - 'snippet': 'vim.snippet', - 'text': 'vim.text', - }, - 'append_only': [ - 'shared.lua', - ], - }, - 'lsp': { - 'mode': 'lua', - 'filename': 'lsp.txt', - 'section_order': [ - 'lsp.lua', - 'buf.lua', - 'diagnostic.lua', - 'codelens.lua', - 'inlay_hint.lua', - 'tagfunc.lua', - 'semantic_tokens.lua', - 'handlers.lua', - 'util.lua', - 'log.lua', - 'rpc.lua', - 'protocol.lua', - ], - 'files': [ - 'runtime/lua/vim/lsp', - 'runtime/lua/vim/lsp.lua', - ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'section_name': {'lsp.lua': 'lsp'}, - 'section_fmt': lambda name: ( - 'Lua module: vim.lsp' - if name.lower() == 'lsp' - else f'Lua module: vim.lsp.{name.lower()}'), - 'helptag_fmt': lambda name: ( - '*lsp-core*' - if name.lower() == 'lsp' - else f'*lsp-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( - f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else - # HACK. TODO(justinmk): class/structure support in lua2dox - '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else - f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'), - 'module_override': {}, - 'append_only': [], - }, - 'diagnostic': { - 'mode': 'lua', - 'filename': 'diagnostic.txt', - 'section_order': [ - 'diagnostic.lua', - ], - 'files': ['runtime/lua/vim/diagnostic.lua'], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'include_tables': False, - 'section_name': {'diagnostic.lua': 'diagnostic'}, - 'section_fmt': lambda _: 'Lua module: vim.diagnostic', - 'helptag_fmt': lambda _: '*diagnostic-api*', - 'fn_helptag_fmt': lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*', - 'module_override': {}, - 'append_only': [], - }, - 'treesitter': { - 'mode': 'lua', - 'filename': 'treesitter.txt', - 'section_order': [ - 'treesitter.lua', - 'language.lua', - 'query.lua', - 'highlighter.lua', - 'languagetree.lua', - 'dev.lua', - ], - 'files': [ - 'runtime/lua/vim/treesitter.lua', - 'runtime/lua/vim/treesitter/', - ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'section_name': {}, - 'section_fmt': lambda name: ( - 'Lua module: vim.treesitter' - if name.lower() == 'treesitter' - else f'Lua module: vim.treesitter.{name.lower()}'), - 'helptag_fmt': lambda name: ( - '*lua-treesitter-core*' - if name.lower() == 'treesitter' - else f'*lua-treesitter-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( - f'*vim.{fstem}.{name}()*' - if fstem == 'treesitter' - else f'*{name}()*' - if name[0].isupper() - else f'*vim.treesitter.{fstem}.{name}()*'), - 'module_override': {}, - 'append_only': [], - } -} - -param_exclude = ( - 'channel_id', -) - -# Annotations are displayed as line items after API function descriptions. -annotation_map = { - 'FUNC_API_FAST': '|api-fast|', - 'FUNC_API_TEXTLOCK': 'not allowed when |textlock| is active or in the |cmdwin|', - 'FUNC_API_TEXTLOCK_ALLOW_CMDWIN': 'not allowed when |textlock| is active', - 'FUNC_API_REMOTE_ONLY': '|RPC| only', - 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only', -} - - -def nvim_api_info() -> Tuple[int, bool]: - """Returns NVIM_API_LEVEL, NVIM_API_PRERELEASE from CMakeLists.txt""" - if not hasattr(nvim_api_info, 'LEVEL'): - script_dir = os.path.dirname(os.path.abspath(__file__)) - cmake_file_path = os.path.join(script_dir, '..', 'CMakeLists.txt') - with open(cmake_file_path, 'r') as cmake_file: - cmake_content = cmake_file.read() - - api_level_match = re.search(r'set\(NVIM_API_LEVEL (\d+)\)', cmake_content) - api_prerelease_match = re.search( - r'set\(NVIM_API_PRERELEASE (\w+)\)', cmake_content - ) - - if not api_level_match or not api_prerelease_match: - raise RuntimeError( - 'Could not find NVIM_API_LEVEL or NVIM_API_PRERELEASE in CMakeLists.txt' - ) - - nvim_api_info.LEVEL = int(api_level_match.group(1)) - nvim_api_info.PRERELEASE = api_prerelease_match.group(1).lower() == 'true' - - return nvim_api_info.LEVEL, nvim_api_info.PRERELEASE - - -# Raises an error with details about `o`, if `cond` is in object `o`, -# or if `cond()` is callable and returns True. -def debug_this(o, cond=True): - name = '' - if cond is False: - return - if not isinstance(o, str): - try: - name = o.nodeName - o = o.toprettyxml(indent=' ', newl='\n') - except Exception: - pass - if (cond is True - or (callable(cond) and cond()) - or (not callable(cond) and cond in o)): - raise RuntimeError('xxx: {}\n{}'.format(name, o)) - - -# Appends a message to a list which will be printed on exit. -def msg(s): - msgs.append(s) - - -# Print all collected messages. -def msg_report(): - for m in msgs: - print(f' {m}') - - -# Print collected messages, then throw an exception. -def fail(s): - msg_report() - raise RuntimeError(s) - - -def find_first(parent, name): - """Finds the first matching node within parent.""" - sub = parent.getElementsByTagName(name) - if not sub: - return None - return sub[0] - - -def iter_children(parent, name): - """Yields matching child nodes within parent.""" - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: - yield child - - -def get_child(parent, name): - """Gets the first matching child node.""" - for child in iter_children(parent, name): - return child - return None - - -def self_or_child(n): - """Gets the first child node, or self.""" - if len(n.childNodes) == 0: - return n - return n.childNodes[0] - - -def align_tags(line): - tag_regex = r"\s(\*.+?\*)(?:\s|$)" - tags = re.findall(tag_regex, line) - - if len(tags) > 0: - line = re.sub(tag_regex, "", line) - tags = " " + " ".join(tags) - line = line + (" " * (78 - len(line) - len(tags))) + tags - return line - - -def clean_lines(text): - """Removes superfluous lines. - - The beginning and end of the string is trimmed. Empty lines are collapsed. - """ - return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text)) - - -def is_blank(text): - return '' == clean_lines(text) - - -def get_text(n): - """Recursively concatenates all text in a node tree.""" - text = '' - if n.nodeType == n.TEXT_NODE: - return n.data - if n.nodeName == 'computeroutput': - for node in n.childNodes: - text += get_text(node) - return '`{}`'.format(text) - if n.nodeName == 'sp': # space, used in "programlisting" nodes - return ' ' - for node in n.childNodes: - if node.nodeType == node.TEXT_NODE: - text += node.data - elif node.nodeType == node.ELEMENT_NODE: - text += get_text(node) - return text - - -# Gets the length of the last line in `text`, excluding newline ("\n") char. -def len_lastline(text): - lastnl = text.rfind('\n') - if -1 == lastnl: - return len(text) - if '\n' == text[-1]: - return lastnl - (1 + text.rfind('\n', 0, lastnl)) - return len(text) - (1 + lastnl) - - -def len_lastline_withoutindent(text, indent): - n = len_lastline(text) - return (n - len(indent)) if n > len(indent) else 0 - - -# Returns True if node `n` contains only inline (not block-level) elements. -def is_inline(n): - # if len(n.childNodes) == 0: - # return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput' - for c in n.childNodes: - if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput': - return False - if not is_inline(c): - return False - return True - - -def doc_wrap(text, prefix='', width=70, func=False, indent=None): - """Wraps text to `width`. - - First line is prefixed with `prefix`, subsequent lines are aligned. - If `func` is True, only wrap at commas. - """ - if not width: - # return prefix + text - return text - - # Whitespace used to indent all lines except the first line. - indent = ' ' * len(prefix) if indent is None else indent - indent_only = (prefix == '' and indent is not None) - - if func: - lines = [prefix] - for part in text.split(', '): - if part[-1] not in ');': - part += ', ' - if len(lines[-1]) + len(part) > width: - lines.append(indent) - lines[-1] += part - return '\n'.join(x.rstrip() for x in lines).rstrip() - - # XXX: Dummy prefix to force TextWrapper() to wrap the first line. - if indent_only: - prefix = indent - - tw = textwrap.TextWrapper(break_long_words=False, - break_on_hyphens=False, - width=width, - initial_indent=prefix, - subsequent_indent=indent) - result = '\n'.join(tw.wrap(text.strip())) - - # XXX: Remove the dummy prefix. - if indent_only: - result = result[len(indent):] - - return result - - -def max_name(names): - if len(names) == 0: - return 0 - return max(len(name) for name in names) - - -def update_params_map(parent, ret_map, width=text_width - indentation): - """Updates `ret_map` with name:desc key-value pairs extracted - from Doxygen XML node `parent`. - """ - params = collections.OrderedDict() - for node in parent.childNodes: - if node.nodeType == node.TEXT_NODE: - continue - name_node = find_first(node, 'parametername') - if name_node.getAttribute('direction') == 'out': - continue - name = get_text(name_node) - if name in param_exclude: - continue - params[name.strip()] = node - max_name_len = max_name(params.keys()) + 8 - # `ret_map` is a name:desc map. - for name, node in params.items(): - desc = '' - desc_node = get_child(node, 'parameterdescription') - if desc_node: - desc = fmt_node_as_vimhelp( - desc_node, width=width, indent=(' ' * max_name_len)) - ret_map[name] = desc - return ret_map - - -def render_node(n, text, prefix='', indent='', width=text_width - indentation, - fmt_vimhelp=False): - """Renders a node as Vim help text, recursively traversing all descendants.""" - - def ind(s): - return s if fmt_vimhelp else '' - - text = '' - # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) - # text += (int(not space_preceding) * ' ') - - if n.nodeName == 'preformatted': - o = get_text(n) - ensure_nl = '' if o[-1] == '\n' else '\n' - if o[0:4] == 'lua\n': - text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1]) - elif o[0:4] == 'vim\n': - text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1]) - elif o[0:5] == 'help\n': - text += o[4:-1] - else: - text += '>{}{}\n<'.format(ensure_nl, o) - elif n.nodeName == 'programlisting': # codeblock (```) - o = get_text(n) - text += '>' - if 'filename' in n.attributes: - filename = n.attributes['filename'].value - text += filename.lstrip('.') - - text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4)) - elif is_inline(n): - text = doc_wrap(get_text(n), prefix=prefix, indent=indent, width=width) - elif n.nodeName == 'verbatim': - # TODO: currently we don't use this. The "[verbatim]" hint is there as - # a reminder that we must decide how to format this if we do use it. - text += ' [verbatim] {}'.format(get_text(n)) - elif n.nodeName == 'listitem': - for c in n.childNodes: - result = render_node( - c, - text, - indent=indent + (' ' * len(prefix)), - width=width - ) - if is_blank(result): - continue - text += indent + prefix + result - elif n.nodeName in ('para', 'heading'): - did_prefix = False - for c in n.childNodes: - if (is_inline(c) - and '' != get_text(c).strip() - and text - and ' ' != text[-1]): - text += ' ' - text += render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width) - did_prefix = True - elif n.nodeName == 'itemizedlist': - for c in n.childNodes: - text += '{}\n'.format(render_node(c, text, prefix='• ', - indent=indent, width=width)) - elif n.nodeName == 'orderedlist': - i = 1 - for c in n.childNodes: - if is_blank(get_text(c)): - text += '\n' - continue - text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i), - indent=indent, width=width)) - i = i + 1 - elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'): - text += ind(' ') - for c in n.childNodes: - if is_blank(render_node(c, text, prefix='• ', indent=' ', width=width)): - continue - text += render_node(c, text, prefix='• ', indent=' ', width=width) - # text += '\n' - elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'): - text += 'Warning:\n ' - for c in n.childNodes: - text += render_node(c, text, indent=' ', width=width) - text += '\n' - elif n.nodeName == 'simplesect' and 'see' == n.getAttribute('kind'): - text += ind(' ') - # Example: - # <simplesect kind="see"> - # <para>|autocommand|</para> - # </simplesect> - for c in n.childNodes: - text += render_node(c, text, prefix='• ', indent=' ', width=width) - elif n.nodeName == 'simplesect' and 'return' == n.getAttribute('kind'): - text += ind(' ') - for c in n.childNodes: - text += render_node(c, text, indent=' ', width=width) - elif n.nodeName == 'computeroutput': - return get_text(n) - else: - raise RuntimeError('unhandled node type: {}\n{}'.format( - n.nodeName, n.toprettyxml(indent=' ', newl='\n'))) - - return text - - -def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=False): - """Extracts a Doxygen XML <para> node to a map. - - Keys: - 'text': Text from this <para> element - 'note': List of @note strings - 'params': <parameterlist> map - 'return': List of @return strings - 'seealso': List of @see strings - 'xrefs': ? - """ - chunks = { - 'text': '', - 'note': [], - 'params': collections.OrderedDict(), - 'return': [], - 'seealso': [], - 'prerelease': False, - 'xrefs': [] - } - - # Ordered dict of ordered lists. - groups = collections.OrderedDict([ - ('note', []), - ('params', []), - ('return', []), - ('seealso', []), - ('xrefs', []), - ]) - - # Gather nodes into groups. Mostly this is because we want "parameterlist" - # nodes to appear together. - text = '' - kind = '' - if is_inline(parent): - # Flatten inline text from a tree of non-block nodes. - text = doc_wrap(render_node(parent, "", fmt_vimhelp=fmt_vimhelp), - indent=indent, width=width) - else: - prev = None # Previous node - for child in parent.childNodes: - if child.nodeName == 'parameterlist': - groups['params'].append(child) - elif child.nodeName == 'xrefsect': - groups['xrefs'].append(child) - elif child.nodeName == 'simplesect': - kind = child.getAttribute('kind') - if kind == 'note': - groups['note'].append(child) - elif kind == 'return': - groups['return'].append(child) - elif kind == 'see': - groups['seealso'].append(child) - elif kind == 'warning': - text += render_node(child, text, indent=indent, - width=width, fmt_vimhelp=fmt_vimhelp) - elif kind == 'since': - since_match = re.match(r'^(\d+)', get_text(child)) - since = int(since_match.group(1)) if since_match else 0 - NVIM_API_LEVEL, NVIM_API_PRERELEASE = nvim_api_info() - if since > NVIM_API_LEVEL or ( - since == NVIM_API_LEVEL and NVIM_API_PRERELEASE - ): - chunks['prerelease'] = True - else: - raise RuntimeError('unhandled simplesect: {}\n{}'.format( - child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) - else: - if (prev is not None - and is_inline(self_or_child(prev)) - and is_inline(self_or_child(child)) - and '' != get_text(self_or_child(child)).strip() - and text - and ' ' != text[-1]): - text += ' ' - - text += render_node(child, text, indent=indent, width=width, - fmt_vimhelp=fmt_vimhelp) - prev = child - - chunks['text'] += text - - # Generate map from the gathered items. - if len(groups['params']) > 0: - for child in groups['params']: - update_params_map(child, ret_map=chunks['params'], width=width) - for child in groups['note']: - chunks['note'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp).rstrip()) - for child in groups['return']: - chunks['return'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp)) - for child in groups['seealso']: - # Example: - # <simplesect kind="see"> - # <para>|autocommand|</para> - # </simplesect> - chunks['seealso'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp)) - - xrefs = set() - for child in groups['xrefs']: - # XXX: Add a space (or any char) to `title` here, otherwise xrefs - # ("Deprecated" section) acts very weird... - title = get_text(get_child(child, 'xreftitle')) + ' ' - xrefs.add(title) - xrefdesc = get_text(get_child(child, 'xrefdescription')) - chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title), - width=width) + '\n') - - return chunks, xrefs - -def is_program_listing(para): - """ - Return True if `para` contains a "programlisting" (i.e. a Markdown code - block ```). - - Sometimes a <para> element will have only a single "programlisting" child - node, but othertimes it will have extra whitespace around the - "programlisting" node. - - @param para XML <para> node - @return True if <para> is a programlisting - """ - - # Remove any child text nodes that are only whitespace - children = [ - n for n in para.childNodes - if n.nodeType != n.TEXT_NODE or n.data.strip() != '' - ] - - return len(children) == 1 and children[0].nodeName == 'programlisting' - -def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='', - fmt_vimhelp=False): - """Renders (nested) Doxygen <para> nodes as Vim :help text. - - NB: Blank lines in a docstring manifest as <para> tags. - """ - rendered_blocks = [] - - def fmt_param_doc(m): - """Renders a params map as Vim :help text.""" - max_name_len = max_name(m.keys()) + 4 - out = '' - for name, desc in m.items(): - if name == 'self': - continue - name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) - out += '{}{}\n'.format(name, desc) - return out.rstrip() - - def has_nonexcluded_params(m): - """Returns true if any of the given params has at least - one non-excluded item.""" - if fmt_param_doc(m) != '': - return True - - for child in parent.childNodes: - para, _ = para_as_map(child, indent, width, fmt_vimhelp) - - # 'programlisting' blocks are Markdown code blocks. Do not include - # these as a separate paragraph, but append to the last non-empty line - # in the text - if is_program_listing(child): - while rendered_blocks and rendered_blocks[-1] == '': - rendered_blocks.pop() - rendered_blocks[-1] += ' ' + para['text'] - continue - - # Generate text from the gathered items. - chunks = [para['text']] - notes = [" This API is pre-release (unstable)."] if para['prerelease'] else [] - notes += para['note'] - if len(notes) > 0: - chunks.append('\nNote: ~') - for s in notes: - chunks.append(s) - if len(para['params']) > 0 and has_nonexcluded_params(para['params']): - chunks.append('\nParameters: ~') - chunks.append(fmt_param_doc(para['params'])) - if len(para['return']) > 0: - chunks.append('\nReturn (multiple): ~' if len(para['return']) > 1 else '\nReturn: ~') - for s in para['return']: - chunks.append(s) - if len(para['seealso']) > 0: - chunks.append('\nSee also: ~') - for s in para['seealso']: - chunks.append(s) - for s in para['xrefs']: - chunks.append(s) - - rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) - rendered_blocks.append('') - - return clean_lines('\n'.join(rendered_blocks).strip()) - - -def extract_from_xml(filename, target, width, fmt_vimhelp): - """Extracts Doxygen info as maps without formatting the text. - - Returns two maps: - 1. Functions - 2. Deprecated functions - - The `fmt_vimhelp` variable controls some special cases for use by - fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) - """ - fns = {} # Map of func_name:docstring. - deprecated_fns = {} # Map of func_name:docstring. - - dom = minidom.parse(filename) - compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) - for member in dom.getElementsByTagName('memberdef'): - if member.getAttribute('static') == 'yes' or \ - member.getAttribute('kind') != 'function' or \ - member.getAttribute('prot') == 'private' or \ - get_text(get_child(member, 'name')).startswith('_'): - continue - - loc = find_first(member, 'location') - if 'private' in loc.getAttribute('file'): - continue - - return_type = get_text(get_child(member, 'type')) - if return_type == '': - continue - - if 'local_function' in return_type: # Special from lua2dox.lua. - continue - - istbl = return_type.startswith('table') # Special from lua2dox.lua. - if istbl and not CONFIG[target].get('include_tables', True): - continue - - if return_type.startswith(('ArrayOf', 'DictionaryOf')): - parts = return_type.strip('_').split('_') - return_type = '{}({})'.format(parts[0], ', '.join(parts[1:])) - - name = get_text(get_child(member, 'name')) - - annotations = get_text(get_child(member, 'argsstring')) - if annotations and ')' in annotations: - annotations = annotations.rsplit(')', 1)[-1].strip() - # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of - # non-void functions. Special-case void functions here. - if name == 'nvim_get_mode' and len(annotations) == 0: - annotations += 'FUNC_API_FAST' - annotations = filter(None, map(lambda x: annotation_map.get(x), - annotations.split())) - - params = [] - type_length = 0 - - for param in iter_children(member, 'param'): - param_type = get_text(get_child(param, 'type')).strip() - param_name = '' - declname = get_child(param, 'declname') - if declname: - param_name = get_text(declname).strip() - elif CONFIG[target]['mode'] == 'lua': - # XXX: this is what lua2dox gives us... - param_name = param_type - param_type = '' - - if param_name in param_exclude: - continue - - if fmt_vimhelp and param_type.endswith('*'): - param_type = param_type.strip('* ') - param_name = '*' + param_name - - type_length = max(type_length, len(param_type)) - params.append((param_type, param_name)) - - # Handle Object Oriented style functions here. - # We make sure they have "self" in the parameters, - # and a parent function - if return_type.startswith('function') \ - and len(return_type.split(' ')) >= 2 \ - and any(x[1] == 'self' for x in params): - split_return = return_type.split(' ') - name = f'{split_return[1]}:{name}' - params = [x for x in params if x[1] != 'self'] - - c_args = [] - for param_type, param_name in params: - c_args.append((' ' if fmt_vimhelp else '') + ( - '%s %s' % (param_type.ljust(type_length), param_name)).strip()) - - if not fmt_vimhelp: - pass - else: - fstem = '?' - if '.' in compoundname: - fstem = compoundname.split('.')[0] - fstem = CONFIG[target]['module_override'].get(fstem, fstem) - vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name, istbl) - - if 'fn_name_fmt' in CONFIG[target]: - name = CONFIG[target]['fn_name_fmt'](fstem, name) - - if istbl: - aopen, aclose = '', '' - else: - aopen, aclose = '(', ')' - - prefix = name + aopen - suffix = ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error', 'Arena', - 'lua_State')) + aclose - - if not fmt_vimhelp: - c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) - signature = prefix + suffix - else: - c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, - ',\n'.join(c_args)), - ' ') - - # Minimum 8 chars between signature and vimtag - lhs = (width - 8) - len(vimtag) - - if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(width) + '\n' - signature += doc_wrap(suffix, width=width, prefix=prefix, - func=True) - else: - signature = prefix + suffix - signature += vimtag.rjust(width - len(signature)) - - # Tracks `xrefsect` titles. As of this writing, used only for separating - # deprecated functions. - xrefs_all = set() - paras = [] - brief_desc = find_first(member, 'briefdescription') - if brief_desc: - for child in brief_desc.childNodes: - para, xrefs = para_as_map(child) - xrefs_all.update(xrefs) - - desc = find_first(member, 'detaileddescription') - if desc: - for child in desc.childNodes: - para, xrefs = para_as_map(child) - paras.append(para) - xrefs_all.update(xrefs) - log.debug( - textwrap.indent( - re.sub(r'\n\s*\n+', '\n', - desc.toprettyxml(indent=' ', newl='\n')), - ' ' * indentation)) - - fn = { - 'annotations': list(annotations), - 'signature': signature, - 'parameters': params, - 'parameters_doc': collections.OrderedDict(), - 'doc': [], - 'return': [], - 'seealso': [], - } - if fmt_vimhelp: - fn['desc_node'] = desc - fn['brief_desc_node'] = brief_desc - - for m in paras: - if 'text' in m: - if not m['text'] == '': - fn['doc'].append(m['text']) - if 'params' in m: - # Merge OrderedDicts. - fn['parameters_doc'].update(m['params']) - if 'return' in m and len(m['return']) > 0: - fn['return'] += m['return'] - if 'seealso' in m and len(m['seealso']) > 0: - fn['seealso'] += m['seealso'] - - if INCLUDE_C_DECL: - fn['c_decl'] = c_decl - - if 'Deprecated' in str(xrefs_all): - deprecated_fns[name] = fn - elif name.startswith(CONFIG[target]['fn_name_prefix']): - fns[name] = fn - - fns = collections.OrderedDict(sorted( - fns.items(), - key=lambda key_item_tuple: key_item_tuple[0].lower())) - deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items())) - return fns, deprecated_fns - - -def fmt_doxygen_xml_as_vimhelp(filename, target): - """Entrypoint for generating Vim :help from from Doxygen XML. - - Returns 2 items: - 1. Vim help text for functions found in `filename`. - 2. Vim help text for deprecated functions. - """ - fns_txt = {} # Map of func_name:vim-help-text. - deprecated_fns_txt = {} # Map of func_name:vim-help-text. - fns, _ = extract_from_xml(filename, target, text_width, True) - - for name, fn in fns.items(): - # Generate Vim :help for parameters. - if fn['desc_node']: - doc = fmt_node_as_vimhelp(fn['desc_node'], fmt_vimhelp=True) - if not doc and fn['brief_desc_node']: - doc = fmt_node_as_vimhelp(fn['brief_desc_node']) - if not doc and name.startswith("nvim__"): - continue - if not doc: - doc = 'TODO: Documentation' - - annotations = '\n'.join(fn['annotations']) - if annotations: - annotations = ('\n\nAttributes: ~\n' + - textwrap.indent(annotations, ' ')) - i = doc.rfind('Parameters: ~') - if i == -1: - doc += annotations - else: - doc = doc[:i] + annotations + '\n\n' + doc[i:] - - if INCLUDE_C_DECL: - doc += '\n\nC Declaration: ~\n>\n' - doc += fn['c_decl'] - doc += '\n<' - - func_doc = fn['signature'] + '\n' - func_doc += textwrap.indent(clean_lines(doc), ' ' * indentation) - - # Verbatim handling. - func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) - - split_lines = func_doc.split('\n') - start = 0 - while True: - try: - start = split_lines.index('>', start) - except ValueError: - break - - try: - end = split_lines.index('<', start) - except ValueError: - break - - split_lines[start + 1:end] = [ - (' ' + x).rstrip() - for x in textwrap.dedent( - "\n".join( - split_lines[start+1:end] - ) - ).split("\n") - ] - - start = end - - func_doc = "\n".join(map(align_tags, split_lines)) - - if (name.startswith(CONFIG[target]['fn_name_prefix']) - and name != "nvim_error_event"): - fns_txt[name] = func_doc - - return ('\n\n'.join(list(fns_txt.values())), - '\n\n'.join(list(deprecated_fns_txt.values()))) - - -def delete_lines_below(filename, tokenstr): - """Deletes all lines below the line containing `tokenstr`, the line itself, - and one line above it. - """ - lines = open(filename).readlines() - i = 0 - found = False - for i, line in enumerate(lines, 1): - if tokenstr in line: - found = True - break - if not found: - raise RuntimeError(f'not found: "{tokenstr}"') - i = max(0, i - 2) - with open(filename, 'wt') as fp: - fp.writelines(lines[0:i]) - - -def extract_defgroups(base: str, dom: Document): - '''Generate module-level (section) docs (@defgroup).''' - section_docs = {} - - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'group': - continue - - # Doxygen "@defgroup" directive. - groupname = get_text(find_first(compound, 'name')) - groupxml = os.path.join(base, '%s.xml' % - compound.getAttribute('refid')) - - group_parsed = minidom.parse(groupxml) - doc_list = [] - brief_desc = find_first(group_parsed, 'briefdescription') - if brief_desc: - for child in brief_desc.childNodes: - doc_list.append(fmt_node_as_vimhelp(child)) - - desc = find_first(group_parsed, 'detaileddescription') - if desc: - doc = fmt_node_as_vimhelp(desc) - - if doc: - doc_list.append(doc) - - # Can't use '.' in @defgroup, so convert to '--' - # "vim.json" => "vim-dot-json" - groupname = groupname.replace('-dot-', '.') - - section_docs[groupname] = "\n".join(doc_list) - - return section_docs - - -def main(doxygen_config, args): - """Generates: - - 1. Vim :help docs - 2. *.mpack files for use by API clients - - Doxygen is called and configured through stdin. - """ - for target in CONFIG: - if args.target is not None and target != args.target: - continue - mpack_file = os.path.join( - base_dir, 'runtime', 'doc', - CONFIG[target]['filename'].replace('.txt', '.mpack')) - if os.path.exists(mpack_file): - os.remove(mpack_file) - - output_dir = out_dir.format(target=target) - log.info("Generating documentation for %s in folder %s", - target, output_dir) - debug = args.log_level >= logging.DEBUG - p = subprocess.Popen( - ['doxygen', '-'], - stdin=subprocess.PIPE, - # silence warnings - # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found - stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL)) - p.communicate( - doxygen_config.format( - input=' '.join( - [f'"{file}"' for file in CONFIG[target]['files']]), - output=output_dir, - filter=filter_cmd, - file_patterns=CONFIG[target]['file_patterns']) - .encode('utf8') - ) - if p.returncode: - sys.exit(p.returncode) - - fn_map_full = {} # Collects all functions as each module is processed. - sections = {} - sep = '=' * text_width - - base = os.path.join(output_dir, 'xml') - dom = minidom.parse(os.path.join(base, 'index.xml')) - - section_docs = extract_defgroups(base, dom) - - # Generate docs for all functions in the current module. - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'file': - continue - - filename = get_text(find_first(compound, 'name')) - if filename.endswith('.c') or filename.endswith('.lua'): - xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) - # Extract unformatted (*.mpack). - fn_map, _ = extract_from_xml(xmlfile, target, 9999, False) - # Extract formatted (:help). - functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( - os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))), target) - - if not functions_text and not deprecated_text: - continue - else: - filename = os.path.basename(filename) - name = os.path.splitext(filename)[0].lower() - sectname = name.upper() if name == 'ui' else name.title() - sectname = CONFIG[target]['section_name'].get(filename, sectname) - title = CONFIG[target]['section_fmt'](sectname) - section_tag = CONFIG[target]['helptag_fmt'](sectname) - # Module/Section id matched against @defgroup. - # "*api-buffer*" => "api-buffer" - section_id = section_tag.strip('*') - - doc = '' - section_doc = section_docs.get(section_id) - if section_doc: - doc += '\n\n' + section_doc - - if functions_text: - doc += '\n\n' + functions_text - - if INCLUDE_DEPRECATED and deprecated_text: - doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n' - doc += deprecated_text - - if doc: - sections[filename] = (title, section_tag, doc) - fn_map_full.update(fn_map) - - if len(sections) == 0: - fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') - if len(sections) > len(CONFIG[target]['section_order']): - raise RuntimeError( - 'found new modules "{}"; update the "section_order" map'.format( - set(sections).difference(CONFIG[target]['section_order']))) - first_section_tag = sections[CONFIG[target]['section_order'][0]][1] - - docs = '' - - for filename in CONFIG[target]['section_order']: - try: - title, section_tag, section_doc = sections.pop(filename) - except KeyError: - msg(f'warning: empty docs, skipping (target={target}): {filename}') - msg(f' existing docs: {sections.keys()}') - continue - if filename not in CONFIG[target]['append_only']: - docs += sep - docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title))) - docs += section_doc - docs += '\n\n\n' - - docs = docs.rstrip() + '\n\n' - docs += f' vim:tw=78:ts=8:sw={indentation}:sts={indentation}:et:ft=help:norl:\n' - - doc_file = os.path.join(base_dir, 'runtime', 'doc', - CONFIG[target]['filename']) - - if os.path.exists(doc_file): - delete_lines_below(doc_file, first_section_tag) - with open(doc_file, 'ab') as fp: - fp.write(docs.encode('utf8')) - - fn_map_full = collections.OrderedDict(sorted(fn_map_full.items())) - with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) - - if not args.keep_tmpfiles: - shutil.rmtree(output_dir) - - msg_report() - - -def filter_source(filename, keep_tmpfiles): - output_dir = out_dir.format(target='lua2dox') - name, extension = os.path.splitext(filename) - if extension == '.lua': - args = [str(nvim), '-l', lua2dox, filename] + (['--outdir', output_dir] if keep_tmpfiles else []) - p = subprocess.run(args, stdout=subprocess.PIPE) - op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8')) - print(op) - else: - """Filters the source to fix macros that confuse Doxygen.""" - with open(filename, 'rt') as fp: - print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))', - lambda m: m.group(1)+'_'.join( - re.split(r'[^\w]+', m.group(2))), - fp.read(), flags=re.M)) - - -def parse_args(): - targets = ', '.join(CONFIG.keys()) - ap = argparse.ArgumentParser( - description="Generate helpdoc from source code") - ap.add_argument( - "--log-level", "-l", choices=LOG_LEVELS.keys(), - default=logging.getLevelName(logging.ERROR), help="Set log verbosity" - ) - ap.add_argument('source_filter', nargs='*', - help="Filter source file(s)") - ap.add_argument('-k', '--keep-tmpfiles', action='store_true', - help="Keep temporary files (tmp-xx-doc/ directories, including tmp-lua2dox-doc/ for lua2dox.lua quasi-C output)") - ap.add_argument('-t', '--target', - help=f'One of ({targets}), defaults to "all"') - return ap.parse_args() - - -Doxyfile = textwrap.dedent(''' - OUTPUT_DIRECTORY = {output} - INPUT = {input} - INPUT_ENCODING = UTF-8 - FILE_PATTERNS = {file_patterns} - RECURSIVE = YES - INPUT_FILTER = "{filter}" - EXCLUDE = - EXCLUDE_SYMLINKS = NO - EXCLUDE_PATTERNS = */private/* */health.lua */_*.lua - EXCLUDE_SYMBOLS = - EXTENSION_MAPPING = lua=C - EXTRACT_PRIVATE = NO - - GENERATE_HTML = NO - GENERATE_DOCSET = NO - GENERATE_HTMLHELP = NO - GENERATE_QHP = NO - GENERATE_TREEVIEW = NO - GENERATE_LATEX = NO - GENERATE_RTF = NO - GENERATE_MAN = NO - GENERATE_DOCBOOK = NO - GENERATE_AUTOGEN_DEF = NO - - GENERATE_XML = YES - XML_OUTPUT = xml - XML_PROGRAMLISTING = NO - - ENABLE_PREPROCESSING = YES - MACRO_EXPANSION = YES - EXPAND_ONLY_PREDEF = NO - MARKDOWN_SUPPORT = YES -''') - -if __name__ == "__main__": - args = parse_args() - print("Setting log level to %s" % args.log_level) - args.log_level = LOG_LEVELS[args.log_level] - log.setLevel(args.log_level) - log.addHandler(logging.StreamHandler()) - - # When invoked as a filter, args won't be passed, so use an env var. - if args.keep_tmpfiles: - os.environ['NVIM_KEEP_TMPFILES'] = '1' - keep_tmpfiles = ('NVIM_KEEP_TMPFILES' in os.environ) - - if len(args.source_filter) > 0: - filter_source(args.source_filter[0], keep_tmpfiles) - else: - main(Doxyfile, args) - -# vim: set ft=python ts=4 sw=4 tw=79 et : diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh index b0bf186f85..e8aac42a9c 100755 --- a/scripts/genappimage.sh +++ b/scripts/genappimage.sh @@ -26,8 +26,8 @@ APP_DIR="$APP.AppDir" ######################################################################## # Build and install nvim into the AppImage -make CMAKE_BUILD_TYPE="${NVIM_BUILD_TYPE}" CMAKE_EXTRA_FLAGS="-DCMAKE_INSTALL_PREFIX=${APP_DIR}/usr -DCMAKE_INSTALL_MANDIR=man" -make install +make CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" +cmake --install build --prefix="$APP_BUILD_DIR/${APP_DIR}/usr" ######################################################################## # Get helper functions and move to AppDir @@ -52,7 +52,7 @@ fi chmod +x "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage # metainfo is not packaged automatically by linuxdeploy -mkdir "$APP_DIR/usr/share/metainfo/" +mkdir -p "$APP_DIR/usr/share/metainfo/" cp "$ROOT_DIR/runtime/nvim.appdata.xml" "$APP_DIR/usr/share/metainfo/" cd "$APP_DIR" || exit diff --git a/scripts/lintcommit.lua b/scripts/lintcommit.lua index d2c8601c25..96f6304247 100644 --- a/scripts/lintcommit.lua +++ b/scripts/lintcommit.lua @@ -16,7 +16,7 @@ local _trace = false -- Print message local function p(s) vim.cmd('set verbose=1') - vim.api.nvim_echo({{s, ''}}, false, {}) + vim.api.nvim_echo({ { s, '' } }, false, {}) vim.cmd('set verbose=0') end @@ -25,7 +25,7 @@ end -- Prints `cmd` if `trace` is enabled. local function run(cmd, or_die) if _trace then - p('run: '..vim.inspect(cmd)) + p('run: ' .. vim.inspect(cmd)) end local rv = vim.trim(vim.fn.system(cmd)) or '' if vim.v.shell_error ~= 0 then @@ -43,14 +43,14 @@ end local function validate_commit(commit_message) -- Return nil if the commit message starts with "fixup" as it signifies it's -- a work in progress and shouldn't be linted yet. - if vim.startswith(commit_message, "fixup") then + if vim.startswith(commit_message, 'fixup') then return nil end - local commit_split = vim.split(commit_message, ":", {plain = true}) + local commit_split = vim.split(commit_message, ':', { plain = true }) -- Return nil if the type is vim-patch since most of the normal rules don't -- apply. - if commit_split[1] == "vim-patch" then + if commit_split[1] == 'vim-patch' then return nil end @@ -74,35 +74,41 @@ local function validate_commit(commit_message) if after_idx > vim.tbl_count(commit_split) then return [[Commit message does not include colons.]] end - local after_colon = commit_split[after_idx] + local after_colon = '' + while after_idx <= vim.tbl_count(commit_split) do + after_colon = after_colon .. commit_split[after_idx] + after_idx = after_idx + 1 + end -- Check if commit introduces a breaking change. - if vim.endswith(before_colon, "!") then + if vim.endswith(before_colon, '!') then before_colon = before_colon:sub(1, -2) end -- Check if type is correct - local type = vim.split(before_colon, "(", {plain = true})[1] - local allowed_types = {'build', 'ci', 'docs', 'feat', 'fix', 'perf', 'refactor', 'revert', 'test', 'vim-patch'} + local type = vim.split(before_colon, '(', { plain = true })[1] + local allowed_types = + { 'build', 'ci', 'docs', 'feat', 'fix', 'perf', 'refactor', 'revert', 'test', 'vim-patch' } if not vim.tbl_contains(allowed_types, type) then return string.format( [[Invalid commit type "%s". Allowed types are: %s. If none of these seem appropriate then use "fix"]], type, - vim.inspect(allowed_types)) + vim.inspect(allowed_types) + ) end -- Check if scope is appropriate - if before_colon:match("%(") then - local scope = vim.trim(commit_message:match("%((.-)%)")) + if before_colon:match('%(') then + local scope = vim.trim(commit_message:match('%((.-)%)')) if scope == '' then return [[Scope can't be empty]] end - if vim.startswith(scope, "nvim_") then - return [[Scope should be "api" instead of "nvim_..."]] + if vim.startswith(scope, 'nvim_') then + return [[Scope should be "api" instead of "nvim_..."]] end local alternative_scope = { @@ -119,17 +125,17 @@ local function validate_commit(commit_message) end -- Check that description doesn't end with a period - if vim.endswith(after_colon, ".") then + if vim.endswith(after_colon, '.') then return [[Description ends with a period (".").]] end -- Check that description starts with a whitespace. - if after_colon:sub(1,1) ~= " " then + if after_colon:sub(1, 1) ~= ' ' then return [[There should be a whitespace after the colon.]] end -- Check that description doesn't start with multiple whitespaces. - if after_colon:sub(1,2) == " " then + if after_colon:sub(1, 2) == ' ' then return [[There should only be one whitespace after the colon.]] end @@ -139,7 +145,7 @@ local function validate_commit(commit_message) end -- Check that description isn't just whitespaces - if vim.trim(after_colon) == "" then + if vim.trim(after_colon) == '' then return [[Description shouldn't be empty.]] end @@ -150,25 +156,25 @@ end function M.main(opt) _trace = not opt or not not opt.trace - local branch = run({'git', 'rev-parse', '--abbrev-ref', 'HEAD'}, true) + local branch = run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' }, true) -- TODO(justinmk): check $GITHUB_REF - local ancestor = run({'git', 'merge-base', 'origin/master', branch}) + local ancestor = run({ 'git', 'merge-base', 'origin/master', branch }) if not ancestor then - ancestor = run({'git', 'merge-base', 'upstream/master', branch}) + ancestor = run({ 'git', 'merge-base', 'upstream/master', branch }) end - local commits_str = run({'git', 'rev-list', ancestor..'..'..branch}, true) + local commits_str = run({ 'git', 'rev-list', ancestor .. '..' .. branch }, true) assert(commits_str) local commits = {} --- @type string[] - for substring in commits_str:gmatch("%S+") do - table.insert(commits, substring) + for substring in commits_str:gmatch('%S+') do + table.insert(commits, substring) end local failed = 0 for _, commit_id in ipairs(commits) do - local msg = run({'git', 'show', '-s', '--format=%s' , commit_id}) + local msg = run({ 'git', 'show', '-s', '--format=%s', commit_id }) if vim.v.shell_error ~= 0 then - p('Invalid commit-id: '..commit_id..'"') + p('Invalid commit-id: ' .. commit_id .. '"') else local invalid_msg = validate_commit(msg) if invalid_msg then @@ -179,20 +185,22 @@ function M.main(opt) p('\n') end - p(string.format([[ + p(string.format( + [[ Invalid commit message: "%s" Commit: %s %s ]], msg, commit_id, - invalid_msg)) + invalid_msg + )) end end end if failed > 0 then - p([[ + p([[ See also: https://github.com/neovim/neovim/blob/master/CONTRIBUTING.md#commit-messages @@ -239,11 +247,14 @@ function M._test() ['refactor(): empty scope'] = false, ['ci( ): whitespace as scope'] = false, ['ci: period at end of sentence.'] = false, + ['ci: period: at end of sentence.'] = false, ['ci: Capitalized first word'] = false, ['ci: UPPER_CASE First Word'] = true, ['unknown: using unknown type'] = false, ['feat: foo:bar'] = true, + ['feat: :foo:bar'] = true, ['feat(something): foo:bar'] = true, + ['feat(something): :foo:bar'] = true, ['feat(:grep): read from pipe'] = true, ['feat(:grep/:make): read from pipe'] = true, ['feat(:grep): foo:bar'] = true, @@ -252,7 +263,7 @@ function M._test() ['feat(:grep/:make)'] = false, ['feat(:grep'] = false, ['feat(:grep/:make'] = false, - ['ci: you\'re saying this commit message just goes on and on and on and on and on and on for way too long?'] = false, + ["ci: you're saying this commit message just goes on and on and on and on and on and on for way too long?"] = false, } local failed = 0 @@ -260,14 +271,15 @@ function M._test() local is_valid = (nil == validate_commit(message)) if is_valid ~= expected then failed = failed + 1 - p(string.format('[ FAIL ]: expected=%s, got=%s\n input: "%s"', expected, is_valid, message)) + p( + string.format('[ FAIL ]: expected=%s, got=%s\n input: "%s"', expected, is_valid, message) + ) end end if failed > 0 then os.exit(1) end - end --- @class LintcommitOptions diff --git a/scripts/lintdoc.lua b/scripts/lintdoc.lua new file mode 100755 index 0000000000..5e78b4cdcb --- /dev/null +++ b/scripts/lintdoc.lua @@ -0,0 +1,20 @@ +#!/usr/bin/env -S nvim -l + +-- Validate vimdoc files on $VIMRUNTIME/doc, and test generating HTML docs. +-- Checks for duplicate/missing tags, parse errors, and invalid links/urls/spellings. +-- See also `make lintdoc`. +-- +-- Usage: +-- $ nvim -l scripts/lintdoc.lua +-- $ make lintdoc + +print('Running lintdoc ...') + +-- gen_help_html requires :helptags to be generated on $VIMRUNTIME/doc +-- :helptags checks for duplicate tags. +vim.cmd [[ helptags ALL ]] + +require('scripts.gen_help_html').run_validate() +require('scripts.gen_help_html').test_gen() + +print('lintdoc PASSED.') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua deleted file mode 100644 index 1c8bc5a3cb..0000000000 --- a/scripts/lua2dox.lua +++ /dev/null @@ -1,475 +0,0 @@ ------------------------------------------------------------------------------ --- Copyright (C) 2012 by Simon Dales -- --- simon@purrsoft.co.uk -- --- -- --- This program is free software; you can redistribute it and/or modify -- --- it under the terms of the GNU General Public License as published by -- --- the Free Software Foundation; either version 2 of the License, or -- --- (at your option) any later version. -- --- -- --- This program is distributed in the hope that it will be useful, -- --- but WITHOUT ANY WARRANTY; without even the implied warranty of -- --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- --- GNU General Public License for more details. -- --- -- --- You should have received a copy of the GNU General Public License -- --- along with this program; if not, write to the -- --- Free Software Foundation, Inc., -- --- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -- ------------------------------------------------------------------------------ - ---[[! -Lua-to-Doxygen converter - -Partially from lua2dox -http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm - -RUNNING -------- - -This script "lua2dox.lua" gets called by "gen_vimdoc.py". - -DEBUGGING/DEVELOPING ---------------------- - -1. To debug, run gen_vimdoc.py with --keep-tmpfiles: - python3 scripts/gen_vimdoc.py -t treesitter --keep-tmpfiles -2. The filtered result will be written to ./tmp-lua2dox-doc/….lua.c - -Doxygen must be on your system. You can experiment like so: - -- Run "doxygen -g" to create a default Doxyfile. -- Then alter it to let it recognise lua. Add the following line: - FILE_PATTERNS = *.lua -- Then run "doxygen". - -The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. -It only has to be good enough for doxygen to see it as legal. - -One limitation is that each line is treated separately (except for long comments). -The implication is that class and function declarations must be on the same line. - -There is hack that will insert the "missing" close paren. -The effect is that you will get the function documented, but not with the parameter list you might expect. -]] - -local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' } - -local TAGGED_TYPES = { 'TSNode', 'LanguageTree' } - --- Document these as 'table' -local ALIAS_TYPES = { - 'Range', 'Range4', 'Range6', 'TSMetadata', - 'vim.filetype.add.filetypes', - 'vim.filetype.match.args' -} - -local debug_outfile = nil --- @type string? -local debug_output = {} - ---- write to stdout ---- @param str? string -local function write(str) - if not str then - return - end - - io.write(str) - if debug_outfile then - table.insert(debug_output, str) - end -end - ---- write to stdout ---- @param str? string -local function writeln(str) - write(str) - write('\n') -end - ---- an input file buffer ---- @class StreamRead ---- @field currentLine string? ---- @field contentsLen integer ---- @field currentLineNo integer ---- @field filecontents string[] -local StreamRead = {} - ---- @return StreamRead ---- @param filename string -function StreamRead.new(filename) - assert(filename, ('invalid file: %s'):format(filename)) - -- get lines from file - -- syphon lines to our table - local filecontents = {} --- @type string[] - for line in io.lines(filename) do - filecontents[#filecontents+1] = line - end - - return setmetatable({ - filecontents = filecontents, - contentsLen = #filecontents, - currentLineNo = 1, - }, { __index = StreamRead }) -end - --- get a line -function StreamRead:getLine() - if self.currentLine then - self.currentLine = nil - return self.currentLine - end - - -- get line - if self.currentLineNo <= self.contentsLen then - local line = self.filecontents[self.currentLineNo] - self.currentLineNo = self.currentLineNo + 1 - return line - end - - return '' -end - --- save line fragment ---- @param line_fragment string -function StreamRead:ungetLine(line_fragment) - self.currentLine = line_fragment -end - --- is it eof? -function StreamRead:eof() - return not self.currentLine and self.currentLineNo > self.contentsLen -end - --- input filter ---- @class Lua2DoxFilter -local Lua2DoxFilter = {} -setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter }) - ---- trim comment off end of string ---- ---- @param line string ---- @return string, string? -local function removeCommentFromLine(line) - local pos_comment = line:find('%-%-') - if not pos_comment then - return line - end - return line:sub(1, pos_comment - 1), line:sub(pos_comment) -end - ---- Processes "@…" directives in a docstring line. ---- ---- @param line string ---- @param generics table<string,string> ---- @return string? -local function process_magic(line, generics) - line = line:gsub('^%s+@', '@') - line = line:gsub('@package', '@private') - line = line:gsub('@nodoc', '@private') - - if not vim.startswith(line, '@') then -- it's a magic comment - return '/// ' .. line - end - - local magic = line:sub(2) - local magic_split = vim.split(magic, ' ', { plain = true }) - local directive = magic_split[1] - - if vim.list_contains({ - 'cast', 'diagnostic', 'overload', 'meta', 'type' - }, directive) then - -- Ignore LSP directives - return '// gg:"' .. line .. '"' - end - - if directive == 'defgroup' or directive == 'addtogroup' then - -- Can't use '.' in defgroup, so convert to '--' - return '/// @' .. magic:gsub('%.', '-dot-') - end - - if directive == 'generic' then - local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)') - if generic_type == '' then - generic_type = 'any' - end - generics[generic_name] = generic_type - return - end - - local type_index = 2 - - if directive == 'param' then - for _, type in ipairs(TYPES) do - magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2') - magic = - magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') - end - magic_split = vim.split(magic, ' ', { plain = true }) - type_index = 3 - elseif directive == 'return' then - for _, type in ipairs(TYPES) do - magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1') - magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1') - end - -- Remove first "#" comment char, if any. https://github.com/LuaLS/lua-language-server/wiki/Annotations#return - magic = magic:gsub('# ', '', 1) - -- handle the return of vim.spell.check - magic = magic:gsub('({.*}%[%])', '`%1`') - magic_split = vim.split(magic, ' ', { plain = true }) - end - - local ty = magic_split[type_index] - - if ty then - -- fix optional parameters - if magic_split[2]:find('%?$') then - if not ty:find('nil') then - ty = ty .. '|nil' - end - magic_split[2] = magic_split[2]:sub(1, -2) - end - - -- replace generic types - for k, v in pairs(generics) do - ty = ty:gsub(k, v) --- @type string - end - - for _, type in ipairs(TAGGED_TYPES) do - ty = ty:gsub(type, '|%1|') - end - - for _, type in ipairs(ALIAS_TYPES) do - ty = ty:gsub('^'..type..'$', 'table') --- @type string - end - - -- surround some types by () - for _, type in ipairs(TYPES) do - ty = ty - :gsub('^(' .. type .. '|nil):?$', '(%1)') - :gsub('^(' .. type .. '):?$', '(%1)') - end - - magic_split[type_index] = ty - - end - - magic = table.concat(magic_split, ' ') - - return '/// @' .. magic -end - ---- @param line string ---- @param in_stream StreamRead ---- @return string -local function process_block_comment(line, in_stream) - local comment_parts = {} --- @type string[] - local done --- @type boolean? - - while not done and not in_stream:eof() do - local thisComment --- @type string? - local closeSquare = line:find(']]') - if not closeSquare then -- need to look on another line - thisComment = line .. '\n' - line = in_stream:getLine() - else - thisComment = line:sub(1, closeSquare - 1) - done = true - - -- unget the tail of the line - -- in most cases it's empty. This may make us less efficient but - -- easier to program - in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2))) - end - comment_parts[#comment_parts+1] = thisComment - end - - local comment = table.concat(comment_parts) - - if comment:sub(1, 1) == '@' then -- it's a long magic comment - return '/*' .. comment .. '*/ ' - end - - -- discard - return '/* zz:' .. comment .. '*/ ' -end - ---- @param line string ---- @return string -local function process_function_header(line) - local pos_fn = assert(line:find('function')) - -- we've got a function - local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8))) - - if fn:sub(1, 1) == '(' then - -- it's an anonymous function - return '// ZZ: '..line - end - -- fn has a name, so is interesting - - -- want to fix for iffy declarations - if fn:find('[%({]') then - -- we might have a missing close paren - if not fn:find('%)') then - fn = fn .. ' ___MissingCloseParenHere___)' - end - end - - -- Big hax - if fn:find(':') then - fn = fn:gsub(':', '.', 1) - - local paren_start = fn:find('(', 1, true) - local paren_finish = fn:find(')', 1, true) - - -- Nothing in between the parens - local comma --- @type string - if paren_finish == paren_start + 1 then - comma = '' - else - comma = ', ' - end - - fn = fn:sub(1, paren_start) - .. 'self' - .. comma - .. fn:sub(paren_start + 1) - end - - if line:match('local') then - -- Special: tell gen_vimdoc.py this is a local function. - return 'local_function ' .. fn .. '{}' - end - - -- add vanilla function - return 'function ' .. fn .. '{}' -end - ---- @param line string ---- @param in_stream StreamRead ---- @param generics table<string,string>> ---- @return string? -local function process_line(line, in_stream, generics) - local line_raw = line - line = vim.trim(line) - - if vim.startswith(line, '---') then - return process_magic(line:sub(4), generics) - end - - if vim.startswith(line, '--'..'[[') then -- it's a long comment - return process_block_comment(line:sub(5), in_stream) - end - - -- Hax... I'm sorry - -- M.fun = vim.memoize(function(...) - -- -> - -- function M.fun(...) - line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') - - if line:find('^function') or line:find('^local%s+function') then - return process_function_header(line) - end - - if not line:match('^local') then - local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=') - if v and v:match('%.') then - -- Special: this lets gen_vimdoc.py handle tables. - return 'table '..v..'() {}' - end - end - - if #line > 0 then -- we don't know what this line means, so just comment it out - return '// zz: ' .. line - end - - return '' -end - --- Processes the file and writes filtered output to stdout. ----@param filename string -function Lua2DoxFilter:filter(filename) - local in_stream = StreamRead.new(filename) - - local generics = {} --- @type table<string,string> - - while not in_stream:eof() do - local line = in_stream:getLine() - - local out_line = process_line(line, in_stream, generics) - - if not vim.startswith(vim.trim(line), '---') then - generics = {} - end - - if out_line then - writeln(out_line) - end - end -end - ---- @class TApp ---- @field timestamp string|osdate ---- @field name string ---- @field version string ---- @field copyright string ---- this application -local TApp = { - timestamp = os.date('%c %Z', os.time()), - name = 'Lua2DoX', - version = '0.2 20130128', - copyright = 'Copyright (c) Simon Dales 2012-13' -} - -setmetatable(TApp, { __index = TApp }) - -function TApp:getRunStamp() - return self.name .. ' (' .. self.version .. ') ' .. self.timestamp -end - -function TApp:getVersion() - return self.name .. ' (' .. self.version .. ') ' -end - ---main - -if arg[1] == '--help' then - writeln(TApp:getVersion()) - writeln(TApp.copyright) - writeln([[ - run as: - nvim -l scripts/lua2dox.lua <param> - -------------- - Param: - <filename> : interprets filename - --version : show version/copyright info - --help : this help text]]) -elseif arg[1] == '--version' then - writeln(TApp:getVersion()) - writeln(TApp.copyright) -else -- It's a filter. - local filename = arg[1] - - if arg[2] == '--outdir' then - local outdir = arg[3] - if type(outdir) ~= 'string' or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) then - error(('invalid --outdir: "%s"'):format(tostring(outdir))) - end - vim.fn.mkdir(outdir, 'p') - debug_outfile = string.format('%s/%s.c', outdir, vim.fs.basename(filename)) - end - - Lua2DoxFilter:filter(filename) - - -- output the tail - writeln('// #######################') - writeln('// app run:' .. TApp:getRunStamp()) - writeln('// #######################') - writeln() - - if debug_outfile then - local f = assert(io.open(debug_outfile, 'w')) - f:write(table.concat(debug_output)) - f:close() - end -end diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua new file mode 100644 index 0000000000..ca26c70156 --- /dev/null +++ b/scripts/luacats_grammar.lua @@ -0,0 +1,184 @@ +--[[! +LPEG grammar for LuaCATS +]] + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local Ct, Cg = lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local ws = rep1(S(' \t')) +local fill = opt(ws) + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local num = R('09') +local ident = letter * rep(letter + num + S '-.') +local string_single = P "'" * rep(any - P "'") * P "'" +local string_double = P('"') * rep(any - P('"')) * P('"') + +local literal = (string_single + string_double + (opt(P('-')) * num) + P('false') + P('true')) + +local lname = (ident + P('...')) * opt(P('?')) + +--- @param x string +local function Pf(x) + return fill * P(x) * fill +end + +--- @param x string +local function Sf(x) + return fill * S(x) * fill +end + +--- @param x vim.lpeg.Pattern +local function paren(x) + return Pf('(') * x * fill * P(')') +end + +--- @param x vim.lpeg.Pattern +local function parenOpt(x) + return paren(x) + x +end + +--- @param x vim.lpeg.Pattern +local function comma1(x) + return parenOpt(x * rep(Pf(',') * x)) +end + +--- @param x vim.lpeg.Pattern +local function comma(x) + return opt(comma1(x)) +end + +--- @type table<string,vim.lpeg.Pattern> +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +local colon = Pf(':') +local opt_exact = opt(Cg(Pf('(exact)'), 'access')) +local access = P('private') + P('protected') + P('package') +local caccess = Cg(access, 'access') +local desc_delim = Sf '#:' + ws +local desc = Cg(rep(any), 'desc') +local opt_desc = opt(desc_delim * desc) +local cname = Cg(ident, 'name') +local opt_parent = opt(colon * Cg(ident, 'parent')) + +--- @class nvim.luacats.Param +--- @field kind 'param' +--- @field name string +--- @field type string +--- @field desc? string + +--- @class nvim.luacats.Return +--- @field kind 'return' +--- @field [integer] { type: string, name?: string} +--- @field desc? string + +--- @class nvim.luacats.Generic +--- @field kind 'generic' +--- @field name string +--- @field type? string + +--- @class nvim.luacats.Class +--- @field kind 'class' +--- @field name string +--- @field parent? string +--- @field access? 'private'|'protected'|'package' + +--- @class nvim.luacats.Field +--- @field kind 'field' +--- @field name string +--- @field type string +--- @field desc? string +--- @field access? 'private'|'protected'|'package' + +--- @class nvim.luacats.Note +--- @field desc? string + +--- @alias nvim.luacats.grammar.result +--- | nvim.luacats.Param +--- | nvim.luacats.Return +--- | nvim.luacats.Generic +--- | nvim.luacats.Class +--- | nvim.luacats.Field +--- | nvim.luacats.Note + +--- @class nvim.luacats.grammar +--- @field match fun(self, input: string): nvim.luacats.grammar.result? + +local function annot(nm, pat) + if type(nm) == 'string' then + nm = P(nm) + end + if pat then + return Ct(Cg(P(nm), 'kind') * fill * pat) + end + return Ct(Cg(P(nm), 'kind')) +end + +local grammar = P { + rep1(P('@') * (v.ats + v.ext_ats)), + + ats = annot('param', Cg(lname, 'name') * ws * v.ctype * opt_desc) + + annot('return', comma1(Ct(v.ctype * opt(ws * cname))) * opt_desc) + + annot('type', comma1(Ct(v.ctype)) * opt_desc) + + annot('cast', cname * ws * opt(Sf('+-')) * v.ctype) + + annot('generic', cname * opt(colon * v.ctype)) + + annot('class', opt_exact * opt(paren(caccess)) * fill * cname * opt_parent) + + annot('field', opt(caccess * ws) * v.field_name * ws * v.ctype * opt_desc) + + annot('operator', cname * opt(paren(Cg(v.ltype, 'argtype'))) * colon * v.ctype) + + annot(access) + + annot('deprecated') + + annot('alias', cname * opt(ws * v.ctype)) + + annot('enum', cname) + + annot('overload', v.ctype) + + annot('see', opt(desc_delim) * desc) + + annot('diagnostic', opt(desc_delim) * desc) + + annot('meta'), + + --- Custom extensions + ext_ats = ( + annot('note', desc) + + annot('since', desc) + + annot('nodoc') + + annot('inlinedoc') + + annot('brief', desc) + ), + + field_name = Cg(lname + (v.ty_index * opt(P('?'))), 'name'), + + ctype = parenOpt(Cg(v.ltype, 'type')), + ltype = parenOpt(v.ty_union), + + ty_union = v.ty_opt * rep(Pf('|') * v.ty_opt), + ty = v.ty_fun + ident + v.ty_table + literal + paren(v.ty), + ty_param = Pf('<') * comma1(v.ltype) * fill * P('>'), + ty_opt = v.ty * opt(v.ty_param) * opt(P('[]')) * opt(P('?')), + ty_index = (Pf('[') * (v.ltype + ident + rep1(num)) * fill * P(']')), + table_key = v.ty_index + lname, + table_elem = v.table_key * colon * v.ltype, + ty_table = Pf('{') * comma1(v.table_elem) * fill * P('}'), + fun_param = lname * opt(colon * v.ltype), + ty_fun = Pf('fun') * paren(comma(lname * opt(colon * v.ltype))) * opt(colon * comma1(v.ltype)), +} + +return grammar --[[@as nvim.luacats.grammar]] diff --git a/scripts/luacats_parser.lua b/scripts/luacats_parser.lua new file mode 100644 index 0000000000..cd671fb9dc --- /dev/null +++ b/scripts/luacats_parser.lua @@ -0,0 +1,537 @@ +local luacats_grammar = require('scripts.luacats_grammar') + +--- @class nvim.luacats.parser.param +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.note +--- @field desc string + +--- @class nvim.luacats.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.luacats.parser.alias +--- @field kind 'alias' +--- @field type string[] +--- @field desc string + +--- @class nvim.luacats.parser.fun +--- @field name string +--- @field params nvim.luacats.parser.param[] +--- @field returns nvim.luacats.parser.return[] +--- @field desc string +--- @field access? 'private'|'package'|'protected' +--- @field class? string +--- @field module? string +--- @field modvar? string +--- @field classvar? string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field generics? table<string,string> +--- @field table? true +--- @field notes? nvim.luacats.parser.note[] +--- @field see? nvim.luacats.parser.note[] + +--- @class nvim.luacats.parser.field +--- @field name string +--- @field type string +--- @field desc string +--- @field access? 'private'|'package'|'protected' + +--- @class nvim.luacats.parser.class +--- @field kind 'class' +--- @field parent? string +--- @field name string +--- @field desc string +--- @field nodoc? true +--- @field inlinedoc? true +--- @field access? 'private'|'package'|'protected' +--- @field fields nvim.luacats.parser.field[] +--- @field notes? string[] + +--- @class nvim.luacats.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.luacats.parser.obj +--- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.luacats.parser.obj +--- | nvim.luacats.parser.class +--- | nvim.luacats.parser.fun +--- | nvim.luacats.parser.brief +--- | nvim.luacats.parser.alias + +-- Remove this when we document classes properly +--- Some doc lines have the form: +--- param name some.complex.type (table) description +--- if so then transform the line to remove the complex type: +--- param name (table) description +--- @param line string +local function use_type_alt(line) + for _, type in ipairs({ 'table', 'function' }) do + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') + + line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1') + line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1') + line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1') + end + return line +end + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.luacats.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.luacats.parser.State +local function process_doc_line(line, state) + line = line:sub(4):gsub('^%s+@', '@') + line = use_type_alt(line) + + local parsed = luacats_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + local kind = parsed.kind + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'class' then + --- @cast parsed nvim.luacats.Class + cur_obj.kind = 'class' + cur_obj.name = parsed.name + cur_obj.parent = parsed.parent + cur_obj.access = parsed.access + cur_obj.desc = state.doc_lines and table.concat(state.doc_lines, '\n') or nil + state.doc_lines = nil + cur_obj.fields = {} + elseif kind == 'field' then + --- @cast parsed nvim.luacats.Field + parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil + if parsed.desc then + parsed.desc = vim.trim(parsed.desc) + end + table.insert(cur_obj.fields, parsed) + state.doc_lines = nil + elseif kind == 'operator' then + parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil + if parsed.desc then + parsed.desc = vim.trim(parsed.desc) + end + table.insert(cur_obj.fields, parsed) + state.doc_lines = nil + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + if vim.endswith(parsed.name, '?') then + parsed.name = parsed.name:sub(1, -2) + parsed.type = parsed.type .. '?' + end + state.last_doc_item = { + name = parsed.name, + type = parsed.type, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = cur_obj.returns or {} + for _, t in ipairs(parsed) do + table.insert(cur_obj.returns, { + name = t.name, + type = t.type, + desc = parsed.desc, + }) + end + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[#cur_obj.returns] + elseif kind == 'private' then + cur_obj.access = 'private' + elseif kind == 'package' then + cur_obj.access = 'package' + elseif kind == 'protected' then + cur_obj.access = 'protected' + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'inlinedoc' then + cur_obj.inlinedoc = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + elseif kind == 'type' then + cur_obj.desc = parsed.desc + parsed.desc = nil + parsed.kind = nil + cur_obj.type = parsed + elseif kind == 'alias' then + state.cur_obj = { + kind = 'alias', + desc = parsed.desc, + } + elseif kind == 'enum' then + -- TODO + state.doc_lines = nil + elseif + vim.tbl_contains({ + 'diagnostic', + 'cast', + 'overload', + 'meta', + }, kind) + then + -- Ignore + return + elseif kind == 'generic' then + cur_obj.generics = cur_obj.generics or {} + cur_obj.generics[parsed.name] = parsed.type or 'any' + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param fun nvim.luacats.parser.fun +--- @return nvim.luacats.parser.field +local function fun2field(fun) + local parts = { 'fun(' } + for _, p in ipairs(fun.params or {}) do + parts[#parts + 1] = string.format('%s: %s', p.name, p.type) + end + parts[#parts + 1] = ')' + if fun.returns then + parts[#parts + 1] = ': ' + local tys = {} --- @type string[] + for _, p in ipairs(fun.returns) do + tys[#tys + 1] = p.type + end + parts[#parts + 1] = table.concat(tys, ', ') + end + + return { + name = fun.name, + type = table.concat(parts, ''), + access = fun.access, + desc = fun.desc, + } +end + +--- Function to normalize known form for declaring functions and normalize into a more standard +--- form. +--- @param line string +--- @return string +local function filter_decl(line) + -- M.fun = vim._memoize(function(...) + -- -> + -- function M.fun(...) + line = line:gsub('^local (.+) = .*_memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)') + line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') + return line +end + +--- @param line string +--- @param state nvim.luacats.parser.State +--- @param classes table<string,nvim.luacats.parser.class> +--- @param classvars table<string,string> +--- @param has_indent boolean +local function process_lua_line(line, state, classes, classvars, has_indent) + line = filter_decl(line) + + if state.cur_obj and state.cur_obj.kind == 'class' then + local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=') + if nm then + classvars[nm] = state.cur_obj.name + end + return + end + + do + local parent_tbl, sep, fun_or_meth_nm = + line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(') + if parent_tbl then + -- Have a decl. Ensure cur_obj + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + -- Match `Class:foo` methods for defined classes + local class = classvars[parent_tbl] + if class then + --- @cast cur_obj nvim.luacats.parser.fun + cur_obj.name = fun_or_meth_nm + cur_obj.class = class + cur_obj.classvar = parent_tbl + -- Add self param to methods + if sep == ':' then + cur_obj.params = cur_obj.params or {} + table.insert(cur_obj.params, 1, { + name = 'self', + type = class, + }) + end + + -- Add method as the field to the class + table.insert(classes[class].fields, fun2field(cur_obj)) + return + end + + -- Match `M.foo` + if cur_obj and parent_tbl == cur_obj.modvar then + cur_obj.name = fun_or_meth_nm + return + end + end + end + + do + -- Handle: `function A.B.C.foo(...)` + local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(') + if fn_nm then + state.cur_obj = state.cur_obj or {} + state.cur_obj.name = fn_nm + return + end + end + + do + -- Handle: `M.foo = {...}` where `M` is the modvar + local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=') + if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + do + -- Handle: `foo = {...}` + local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=') + if tbl_nm and not has_indent then + state.cur_obj = state.cur_obj or {} + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + do + -- Handle: `vim.foo = {...}` + local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=') + if state.cur_obj and tbl_nm and not has_indent then + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + if state.cur_obj then + if line:find('^%s*%-%- luacheck:') then + state.cur_obj = nil + elseif line:find('^%s*local%s+') then + state.cur_obj = nil + elseif line:find('^%s*return%s+') then + state.cur_obj = nil + elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then + state.cur_obj = nil + end + end +end + +--- Determine the table name used to export functions of a module +--- Usually this is `M`. +--- @param str string +--- @return string? +local function determine_modvar(str) + local modvar --- @type string? + for line in vim.gsplit(str, '\n') do + do + --- @type string? + local m = line:match('^return%s+([a-zA-Z_]+)') + if m then + modvar = m + end + end + do + --- @type string? + local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),') + if m then + modvar = m + end + end + end + return modvar +end + +--- @param obj nvim.luacats.parser.obj +--- @param funs nvim.luacats.parser.fun[] +--- @param classes table<string,nvim.luacats.parser.class> +--- @param briefs string[] +--- @param uncommitted nvim.luacats.parser.obj[] +local function commit_obj(obj, classes, funs, briefs, uncommitted) + local commit = false + if obj.kind == 'class' then + --- @cast obj nvim.luacats.parser.class + if not classes[obj.name] then + classes[obj.name] = obj + commit = true + end + elseif obj.kind == 'alias' then + -- Just pretend + commit = true + elseif obj.kind == 'brief' then + --- @cast obj nvim.luacats.parser.brief` + briefs[#briefs + 1] = obj.desc + commit = true + else + --- @cast obj nvim.luacats.parser.fun` + if obj.name then + funs[#funs + 1] = obj + commit = true + end + end + if not commit then + table.insert(uncommitted, obj) + end + return commit +end + +--- @param filename string +--- @param uncommitted nvim.luacats.parser.obj[] +-- luacheck: no unused +local function dump_uncommitted(filename, uncommitted) + local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt' + if #uncommitted > 0 then + print(string.format('Could not commit %d objects in %s', #uncommitted, filename)) + vim.fn.mkdir(assert(vim.fs.dirname(out_path)), 'p') + local f = assert(io.open(out_path, 'w')) + for i, x in ipairs(uncommitted) do + f:write(i) + f:write(': ') + f:write(vim.inspect(x)) + f:write('\n') + end + f:close() + else + vim.fn.delete(out_path) + end +end + +local M = {} + +function M.parse_str(str, filename) + local funs = {} --- @type nvim.luacats.parser.fun[] + local classes = {} --- @type table<string,nvim.luacats.parser.class> + local briefs = {} --- @type string[] + + local mod_return = determine_modvar(str) + + --- @type string + local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename + module = module:gsub('/', '.') + + local classvars = {} --- @type table<string,string> + + local state = {} --- @type nvim.luacats.parser.State + + -- Keep track of any partial objects we don't commit + local uncommitted = {} --- @type nvim.luacats.parser.obj[] + + for line in vim.gsplit(str, '\n') do + local has_indent = line:match('^%s+') ~= nil + line = vim.trim(line) + if vim.startswith(line, '---') then + process_doc_line(line, state) + else + add_doc_lines_to_obj(state) + + if state.cur_obj then + state.cur_obj.modvar = mod_return + state.cur_obj.module = module + end + + process_lua_line(line, state, classes, classvars, has_indent) + + -- Commit the object + local cur_obj = state.cur_obj + if cur_obj then + if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then + --- @diagnostic disable-next-line:inject-field + cur_obj.line = line + end + end + + state = {} + end + end + + -- dump_uncommitted(filename, uncommitted) + + return classes, funs, briefs, uncommitted +end + +--- @param filename string +function M.parse(filename) + local f = assert(io.open(filename, 'r')) + local txt = f:read('*all') + f:close() + + return M.parse_str(txt, filename) +end + +return M diff --git a/scripts/text_utils.lua b/scripts/text_utils.lua new file mode 100644 index 0000000000..937408c546 --- /dev/null +++ b/scripts/text_utils.lua @@ -0,0 +1,361 @@ +local fmt = string.format + +--- @class nvim.text_utils.MDNode +--- @field [integer] nvim.text_utils.MDNode +--- @field type string +--- @field text? string + +local INDENTATION = 4 + +local NBSP = string.char(160) + +local M = {} + +local function contains(t, xs) + return vim.tbl_contains(xs, t) +end + +--- @param txt string +--- @param srow integer +--- @param scol integer +--- @param erow? integer +--- @param ecol? integer +--- @return string +local function slice_text(txt, srow, scol, erow, ecol) + local lines = vim.split(txt, '\n') + + if srow == erow then + return lines[srow + 1]:sub(scol + 1, ecol) + end + + if erow then + -- Trim the end + for _ = erow + 2, #lines do + table.remove(lines, #lines) + end + end + + -- Trim the start + for _ = 1, srow do + table.remove(lines, 1) + end + + lines[1] = lines[1]:sub(scol + 1) + lines[#lines] = lines[#lines]:sub(1, ecol) + + return table.concat(lines, '\n') +end + +--- @param text string +--- @return nvim.text_utils.MDNode +local function parse_md_inline(text) + local parser = vim.treesitter.languagetree.new(text, 'markdown_inline') + local root = parser:parse(true)[1]:root() + + --- @param node TSNode + --- @return nvim.text_utils.MDNode? + local function extract(node) + local ntype = node:type() + + if ntype:match('^%p$') then + return + end + + --- @type table<any,any> + local ret = { type = ntype } + ret.text = vim.treesitter.get_node_text(node, text) + + local row, col = 0, 0 + + for child, child_field in node:iter_children() do + local e = extract(child) + if e and ntype == 'inline' then + local srow, scol = child:start() + if (srow == row and scol > col) or srow > row then + local t = slice_text(ret.text, row, col, srow, scol) + if t and t ~= '' then + table.insert(ret, { type = 'text', j = true, text = t }) + end + end + row, col = child:end_() + end + + if child_field then + ret[child_field] = e + else + table.insert(ret, e) + end + end + + if ntype == 'inline' and (row > 0 or col > 0) then + local t = slice_text(ret.text, row, col) + if t and t ~= '' then + table.insert(ret, { type = 'text', text = t }) + end + end + + return ret + end + + return extract(root) or {} +end + +--- @param text string +--- @return nvim.text_utils.MDNode +local function parse_md(text) + local parser = vim.treesitter.languagetree.new(text, 'markdown', { + injections = { markdown = '' }, + }) + + local root = parser:parse(true)[1]:root() + + local EXCLUDE_TEXT_TYPE = { + list = true, + list_item = true, + section = true, + document = true, + fenced_code_block = true, + fenced_code_block_delimiter = true, + } + + --- @param node TSNode + --- @return nvim.text_utils.MDNode? + local function extract(node) + local ntype = node:type() + + if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then + return + end + + --- @type table<any,any> + local ret = { type = ntype } + + if not EXCLUDE_TEXT_TYPE[ntype] then + ret.text = vim.treesitter.get_node_text(node, text) + end + + if ntype == 'inline' then + ret = parse_md_inline(ret.text) + end + + for child, child_field in node:iter_children() do + local e = extract(child) + if child_field then + ret[child_field] = e + else + table.insert(ret, e) + end + end + + return ret + end + + return extract(root) or {} +end + +--- @param x string +--- @param start_indent integer +--- @param indent integer +--- @param text_width integer +--- @return string +function M.wrap(x, start_indent, indent, text_width) + local words = vim.split(vim.trim(x), '%s+') + local parts = { string.rep(' ', start_indent) } --- @type string[] + local count = indent + + for i, w in ipairs(words) do + if count > indent and count + #w > text_width - 1 then + parts[#parts + 1] = '\n' + parts[#parts + 1] = string.rep(' ', indent) + count = indent + elseif i ~= 1 then + parts[#parts + 1] = ' ' + count = count + 1 + end + count = count + #w + parts[#parts + 1] = w + end + + return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', '')) +end + +--- @param node nvim.text_utils.MDNode +--- @param start_indent integer +--- @param indent integer +--- @param text_width integer +--- @param level integer +--- @return string[] +local function render_md(node, start_indent, indent, text_width, level, is_list) + local parts = {} --- @type string[] + + -- For debugging + local add_tag = false + -- local add_tag = true + + local ntype = node.type + + if add_tag then + parts[#parts + 1] = '<' .. ntype .. '>' + end + + if ntype == 'text' then + parts[#parts + 1] = node.text + elseif ntype == 'html_tag' then + error('html_tag: ' .. node.text) + elseif ntype == 'inline_link' then + vim.list_extend(parts, { '*', node[1].text, '*' }) + elseif ntype == 'shortcut_link' then + if node[1].text:find('^<.*>$') then + parts[#parts + 1] = node[1].text + else + vim.list_extend(parts, { '|', node[1].text, '|' }) + end + elseif ntype == 'backslash_escape' then + parts[#parts + 1] = node.text + elseif ntype == 'emphasis' then + parts[#parts + 1] = node.text:sub(2, -2) + elseif ntype == 'code_span' then + vim.list_extend(parts, { '`', node.text:sub(2, -2):gsub(' ', NBSP), '`' }) + elseif ntype == 'inline' then + if #node == 0 then + local text = assert(node.text) + parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width) + else + for _, child in ipairs(node) do + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1)) + end + end + elseif ntype == 'paragraph' then + local pparts = {} + for _, child in ipairs(node) do + vim.list_extend(pparts, render_md(child, start_indent, indent, text_width, level + 1)) + end + parts[#parts + 1] = M.wrap(table.concat(pparts), start_indent, indent, text_width) + parts[#parts + 1] = '\n' + elseif ntype == 'code_fence_content' then + local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n') + + local cindent = indent + INDENTATION + if level > 3 then + -- The tree-sitter markdown parser doesn't parse the code blocks indents + -- correctly in lists. Fudge it! + lines[1] = ' ' .. lines[1] -- ¯\_(ツ)_/¯ + cindent = indent - level + local _, initial_indent = lines[1]:find('^%s*') + initial_indent = initial_indent + cindent + if initial_indent < indent then + cindent = indent - INDENTATION + end + end + + for _, l in ipairs(lines) do + if #l > 0 then + parts[#parts + 1] = string.rep(' ', cindent) + parts[#parts + 1] = l + end + parts[#parts + 1] = '\n' + end + elseif ntype == 'fenced_code_block' then + parts[#parts + 1] = '>' + for _, child in ipairs(node) do + if child.type == 'info_string' then + parts[#parts + 1] = child.text + break + end + end + parts[#parts + 1] = '\n' + for _, child in ipairs(node) do + if child.type ~= 'info_string' then + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1)) + end + end + parts[#parts + 1] = '<\n' + elseif ntype == 'html_block' then + local text = node.text:gsub('^<pre>help', '') + text = text:gsub('</pre>%s*$', '') + parts[#parts + 1] = text + elseif ntype == 'list_marker_dot' then + parts[#parts + 1] = node.text + elseif contains(ntype, { 'list_marker_minus', 'list_marker_star' }) then + parts[#parts + 1] = '• ' + elseif ntype == 'list_item' then + parts[#parts + 1] = string.rep(' ', indent) + local offset = node[1].type == 'list_marker_dot' and 3 or 2 + for i, child in ipairs(node) do + local sindent = i <= 2 and 0 or (indent + offset) + vim.list_extend( + parts, + render_md(child, sindent, indent + offset, text_width, level + 1, true) + ) + end + else + if node.text then + error(fmt('cannot render:\n%s', vim.inspect(node))) + end + for i, child in ipairs(node) do + local start_indent0 = i == 1 and start_indent or indent + vim.list_extend( + parts, + render_md(child, start_indent0, indent, text_width, level + 1, is_list) + ) + if ntype ~= 'list' and i ~= #node then + if (node[i + 1] or {}).type ~= 'list' then + parts[#parts + 1] = '\n' + end + end + end + end + + if add_tag then + parts[#parts + 1] = '</' .. ntype .. '>' + end + + return parts +end + +--- @param text_width integer +local function align_tags(text_width) + --- @param line string + --- @return string + return function(line) + local tag_pat = '%s+(%*[^ ]+%*)%s*$' + local tags = {} + for m in line:gmatch(tag_pat) do + table.insert(tags, m) + end + + if #tags > 0 then + line = line:gsub(tag_pat, '') + local tags_str = ' ' .. table.concat(tags, ' ') + local pad = string.rep(' ', text_width - #line - #tags_str) + return line .. pad .. tags_str + end + + return line + end +end + +--- @param text string +--- @param start_indent integer +--- @param indent integer +--- @param is_list? boolean +--- @return string +function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list) + -- Add an extra newline so the parser can properly capture ending ``` + local parsed = parse_md(text .. '\n') + local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list) + + local lines = vim.split(table.concat(ret):gsub(NBSP, ' '), '\n') + + lines = vim.tbl_map(align_tags(text_width), lines) + + local s = table.concat(lines, '\n') + + -- Reduce whitespace in code-blocks + s = s:gsub('\n+%s*>([a-z]+)\n?\n', ' >%1\n') + s = s:gsub('\n+%s*>\n?\n', ' >\n') + + return s +end + +return M diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 47c6d293bc..45dd7f5fee 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -24,7 +24,7 @@ CREATED_FILES=() usage() { echo "Port Vim patches to Neovim" - echo "https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim" + echo "https://neovim.io/doc/user/dev_vimpatch.html" echo echo "Usage: ${BASENAME} [-h | -l | -p vim-revision | -r pr-number]" echo @@ -174,7 +174,7 @@ assign_commit_details() { vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}" vim_message="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:%B' "${vim_commit}" \ - | sed -Ee 's/(#[0-9]{1,})/vim\/vim\1/g')" + | sed -Ee 's/([^A-Za-z0-9])(#[0-9]{1,})/\1vim\/vim\2/g')" local vim_coauthor0 vim_coauthor0="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:Co-authored-by: %an <%ae>' "${vim_commit}")" # Extract co-authors from the commit message. @@ -427,7 +427,7 @@ stage_patch() { or "%s -s --draft" to create a draft pull request. See the wiki for more information: - * https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim + * https://neovim.io/doc/user/dev_vimpatch.html ' "${vim_version}" "${BASENAME}" "${BASENAME}" "${BASENAME}" return $ret } diff --git a/scripts/vimpatch.lua b/scripts/vimpatch.lua index 836f672f6e..cbec50fc17 100755 --- a/scripts/vimpatch.lua +++ b/scripts/vimpatch.lua @@ -10,13 +10,13 @@ local function systemlist(...) local err = nvim.nvim_get_vvar('shell_error') local args_str = nvim.nvim_call_function('string', ...) if 0 ~= err then - error('command failed: '..args_str) + error('command failed: ' .. args_str) end return rv end local function vimpatch_sh_list_numbers() - return systemlist( { { 'bash', '-c', 'scripts/vim-patch.sh -M', } } ) + return systemlist({ { 'bash', '-c', 'scripts/vim-patch.sh -M' } }) end -- Generates the lines to be inserted into the src/version.c @@ -55,9 +55,9 @@ local function patch_version_c() nvim.nvim_command('silent normal! j0d/};\rk') -- Insert the lines. nvim.nvim_call_function('append', { - nvim.nvim_eval('line(".")'), - lines, - }) + nvim.nvim_eval('line(".")'), + lines, + }) nvim.nvim_command('silent write') end diff --git a/scripts/windows.ti b/scripts/windows.ti index c3a367e6d4..34028b8e00 100644 --- a/scripts/windows.ti +++ b/scripts/windows.ti @@ -34,7 +34,7 @@ conemu|ANIS X3.64 and Xterm 256 colors for ConEmu with libuv, smcup=\E[?1049h, rmir@, rmkx@, rmm@, rs1@, rs2@, setab=\E[48;5;%p1%dm, setaf=\E[38;5;%p1%dm, sgr=\E[0%?%p1%p3%|%t;7%;%?%p2%t;4%;%?%p6%t;1%;m, - sgr0=\E[0m, smam@, smglr@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, + sgr0=\E[0m, smam@, smglp@, smgrp@, smglr@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, Cr@, Cs@, Ms@, XM@, kDC3@, kDC4@, kDC5@, kDC6@, kDC7@, kDN@, kDN3@, kDN4@, kDN5@, kDN6@, kDN7@, kEND3@, kEND4@, kEND5@, kEND6@, kEND7@, @@ -57,7 +57,7 @@ vtpcon|ANIS emulation for console virtual terminal sequence with libuv, mc0@, mc4@, mc5@, meml@, memu@, oc@, rmam@, rmcup=\E[?1049l, smcup=\E[?1049h, rmir@, rmkx@, rmm@, rs1@, rs2@, sgr=\E[0%?%p1%p3%|%t;7%;%?%p2%t;4%;%?%p6%t;1%;m, - sgr0=\E[0m, smam@, smglr@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, + sgr0=\E[0m, smam@, smglp@, smgrp@, smglr@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, Cr@, Cs@, Ms@, XM@, kDC3@, kDC4@, kDC5@, kDC6@, kDC7@, kDN@, kDN3@, kDN4@, kDN5@, kDN6@, kDN7@, kEND3@, kEND4@, kEND5@, kEND6@, kEND7@, @@ -68,4 +68,4 @@ vtpcon|ANIS emulation for console virtual terminal sequence with libuv, kPRV3@, kPRV4@, kPRV5@, kPRV6@, kPRV7@, kRIT3@, kRIT4@, kRIT5@, kRIT6@, kRIT7@, kUP3@, kUP4@, kUP5@, kUP6@, kUP7@, rmxx@, smxx@, xm@, - use=libuv+basekey, use=libuv+exkey, use=xterm+256color, use=xterm-new, + use=libuv+basekey, use=libuv+exkey, use=xterm+256color, use=xterm+sl, use=xterm-new, |