diff options
| author | Lewis Russell <lewis6991@gmail.com> | 2024-02-15 17:16:04 +0000 |
|---|---|---|
| committer | Lewis Russell <me@lewisr.dev> | 2024-02-27 14:41:17 +0000 |
| commit | 9beb40a4db5613601fc1a4b828a44e5977eca046 (patch) | |
| tree | 314096d28ccdf2a2b035091783baa35193887d6a /scripts | |
| parent | 7ad2e3c64562bfb0ea2f7be305e4b0e6d2474d64 (diff) | |
| download | rneovim-9beb40a4db5613601fc1a4b828a44e5977eca046.tar.gz rneovim-9beb40a4db5613601fc1a4b828a44e5977eca046.tar.bz2 rneovim-9beb40a4db5613601fc1a4b828a44e5977eca046.zip | |
feat(docs): replace lua2dox.lua
Problem:
The documentation flow (`gen_vimdoc.py`) has several issues:
- it's not very versatile
- depends on doxygen
- doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C.
- The intermediate XML files and filters makes it too much like a rube goldberg machine.
Solution:
Re-implement the flow using Lua, LPEG and treesitter.
- `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic.
- `lua2dox.lua` is gone!
- No more XML files.
- Doxygen is now longer used and instead we now use:
- LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`).
- LPEG for C parsing (see `scripts/cdoc_parser.lua`)
- Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`).
- Treesitter for Markdown parsing (see `scripts/text_utils.lua`).
- The generated `runtime/doc/*.mpack` files have been removed.
- `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly.
- Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/cdoc_grammar.lua | 87 | ||||
| -rw-r--r-- | scripts/cdoc_parser.lua | 223 | ||||
| -rwxr-xr-x | scripts/gen_eval_files.lua | 100 | ||||
| -rwxr-xr-x | scripts/gen_vimdoc.lua | 787 | ||||
| -rwxr-xr-x | scripts/gen_vimdoc.py | 1766 | ||||
| -rw-r--r-- | scripts/lua2dox.lua | 544 | ||||
| -rw-r--r-- | scripts/luacats_grammar.lua | 218 | ||||
| -rw-r--r-- | scripts/luacats_parser.lua | 521 | ||||
| -rw-r--r-- | scripts/text_utils.lua | 239 |
9 files changed, 2135 insertions, 2350 deletions
diff --git a/scripts/cdoc_grammar.lua b/scripts/cdoc_grammar.lua new file mode 100644 index 0000000000..6a7610883b --- /dev/null +++ b/scripts/cdoc_grammar.lua @@ -0,0 +1,87 @@ +--[[! +LPEG grammar for C doc comments +]] + +--- @class nvim.cdoc.Param +--- @field kind 'param' +--- @field name string +--- @field desc? string + +--- @class nvim.cdoc.Return +--- @field kind 'return' +--- @field desc string + +--- @class nvim.cdoc.Note +--- @field desc? string + +--- @alias nvim.cdoc.grammar.result +--- | nvim.cdoc.Param +--- | nvim.cdoc.Return +--- | nvim.cdoc.Note + +--- @class nvim.cdoc.grammar +--- @field match fun(self, input: string): nvim.cdoc.grammar.result? + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local Ct, Cg = lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local nl = P('\r\n') + P('\n') +local ws = rep1(S(' \t') + nl) + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local ident = letter * rep(letter + R('09')) + +local io = P('[') * (P('in') + P('out') + P('inout')) * P(']') + +--- @param x string +local function Pf(x) + return opt(ws) * P(x) * opt(ws) +end + +--- @type table<string,vim.lpeg.Pattern> +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +local grammar = P { + rep1(P('@') * v.ats), + + ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc, + + at_param = Ct( + Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc')) + ), + + at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))), + + at_deprecated = Ct(Cg(P('deprecated'), 'kind')), + + at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + + at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_nodoc = Ct(Cg(P('nodoc'), 'kind')), +} + +return grammar --[[@as nvim.cdoc.grammar]] diff --git a/scripts/cdoc_parser.lua b/scripts/cdoc_parser.lua new file mode 100644 index 0000000000..5f0dc7be2c --- /dev/null +++ b/scripts/cdoc_parser.lua @@ -0,0 +1,223 @@ +local cdoc_grammar = require('scripts.cdoc_grammar') +local c_grammar = require('src.nvim.generators.c_grammar') + +--- @class nvim.cdoc.parser.param +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.note +--- @field desc string + +--- @class nvim.cdoc.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.cdoc.parser.fun +--- @field name string +--- @field params nvim.cdoc.parser.param[] +--- @field returns nvim.cdoc.parser.return[] +--- @field desc string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field notes? nvim.cdoc.parser.note[] +--- @field see? nvim.cdoc.parser.note[] + +--- @class nvim.cdoc.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.cdoc.parser.obj +--- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.cdoc.parser.obj +--- | nvim.cdoc.parser.fun +--- | nvim.cdoc.parser.brief + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.cdoc.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.cdoc.parser.State +local function process_doc_line(line, state) + line = line:gsub('^%s+@', '@') + + local parsed = cdoc_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + + local kind = parsed.kind + + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + state.last_doc_item = { + name = parsed.name, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = { { + desc = parsed.desc, + } } + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[1] + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param item table +--- @param state nvim.cdoc.parser.State +local function process_proto(item, state) + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + cur_obj.name = item.name + cur_obj.params = cur_obj.params or {} + + for _, p in ipairs(item.parameters) do + local param = { name = p[2], type = p[1] } + local added = false + for _, cp in ipairs(cur_obj.params) do + if cp.name == param.name then + cp.type = param.type + added = true + break + end + end + + if not added then + table.insert(cur_obj.params, param) + end + end + + cur_obj.returns = cur_obj.returns or { {} } + cur_obj.returns[1].type = item.return_type + + for _, a in ipairs({ + 'fast', + 'remote_only', + 'lua_only', + 'textlock', + 'textlock_allow_cmdwin', + }) do + if item[a] then + cur_obj.attrs = cur_obj.attrs or {} + table.insert(cur_obj.attrs, a) + end + end + + cur_obj.deprecated_since = item.deprecated_since + + -- Remove some arguments + for i = #cur_obj.params, 1, -1 do + local p = cur_obj.params[i] + if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then + table.remove(cur_obj.params, i) + end + end +end + +local M = {} + +--- @param filename string +--- @return {} classes +--- @return nvim.cdoc.parser.fun[] funs +--- @return string[] briefs +function M.parse(filename) + local funs = {} --- @type nvim.cdoc.parser.fun[] + local briefs = {} --- @type string[] + local state = {} --- @type nvim.cdoc.parser.State + + local txt = assert(io.open(filename, 'r')):read('*all') + + local parsed = c_grammar.grammar:match(txt) + for _, item in ipairs(parsed) do + if item.comment then + process_doc_line(item.comment, state) + else + add_doc_lines_to_obj(state) + if item[1] == 'proto' then + process_proto(item, state) + table.insert(funs, state.cur_obj) + end + local cur_obj = state.cur_obj + if cur_obj and not item.static then + if cur_obj.kind == 'brief' then + table.insert(briefs, cur_obj.desc) + end + end + state = {} + end + end + + return {}, funs, briefs +end + +-- M.parse('src/nvim/api/vim.c') + +return M diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua index b7f17a2d58..895033d5af 100755 --- a/scripts/gen_eval_files.lua +++ b/scripts/gen_eval_files.lua @@ -3,7 +3,6 @@ -- Generator for various vimdoc and Lua type files local DEP_API_METADATA = 'build/api_metadata.mpack' -local DEP_API_DOC = 'runtime/doc/api.mpack' --- @class vim.api.metadata --- @field name string @@ -210,44 +209,65 @@ end --- @return table<string, vim.EvalFn> local function get_api_meta() - local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) - local metadata = vim.mpack.decode(mpack_f:read('*all')) --[[@as vim.api.metadata[] ]] local ret = {} --- @type table<string, vim.EvalFn> - local doc_mpack_f = assert(io.open(DEP_API_DOC, 'rb')) - local doc_metadata = vim.mpack.decode(doc_mpack_f:read('*all')) --[[@as table<string,vim.gen_vim_doc_fun>]] - - for _, fun in ipairs(metadata) do - if fun.lua then - local fdoc = doc_metadata[fun.name] - - local params = {} --- @type {[1]:string,[2]:string}[] - for _, p in ipairs(fun.parameters) do - local ptype, pname = p[1], p[2] - params[#params + 1] = { - pname, - api_type(ptype), - fdoc and fdoc.parameters_doc[pname] or nil, - } - end + local cdoc_parser = require('scripts.cdoc_parser') - local r = { - signature = 'NA', - name = fun.name, - params = params, - returns = api_type(fun.return_type), - deprecated = fun.deprecated_since ~= nil, - } + local f = 'src/nvim/api' - if fdoc then - if #fdoc.doc > 0 then - r.desc = table.concat(fdoc.doc, '\n') + local function include(fun) + if not vim.startswith(fun.name, 'nvim_') then + return false + end + if vim.tbl_contains(fun.attrs or {}, 'lua_only') then + return true + end + if vim.tbl_contains(fun.attrs or {}, 'remote_only') then + return false + end + return true + end + + --- @type table<string,nvim.cdoc.parser.fun> + local functions = {} + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + local filename = vim.fs.joinpath(f, path) + local _, funs = cdoc_parser.parse(filename) + for _, fn in ipairs(funs) do + if include(fn) then + functions[fn.name] = fn end - r.return_desc = (fdoc['return'] or {})[1] end + end + end + + for _, fun in pairs(functions) do + local deprecated = fun.deprecated_since ~= nil + + local params = {} --- @type {[1]:string,[2]:string}[] + for _, p in ipairs(fun.params) do + params[#params + 1] = { + p.name, + api_type(p.type), + not deprecated and p.desc or nil, + } + end + + local r = { + signature = 'NA', + name = fun.name, + params = params, + returns = api_type(fun.returns[1].type), + deprecated = deprecated, + } - ret[fun.name] = r + if not deprecated then + r.desc = fun.desc + r.return_desc = fun.returns[1].desc end + + ret[fun.name] = r end return ret end @@ -275,12 +295,10 @@ end --- @param fun vim.EvalFn --- @param write fun(line: string) local function render_api_meta(_f, fun, write) - if not vim.startswith(fun.name, 'nvim_') then - return - end - write('') + local text_utils = require('scripts.text_utils') + if vim.startswith(fun.name, 'nvim__') then write('--- @private') end @@ -291,10 +309,10 @@ local function render_api_meta(_f, fun, write) local desc = fun.desc if desc then + desc = text_utils.md_to_vimdoc(desc, 0, 0, 74) for _, l in ipairs(split(norm_text(desc))) do write('--- ' .. l) end - write('---') end local param_names = {} --- @type string[] @@ -303,8 +321,11 @@ local function render_api_meta(_f, fun, write) param_names[#param_names + 1] = p[1] local pdesc = p[3] if pdesc then - local pdesc_a = split(norm_text(pdesc)) - write('--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' .. pdesc_a[1]) + local s = '--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' + local indent = #('@param ' .. p[1] .. ' ') + pdesc = text_utils.md_to_vimdoc(pdesc, #s, indent, 74, true) + local pdesc_a = split(vim.trim(norm_text(pdesc))) + write(s .. pdesc_a[1]) for i = 2, #pdesc_a do if not pdesc_a[i] then break @@ -317,6 +338,7 @@ local function render_api_meta(_f, fun, write) end if fun.returns ~= '' then local ret_desc = fun.returns_desc and ' : ' .. fun.returns_desc or '' + ret_desc = text_utils.md_to_vimdoc(ret_desc, 0, 0, 74) local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns write('--- @return ' .. ret .. ret_desc) end @@ -328,8 +350,6 @@ end --- @return table<string, vim.EvalFn> local function get_api_keysets_meta() local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) - - --- @diagnostic disable-next-line:no-unknown local metadata = assert(vim.mpack.decode(mpack_f:read('*all'))) local ret = {} --- @type table<string, vim.EvalFn> diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua new file mode 100755 index 0000000000..290cd83fbc --- /dev/null +++ b/scripts/gen_vimdoc.lua @@ -0,0 +1,787 @@ +#!/usr/bin/env -S nvim -l +--- Generates Nvim :help docs from Lua/C docstrings +--- +--- The generated :help text for each function is formatted as follows: +--- - Max width of 78 columns (`TEXT_WIDTH`). +--- - Indent with spaces (not tabs). +--- - Indent of 4 columns for body text (`INDENTATION`). +--- - Function signature and helptag (right-aligned) on the same line. +--- - Signature and helptag must have a minimum of 8 spaces between them. +--- - If the signature is too long, it is placed on the line after the helptag. +--- Signature wraps with subsequent lines indented to the open parenthesis. +--- - Subsection bodies are indented an additional 4 spaces. +--- - Body consists of function description, parameters, return description, and +--- C declaration (`INCLUDE_C_DECL`). +--- - Parameters are omitted for the `void` and `Error *` types, or if the +--- parameter is marked as [out]. +--- - Each function documentation is separated by a single line. + +local luacats_parser = require('scripts.luacats_parser') +local cdoc_parser = require('scripts.cdoc_parser') +local text_utils = require('scripts.text_utils') + +local fmt = string.format + +local wrap = text_utils.wrap +local md_to_vimdoc = text_utils.md_to_vimdoc + +local TEXT_WIDTH = 78 +local INDENTATION = 4 + +--- @class (exact) nvim.gen_vimdoc.Config +--- +--- Generated documentation target, e.g. api.txt +--- @field filename string +--- +--- @field section_order string[] +--- +--- List of files/directories for doxygen to read, relative to `base_dir`. +--- @field files string[] +--- +--- @field exclude_types? true +--- +--- Section name overrides. Key: filename (e.g., vim.c) +--- @field section_name? table<string,string> +--- +--- @field fn_name_pat? string +--- +--- @field fn_xform? fun(fun: nvim.luacats.parser.fun) +--- +--- For generated section names. +--- @field section_fmt fun(name: string): string +--- +--- @field helptag_fmt fun(name: string): string +--- +--- Per-function helptag. +--- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string +--- +--- @field append_only? string[] + +local function contains(t, xs) + return vim.tbl_contains(xs, t) +end + +--- @type {level:integer, prerelease:boolean}? +local nvim_api_info_ + +--- @return {level: integer, prerelease:boolean} +local function nvim_api_info() + if not nvim_api_info_ then + --- @type integer?, boolean? + local level, prerelease + for l in io.lines('CMakeLists.txt') do + --- @cast l string + if level and prerelease then + break + end + local m1 = l:match('^set%(NVIM_API_LEVEL%s+(%d+)%)') + if m1 then + level = tonumber(m1) --[[@as integer]] + end + local m2 = l:match('^set%(NVIM_API_PRERELEASE%s+(%w+)%)') + if m2 then + prerelease = m2 == 'true' + end + end + nvim_api_info_ = { level = level, prerelease = prerelease } + end + + return nvim_api_info_ +end + +--- @param fun nvim.luacats.parser.fun +--- @return string +local function fn_helptag_fmt_common(fun) + local fn_sfx = fun.table and '' or '()' + if fun.classvar then + return fmt('*%s:%s%s*', fun.classvar, fun.name, fn_sfx) + end + if fun.module then + return fmt('*%s.%s%s*', fun.module, fun.name, fn_sfx) + end + return fmt('*%s%s*', fun.name, fn_sfx) +end + +--- @type table<string,nvim.gen_vimdoc.Config> +local config = { + api = { + filename = 'api.txt', + section_order = { + 'vim.c', + 'vimscript.c', + 'command.c', + 'options.c', + 'buffer.c', + 'extmark.c', + 'window.c', + 'win_config.c', + 'tabpage.c', + 'autocmd.c', + 'ui.c', + }, + exclude_types = true, + fn_name_pat = 'nvim_.*', + files = { 'src/nvim/api' }, + section_name = { + ['vim.c'] = 'Global', + }, + section_fmt = function(name) + return name .. ' Functions' + end, + helptag_fmt = function(name) + return fmt('*api-%s*', name:lower()) + end, + }, + lua = { + filename = 'lua.txt', + section_order = { + 'highlight.lua', + 'diff.lua', + 'mpack.lua', + 'json.lua', + 'base64.lua', + 'spell.lua', + 'builtin.lua', + '_options.lua', + '_editor.lua', + '_inspector.lua', + 'shared.lua', + 'loader.lua', + 'uri.lua', + 'ui.lua', + 'filetype.lua', + 'keymap.lua', + 'fs.lua', + 'glob.lua', + 'lpeg.lua', + 're.lua', + 'regex.lua', + 'secure.lua', + 'version.lua', + 'iter.lua', + 'snippet.lua', + 'text.lua', + }, + files = { + 'runtime/lua/vim/iter.lua', + 'runtime/lua/vim/_editor.lua', + 'runtime/lua/vim/_options.lua', + 'runtime/lua/vim/shared.lua', + 'runtime/lua/vim/loader.lua', + 'runtime/lua/vim/uri.lua', + 'runtime/lua/vim/ui.lua', + 'runtime/lua/vim/filetype.lua', + 'runtime/lua/vim/keymap.lua', + 'runtime/lua/vim/fs.lua', + 'runtime/lua/vim/highlight.lua', + 'runtime/lua/vim/secure.lua', + 'runtime/lua/vim/version.lua', + 'runtime/lua/vim/_inspector.lua', + 'runtime/lua/vim/snippet.lua', + 'runtime/lua/vim/text.lua', + 'runtime/lua/vim/glob.lua', + 'runtime/lua/vim/_meta/builtin.lua', + 'runtime/lua/vim/_meta/diff.lua', + 'runtime/lua/vim/_meta/mpack.lua', + 'runtime/lua/vim/_meta/json.lua', + 'runtime/lua/vim/_meta/base64.lua', + 'runtime/lua/vim/_meta/regex.lua', + 'runtime/lua/vim/_meta/lpeg.lua', + 'runtime/lua/vim/_meta/re.lua', + 'runtime/lua/vim/_meta/spell.lua', + }, + fn_xform = function(fun) + if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then + fun.module = 'vim' + end + + if fun.module == 'vim' and contains(fun.name, { 'cmd', 'inspect' }) then + fun.table = nil + end + + if fun.classvar or vim.startswith(fun.name, 'vim.') or fun.module == 'vim.iter' then + return + end + + fun.name = fmt('%s.%s', fun.module, fun.name) + end, + section_name = { + ['_inspector.lua'] = 'inspector', + }, + section_fmt = function(name) + name = name:lower() + if name == '_editor' then + return 'Lua module: vim' + elseif name == '_options' then + return 'LUA-VIMSCRIPT BRIDGE' + elseif name == 'builtin' then + return 'VIM' + end + if + contains(name, { + 'highlight', + 'mpack', + 'json', + 'base64', + 'diff', + 'spell', + 'regex', + 'lpeg', + 're', + }) + then + return 'VIM.' .. name:upper() + end + return 'Lua module: vim.' .. name + end, + helptag_fmt = function(name) + if name == '_editor' then + return '*lua-vim*' + elseif name == '_options' then + return '*lua-vimscript*' + end + return '*vim.' .. name:lower() .. '*' + end, + fn_helptag_fmt = function(fun) + local name = fun.name + + if vim.startswith(name, 'vim.') then + local fn_sfx = fun.table and '' or '()' + return fmt('*%s%s*', name, fn_sfx) + elseif fun.classvar == 'Option' then + return fmt('*vim.opt:%s()*', name) + end + + return fn_helptag_fmt_common(fun) + end, + append_only = { + 'shared.lua', + }, + }, + lsp = { + filename = 'lsp.txt', + section_order = { + 'lsp.lua', + 'buf.lua', + 'diagnostic.lua', + 'codelens.lua', + 'inlay_hint.lua', + 'tagfunc.lua', + 'semantic_tokens.lua', + 'handlers.lua', + 'util.lua', + 'log.lua', + 'rpc.lua', + 'protocol.lua', + }, + files = { + 'runtime/lua/vim/lsp', + 'runtime/lua/vim/lsp.lua', + }, + fn_xform = function(fun) + fun.name = fun.name:gsub('result%.', '') + end, + section_fmt = function(name) + if name:lower() == 'lsp' then + return 'Lua module: vim.lsp' + end + return 'Lua module: vim.lsp.' .. name:lower() + end, + helptag_fmt = function(name) + if name:lower() == 'lsp' then + return '*lsp-core*' + end + return fmt('*lsp-%s*', name:lower()) + end, + }, + diagnostic = { + filename = 'diagnostic.txt', + section_order = { + 'diagnostic.lua', + }, + files = { 'runtime/lua/vim/diagnostic.lua' }, + section_fmt = function() + return 'Lua module: vim.diagnostic' + end, + helptag_fmt = function() + return '*diagnostic-api*' + end, + }, + treesitter = { + filename = 'treesitter.txt', + section_order = { + 'treesitter.lua', + 'language.lua', + 'query.lua', + 'highlighter.lua', + 'languagetree.lua', + 'dev.lua', + }, + files = { + 'runtime/lua/vim/treesitter.lua', + 'runtime/lua/vim/treesitter/', + }, + section_fmt = function(name) + if name:lower() == 'treesitter' then + return 'Lua module: vim.treesitter' + end + return 'Lua module: vim.treesitter.' .. name:lower() + end, + helptag_fmt = function(name) + if name:lower() == 'treesitter' then + return '*lua-treesitter-core*' + end + return '*lua-treesitter-' .. name:lower() .. '*' + end, + }, +} + +--- @param ty string +--- @param generics table<string,string> +--- @return string +local function replace_generics(ty, generics) + if ty:sub(-2) == '[]' then + local ty0 = ty:sub(1, -3) + if generics[ty0] then + return generics[ty0] .. '[]' + end + elseif ty:sub(-1) == '?' then + local ty0 = ty:sub(1, -2) + if generics[ty0] then + return generics[ty0] .. '?' + end + end + + return generics[ty] or ty +end + +--- @param ty string +--- @param generics? table<string,string> +local function render_type(ty, generics) + if generics then + ty = replace_generics(ty, generics) + end + ty = ty:gsub('%s*|%s*nil', '?') + ty = ty:gsub('nil%s*|%s*(.*)', '%1?') + ty = ty:gsub('%s*|%s*', '|') + return fmt('(`%s`)', ty) +end + +--- @param p nvim.luacats.parser.param|nvim.luacats.parser.field +local function should_render_param(p) + return not p.access and not contains(p.name, { '_', 'self' }) +end + +--- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[] +--- @param generics? table<string,string> +--- @param exclude_types? true +local function render_fields_or_params(xs, generics, exclude_types) + local ret = {} --- @type string[] + + xs = vim.tbl_filter(should_render_param, xs) + + local indent = 0 + for _, p in ipairs(xs) do + if p.type or p.desc then + indent = math.max(indent, #p.name + 3) + end + if exclude_types then + p.type = nil + end + end + + for _, p in ipairs(xs) do + local nm, ty = p.name, p.type + local desc = p.desc + local pnm = fmt(' • %-' .. indent .. 's', '{' .. nm .. '}') + if ty then + local pty = render_type(ty, generics) + if desc then + desc = fmt('%s %s', pty, desc) + table.insert(ret, pnm) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + else + table.insert(ret, fmt('%s %s\n', pnm, pty)) + end + else + if desc then + table.insert(ret, pnm) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + end + end + end + + return table.concat(ret) +end + +-- --- @param class lua2vimdoc.class +-- local function render_class(class) +-- writeln(fmt('*%s*', class.name)) +-- writeln() +-- if #class.fields > 0 then +-- writeln(' Fields: ~') +-- render_fields_or_params(class.fields) +-- end +-- writeln() +-- end + +-- --- @param cls table<string,lua2vimdoc.class> +-- local function render_classes(cls) +-- --- @diagnostic disable-next-line:no-unknown +-- for _, class in vim.spairs(cls) do +-- render_class(class) +-- end +-- end + +--- @param fun nvim.luacats.parser.fun +--- @param cfg nvim.gen_vimdoc.Config +local function render_fun_header(fun, cfg) + local ret = {} --- @type string[] + + local args = {} --- @type string[] + for _, p in ipairs(fun.params or {}) do + if p.name ~= 'self' then + args[#args + 1] = fmt('{%s}', p.name:gsub('%?$', '')) + end + end + + local nm = fun.name + if fun.classvar then + nm = fmt('%s:%s', fun.classvar, nm) + end + + local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')' + + if not cfg.fn_helptag_fmt then + cfg.fn_helptag_fmt = fn_helptag_fmt_common + end + + local tag = cfg.fn_helptag_fmt(fun) + + if #proto + #tag > TEXT_WIDTH - 8 then + table.insert(ret, fmt('%78s\n', tag)) + local name, pargs = proto:match('([^(]+%()(.*)') + table.insert(ret, name) + table.insert(ret, wrap(pargs, 0, #name, TEXT_WIDTH)) + else + local pad = TEXT_WIDTH - #proto - #tag + table.insert(ret, proto .. string.rep(' ', pad) .. tag) + end + + return table.concat(ret) +end + +--- @param returns nvim.luacats.parser.return[] +--- @param generics? table<string,string> +--- @param exclude_types boolean +local function render_returns(returns, generics, exclude_types) + local ret = {} --- @type string[] + + returns = vim.deepcopy(returns) + if exclude_types then + for _, r in ipairs(returns) do + r.type = nil + end + end + + if #returns > 1 then + table.insert(ret, ' Return (multiple): ~\n') + elseif #returns == 1 and next(returns[1]) then + table.insert(ret, ' Return: ~\n') + end + + for _, p in ipairs(returns) do + local rnm, ty, desc = p.name, p.type, p.desc + local blk = '' + if ty then + blk = render_type(ty, generics) + end + if rnm then + blk = blk .. ' ' .. rnm + end + if desc then + blk = blk .. ' ' .. desc + end + table.insert(ret, md_to_vimdoc(blk, 8, 8, TEXT_WIDTH, true)) + end + + return table.concat(ret) +end + +--- @param fun nvim.luacats.parser.fun +--- @param cfg nvim.gen_vimdoc.Config +local function render_fun(fun, cfg) + if fun.access or fun.deprecated or fun.nodoc then + return + end + + if cfg.fn_name_pat and not fun.name:match(cfg.fn_name_pat) then + return + end + + if vim.startswith(fun.name, '_') or fun.name:find('[:.]_') then + return + end + + local ret = {} --- @type string[] + + table.insert(ret, render_fun_header(fun, cfg)) + table.insert(ret, '\n') + + if fun.desc then + table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH)) + end + + if fun.since then + local since = tonumber(fun.since) + local info = nvim_api_info() + if since and (since > info.level or since == info.level and info.prerelease) then + fun.notes = fun.notes or {} + table.insert(fun.notes, { desc = 'This API is pre-release (unstable).' }) + end + end + + if fun.notes then + table.insert(ret, '\n Note: ~\n') + for _, p in ipairs(fun.notes) do + table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true)) + end + end + + if fun.attrs then + table.insert(ret, '\n Attributes: ~\n') + for _, attr in ipairs(fun.attrs) do + local attr_str = ({ + textlock = 'not allowed when |textlock| is active or in the |cmdwin|', + textlock_allow_cmdwin = 'not allowed when |textlock| is active', + fast = '|api-fast|', + remote_only = '|RPC| only', + lua_only = 'Lua |vim.api| only', + })[attr] or attr + table.insert(ret, fmt(' %s\n', attr_str)) + end + end + + if fun.params and #fun.params > 0 then + local param_txt = render_fields_or_params(fun.params, fun.generics, cfg.exclude_types) + if not param_txt:match('^%s*$') then + table.insert(ret, '\n Parameters: ~\n') + ret[#ret + 1] = param_txt + end + end + + if fun.returns then + local txt = render_returns(fun.returns, fun.generics, cfg.exclude_types) + if not txt:match('^%s*$') then + table.insert(ret, '\n') + ret[#ret + 1] = txt + end + end + + if fun.see then + table.insert(ret, '\n See also: ~\n') + for _, p in ipairs(fun.see) do + table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true)) + end + end + + table.insert(ret, '\n') + return table.concat(ret) +end + +--- @param funs nvim.luacats.parser.fun[] +--- @param cfg nvim.gen_vimdoc.Config +local function render_funs(funs, cfg) + local ret = {} --- @type string[] + + for _, f in ipairs(funs) do + if cfg.fn_xform then + cfg.fn_xform(f) + end + ret[#ret + 1] = render_fun(f, cfg) + end + + -- Sort via prototype + table.sort(ret, function(a, b) + local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n') + local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n') + return a1:lower() < b1:lower() + end) + + return table.concat(ret) +end + +--- @return string +local function get_script_path() + local str = debug.getinfo(2, 'S').source:sub(2) + return str:match('(.*[/\\])') or './' +end + +local script_path = get_script_path() +local base_dir = vim.fs.dirname(assert(vim.fs.dirname(script_path))) + +local function delete_lines_below(doc_file, tokenstr) + local lines = {} --- @type string[] + local found = false + for line in io.lines(doc_file) do + if line:find(vim.pesc(tokenstr)) then + found = true + break + end + lines[#lines + 1] = line + end + if not found then + error(fmt('not found: %s in %s', tokenstr, doc_file)) + end + lines[#lines] = nil + local fp = assert(io.open(doc_file, 'w')) + fp:write(table.concat(lines, '\n')) + fp:write('\n') + fp:close() +end + +--- @param x string +local function mktitle(x) + if x == 'ui' then + return 'UI' + end + return x:sub(1, 1):upper() .. x:sub(2) +end + +--- @class nvim.gen_vimdoc.Section +--- @field name string +--- @field title string +--- @field help_tag string +--- @field funs_txt string +--- @field doc? string[] + +--- @param filename string +--- @param cfg nvim.gen_vimdoc.Config +--- @param section_docs table<string,nvim.gen_vimdoc.Section> +--- @param funs_txt string +--- @return nvim.gen_vimdoc.Section? +local function make_section(filename, cfg, section_docs, funs_txt) + -- filename: e.g., 'autocmd.c' + -- name: e.g. 'autocmd' + local name = filename:match('(.*)%.[a-z]+') + + -- Formatted (this is what's going to be written in the vimdoc) + -- e.g., "Autocmd Functions" + local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name) + + -- section tag: e.g., "*api-autocmd*" + local help_tag = cfg.helptag_fmt(sectname) + + if funs_txt == '' and #section_docs == 0 then + return + end + + return { + name = sectname, + title = cfg.section_fmt(sectname), + help_tag = help_tag, + funs_txt = funs_txt, + doc = section_docs, + } +end + +--- @param section nvim.gen_vimdoc.Section +--- @param add_header? boolean +local function render_section(section, add_header) + local doc = {} --- @type string[] + + if add_header ~= false then + vim.list_extend(doc, { + string.rep('=', TEXT_WIDTH), + '\n', + section.title, + fmt('%' .. (TEXT_WIDTH - section.title:len()) .. 's', section.help_tag), + }) + end + + if section.doc and #section.doc > 0 then + table.insert(doc, '\n\n') + vim.list_extend(doc, section.doc) + end + + if section.funs_txt then + table.insert(doc, '\n\n') + table.insert(doc, section.funs_txt) + end + + return table.concat(doc) +end + +local parsers = { + lua = luacats_parser.parse, + c = cdoc_parser.parse, + h = cdoc_parser.parse, +} + +--- @param files string[] +local function expand_files(files) + for k, f in pairs(files) do + if vim.fn.isdirectory(f) == 1 then + table.remove(files, k) + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + table.insert(files, vim.fs.joinpath(f, path)) + end + end + end + end +end + +--- @param cfg nvim.gen_vimdoc.Config +local function gen_target(cfg) + local sections = {} --- @type table<string,nvim.gen_vimdoc.Section> + + expand_files(cfg.files) + + for _, f in pairs(cfg.files) do + local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']] + local parser = assert(parsers[ext]) + local _, funs, briefs = parser(f) + local briefs_txt = {} --- @type string[] + for _, b in ipairs(briefs) do + briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH) + end + local funs_txt = render_funs(funs, cfg) + -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua` + local f_base = assert(vim.fs.basename(f)) + sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt) + end + + local first_section_tag = sections[cfg.section_order[1]].help_tag + local docs = {} --- @type string[] + for _, f in ipairs(cfg.section_order) do + local section = sections[f] + if section then + local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f) + table.insert(docs, render_section(section, add_sep_and_header)) + end + end + + table.insert( + docs, + fmt(' vim:tw=78:ts=8:sw=%d:sts=%d:et:ft=help:norl:\n', INDENTATION, INDENTATION) + ) + + local doc_file = vim.fs.joinpath(base_dir, 'runtime', 'doc', cfg.filename) + + if vim.uv.fs_stat(doc_file) then + delete_lines_below(doc_file, first_section_tag) + end + + local fp = assert(io.open(doc_file, 'a')) + fp:write(table.concat(docs, '\n')) + fp:close() +end + +local function run() + for _, cfg in pairs(config) do + gen_target(cfg) + end +end + +run() diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py deleted file mode 100755 index c1a2183f24..0000000000 --- a/scripts/gen_vimdoc.py +++ /dev/null @@ -1,1766 +0,0 @@ -#!/usr/bin/env python3 - -r"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. - -Also generates *.mpack files. To inspect the *.mpack structure: - :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B'))) - -Flow: - main - extract_from_xml - fmt_node_as_vimhelp \ - para_as_map } recursive - update_params_map / - render_node - -TODO: eliminate this script and use Lua+treesitter (requires parsers for C and -Lua markdown-style docstrings). - -The generated :help text for each function is formatted as follows: - - - Max width of 78 columns (`text_width`). - - Indent with spaces (not tabs). - - Indent of 4 columns for body text (`indentation`). - - Function signature and helptag (right-aligned) on the same line. - - Signature and helptag must have a minimum of 8 spaces between them. - - If the signature is too long, it is placed on the line after the helptag. - Signature wraps at `text_width - 8` characters with subsequent - lines indented to the open parenthesis. - - Subsection bodies are indented an additional 4 spaces. - - Body consists of function description, parameters, return description, and - C declaration (`INCLUDE_C_DECL`). - - Parameters are omitted for the `void` and `Error *` types, or if the - parameter is marked as [out]. - - Each function documentation is separated by a single line. -""" - -from __future__ import annotations # PEP-563, python 3.7+ - -import argparse -import collections -import dataclasses -import logging -import os -import re -import shutil -import subprocess -import sys -import textwrap -from pathlib import Path -from typing import Any, Callable, Dict, List, Tuple -from xml.dom import minidom - -if sys.version_info >= (3, 8): - from typing import Literal - -import msgpack - -Element = minidom.Element -Document = minidom.Document - -MIN_PYTHON_VERSION = (3, 7) -MIN_DOXYGEN_VERSION = (1, 9, 0) - -if sys.version_info < MIN_PYTHON_VERSION: - print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION)) - sys.exit(1) - -doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"], - universal_newlines=True).split()[0].split('.'))) - -if doxygen_version < MIN_DOXYGEN_VERSION: - print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION)) - print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version)) - sys.exit(1) - - -# Need a `nvim` that supports `-l`, try the local build -nvim_path = Path(__file__).parent / "../build/bin/nvim" -if nvim_path.exists(): - nvim = nvim_path.resolve() -else: - # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works. - nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True) - nvim_version = [line for line in nvim_out.split('\n') - if '-l ' in line] - if len(nvim_version) == 0: - print(( - "\nYou need to have a local Neovim build or a `nvim` version 0.9 for `-l` " - "support to build the documentation.")) - sys.exit(1) - nvim = 'nvim' - - -# DEBUG = ('DEBUG' in os.environ) -INCLUDE_C_DECL = os.environ.get('INCLUDE_C_DECL', '0') != '0' -INCLUDE_DEPRECATED = os.environ.get('INCLUDE_DEPRECATED', '0') != '0' - -log = logging.getLogger(__name__) - -LOG_LEVELS = { - logging.getLevelName(level): level for level in [ - logging.DEBUG, logging.INFO, logging.ERROR - ] -} - -text_width = 78 -indentation = 4 -SECTION_SEP = '=' * text_width - -script_path = os.path.abspath(__file__) -base_dir = os.path.dirname(os.path.dirname(script_path)) -out_dir = os.path.join(base_dir, 'tmp-{target}-doc') -filter_cmd = '%s %s' % (sys.executable, script_path) -msgs = [] # Messages to show on exit. -lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') - - -SectionName = str - -Docstring = str # Represents (formatted) vimdoc string - -FunctionName = str - - -@dataclasses.dataclass -class Config: - """Config for documentation.""" - - mode: Literal['c', 'lua'] - - filename: str - """Generated documentation target, e.g. api.txt""" - - section_order: List[str] - """Section ordering.""" - - files: List[str] - """List of files/directories for doxygen to read, relative to `base_dir`.""" - - file_patterns: str - """file patterns used by doxygen.""" - - section_name: Dict[str, SectionName] - """Section name overrides. Key: filename (e.g., vim.c)""" - - section_fmt: Callable[[SectionName], str] - """For generated section names.""" - - helptag_fmt: Callable[[SectionName], str] - """Section helptag.""" - - fn_helptag_fmt: Callable[[str, str, bool], str] - """Per-function helptag.""" - - module_override: Dict[str, str] - """Module name overrides (for Lua).""" - - append_only: List[str] - """Append the docs for these modules, do not start a new section.""" - - fn_name_prefix: str - """Only function with this prefix are considered""" - - fn_name_fmt: Callable[[str, str], str] | None = None - - include_tables: bool = True - - -CONFIG: Dict[str, Config] = { - 'api': Config( - mode='c', - filename = 'api.txt', - # Section ordering. - section_order=[x for x in [ - 'vim.c', - 'vimscript.c', - 'command.c', - 'options.c', - 'buffer.c', - 'extmark.c', - 'window.c', - 'win_config.c', - 'tabpage.c', - 'autocmd.c', - 'ui.c', - 'deprecated.c' if INCLUDE_DEPRECATED else '' - ] if x], - files=['src/nvim/api'], - file_patterns = '*.h *.c', - fn_name_prefix = 'nvim_', - section_name={ - 'vim.c': 'Global', - }, - section_fmt=lambda name: f'{name} Functions', - helptag_fmt=lambda name: f'*api-{name.lower()}*', - fn_helptag_fmt=lambda fstem, name, istbl: f'*{name}()*', - module_override={}, - append_only=[], - ), - 'lua': Config( - mode='lua', - filename='lua.txt', - section_order=[ - 'highlight.lua', - 'diff.lua', - 'mpack.lua', - 'json.lua', - 'base64.lua', - 'spell.lua', - 'builtin.lua', - '_options.lua', - '_editor.lua', - '_inspector.lua', - 'shared.lua', - 'loader.lua', - 'uri.lua', - 'ui.lua', - 'filetype.lua', - 'keymap.lua', - 'fs.lua', - 'glob.lua', - 'lpeg.lua', - 're.lua', - 'regex.lua', - 'secure.lua', - 'version.lua', - 'iter.lua', - 'snippet.lua', - 'text.lua', - ], - files=[ - 'runtime/lua/vim/iter.lua', - 'runtime/lua/vim/_editor.lua', - 'runtime/lua/vim/_options.lua', - 'runtime/lua/vim/shared.lua', - 'runtime/lua/vim/loader.lua', - 'runtime/lua/vim/uri.lua', - 'runtime/lua/vim/ui.lua', - 'runtime/lua/vim/filetype.lua', - 'runtime/lua/vim/keymap.lua', - 'runtime/lua/vim/fs.lua', - 'runtime/lua/vim/highlight.lua', - 'runtime/lua/vim/secure.lua', - 'runtime/lua/vim/version.lua', - 'runtime/lua/vim/_inspector.lua', - 'runtime/lua/vim/snippet.lua', - 'runtime/lua/vim/text.lua', - 'runtime/lua/vim/glob.lua', - 'runtime/lua/vim/_meta/builtin.lua', - 'runtime/lua/vim/_meta/diff.lua', - 'runtime/lua/vim/_meta/mpack.lua', - 'runtime/lua/vim/_meta/json.lua', - 'runtime/lua/vim/_meta/base64.lua', - 'runtime/lua/vim/_meta/regex.lua', - 'runtime/lua/vim/_meta/lpeg.lua', - 'runtime/lua/vim/_meta/re.lua', - 'runtime/lua/vim/_meta/spell.lua', - ], - file_patterns='*.lua', - fn_name_prefix='', - fn_name_fmt=lambda fstem, name: ( - name if fstem in [ 'vim.iter' ] else - f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else - f'vim.{name}' if fstem == '_options' and not name[0].isupper() else - f'{fstem}.{name}' if fstem.startswith('vim') else - name - ), - section_name={ - 'lsp.lua': 'core', - '_inspector.lua': 'inspector', - }, - section_fmt=lambda name: ( - 'Lua module: vim' if name.lower() == '_editor' else - 'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else - f'VIM.{name.upper()}' if name.lower() in [ - 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', - 'regex', 'lpeg', 're', - ] else - 'VIM' if name.lower() == 'builtin' else - f'Lua module: vim.{name.lower()}'), - helptag_fmt=lambda name: ( - '*lua-vim*' if name.lower() == '_editor' else - '*lua-vimscript*' if name.lower() == '_options' else - f'*vim.{name.lower()}*'), - fn_helptag_fmt=lambda fstem, name, istbl: ( - f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else - # Exclude fstem for methods - f'*{name}()*' if ':' in name else - f'*vim.{name}()*' if fstem.lower() == '_editor' else - f'*vim.{name}*' if fstem.lower() == '_options' and istbl else - # Prevents vim.regex.regex - f'*{fstem}()*' if fstem.endswith('.' + name) else - f'*{fstem}.{name}{"" if istbl else "()"}*' - ), - module_override={ - # `shared` functions are exposed on the `vim` module. - 'shared': 'vim', - '_inspector': 'vim', - 'uri': 'vim', - 'ui': 'vim.ui', - 'loader': 'vim.loader', - 'filetype': 'vim.filetype', - 'keymap': 'vim.keymap', - 'fs': 'vim.fs', - 'highlight': 'vim.highlight', - 'secure': 'vim.secure', - 'version': 'vim.version', - 'iter': 'vim.iter', - 'diff': 'vim', - 'builtin': 'vim', - 'mpack': 'vim.mpack', - 'json': 'vim.json', - 'base64': 'vim.base64', - 'regex': 'vim.regex', - 'lpeg': 'vim.lpeg', - 're': 'vim.re', - 'spell': 'vim.spell', - 'snippet': 'vim.snippet', - 'text': 'vim.text', - 'glob': 'vim.glob', - }, - append_only=[ - 'shared.lua', - ], - ), - 'lsp': Config( - mode='lua', - filename='lsp.txt', - section_order=[ - 'lsp.lua', - 'buf.lua', - 'diagnostic.lua', - 'codelens.lua', - 'inlay_hint.lua', - 'tagfunc.lua', - 'semantic_tokens.lua', - 'handlers.lua', - 'util.lua', - 'log.lua', - 'rpc.lua', - 'protocol.lua', - ], - files=[ - 'runtime/lua/vim/lsp', - 'runtime/lua/vim/lsp.lua', - ], - file_patterns='*.lua', - fn_name_prefix='', - section_name={'lsp.lua': 'lsp'}, - section_fmt=lambda name: ( - 'Lua module: vim.lsp' - if name.lower() == 'lsp' - else f'Lua module: vim.lsp.{name.lower()}'), - helptag_fmt=lambda name: ( - '*lsp-core*' - if name.lower() == 'lsp' - else f'*lsp-{name.lower()}*'), - fn_helptag_fmt=lambda fstem, name, istbl: ( - f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else - # HACK. TODO(justinmk): class/structure support in lua2dox - '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else - f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'), - module_override={}, - append_only=[], - ), - 'diagnostic': Config( - mode='lua', - filename='diagnostic.txt', - section_order=[ - 'diagnostic.lua', - ], - files=['runtime/lua/vim/diagnostic.lua'], - file_patterns='*.lua', - fn_name_prefix='', - include_tables=False, - section_name={'diagnostic.lua': 'diagnostic'}, - section_fmt=lambda _: 'Lua module: vim.diagnostic', - helptag_fmt=lambda _: '*diagnostic-api*', - fn_helptag_fmt=lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*', - module_override={}, - append_only=[], - ), - 'treesitter': Config( - mode='lua', - filename='treesitter.txt', - section_order=[ - 'treesitter.lua', - 'language.lua', - 'query.lua', - 'highlighter.lua', - 'languagetree.lua', - 'dev.lua', - ], - files=[ - 'runtime/lua/vim/treesitter.lua', - 'runtime/lua/vim/treesitter/', - ], - file_patterns='*.lua', - fn_name_prefix='', - section_name={}, - section_fmt=lambda name: ( - 'Lua module: vim.treesitter' - if name.lower() == 'treesitter' - else f'Lua module: vim.treesitter.{name.lower()}'), - helptag_fmt=lambda name: ( - '*lua-treesitter-core*' - if name.lower() == 'treesitter' - else f'*lua-treesitter-{name.lower()}*'), - fn_helptag_fmt=lambda fstem, name, istbl: ( - f'*vim.{fstem}.{name}()*' - if fstem == 'treesitter' - else f'*{name}()*' - if name[0].isupper() - else f'*vim.treesitter.{fstem}.{name}()*'), - module_override={}, - append_only=[], - ), -} - -param_exclude = ( - 'channel_id', -) - -# Annotations are displayed as line items after API function descriptions. -annotation_map = { - 'FUNC_API_FAST': '|api-fast|', - 'FUNC_API_TEXTLOCK': 'not allowed when |textlock| is active or in the |cmdwin|', - 'FUNC_API_TEXTLOCK_ALLOW_CMDWIN': 'not allowed when |textlock| is active', - 'FUNC_API_REMOTE_ONLY': '|RPC| only', - 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only', -} - - -def nvim_api_info() -> Tuple[int, bool]: - """Returns NVIM_API_LEVEL, NVIM_API_PRERELEASE from CMakeLists.txt""" - if not hasattr(nvim_api_info, 'LEVEL'): - script_dir = os.path.dirname(os.path.abspath(__file__)) - cmake_file_path = os.path.join(script_dir, '..', 'CMakeLists.txt') - with open(cmake_file_path, 'r') as cmake_file: - cmake_content = cmake_file.read() - - api_level_match = re.search(r'set\(NVIM_API_LEVEL (\d+)\)', cmake_content) - api_prerelease_match = re.search( - r'set\(NVIM_API_PRERELEASE (\w+)\)', cmake_content - ) - - if not api_level_match or not api_prerelease_match: - raise RuntimeError( - 'Could not find NVIM_API_LEVEL or NVIM_API_PRERELEASE in CMakeLists.txt' - ) - - nvim_api_info.LEVEL = int(api_level_match.group(1)) - nvim_api_info.PRERELEASE = api_prerelease_match.group(1).lower() == 'true' - - return nvim_api_info.LEVEL, nvim_api_info.PRERELEASE - - -# Raises an error with details about `o`, if `cond` is in object `o`, -# or if `cond()` is callable and returns True. -def debug_this(o, cond=True): - name = '' - if cond is False: - return - if not isinstance(o, str): - try: - name = o.nodeName - o = o.toprettyxml(indent=' ', newl='\n') - except Exception: - pass - if (cond is True - or (callable(cond) and cond()) - or (not callable(cond) and cond in o)): - raise RuntimeError('xxx: {}\n{}'.format(name, o)) - - -# Appends a message to a list which will be printed on exit. -def msg(s): - msgs.append(s) - - -# Print all collected messages. -def msg_report(): - for m in msgs: - print(f' {m}') - - -# Print collected messages, then throw an exception. -def fail(s): - msg_report() - raise RuntimeError(s) - - -def find_first(parent, name): - """Finds the first matching node within parent.""" - sub = parent.getElementsByTagName(name) - if not sub: - return None - return sub[0] - - -def iter_children(parent, name): - """Yields matching child nodes within parent.""" - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: - yield child - - -def get_child(parent, name): - """Gets the first matching child node.""" - for child in iter_children(parent, name): - return child - return None - - -def self_or_child(n): - """Gets the first child node, or self.""" - if len(n.childNodes) == 0: - return n - return n.childNodes[0] - - -def align_tags(line): - tag_regex = r"\s(\*.+?\*)(?:\s|$)" - tags = re.findall(tag_regex, line) - - if len(tags) > 0: - line = re.sub(tag_regex, "", line) - tags = " " + " ".join(tags) - line = line + (" " * (78 - len(line) - len(tags))) + tags - return line - - -def clean_lines(text): - """Removes superfluous lines. - - The beginning and end of the string is trimmed. Empty lines are collapsed. - """ - return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text)) - - -def is_blank(text): - return '' == clean_lines(text) - - -def get_text(n): - """Recursively concatenates all text in a node tree.""" - text = '' - if n.nodeType == n.TEXT_NODE: - return n.data - if n.nodeName == 'computeroutput': - for node in n.childNodes: - text += get_text(node) - return '`{}`'.format(text) - if n.nodeName == 'sp': # space, used in "programlisting" nodes - return ' ' - for node in n.childNodes: - if node.nodeType == node.TEXT_NODE: - text += node.data - elif node.nodeType == node.ELEMENT_NODE: - text += get_text(node) - return text - - -# Gets the length of the last line in `text`, excluding newline ("\n") char. -def len_lastline(text): - lastnl = text.rfind('\n') - if -1 == lastnl: - return len(text) - if '\n' == text[-1]: - return lastnl - (1 + text.rfind('\n', 0, lastnl)) - return len(text) - (1 + lastnl) - - -def len_lastline_withoutindent(text, indent): - n = len_lastline(text) - return (n - len(indent)) if n > len(indent) else 0 - - -# Returns True if node `n` contains only inline (not block-level) elements. -def is_inline(n): - # if len(n.childNodes) == 0: - # return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput' - for c in n.childNodes: - if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput': - return False - if not is_inline(c): - return False - return True - - -def doc_wrap(text, prefix='', width=70, func=False, indent=None) -> str: - """Wraps text to `width`. - - First line is prefixed with `prefix`, subsequent lines are aligned. - If `func` is True, only wrap at commas. - """ - if not width: - # return prefix + text - return text - - # Whitespace used to indent all lines except the first line. - indent = ' ' * len(prefix) if indent is None else indent - indent_only = (prefix == '' and indent is not None) - - if func: - lines = [prefix] - for part in text.split(', '): - if part[-1] not in ');': - part += ', ' - if len(lines[-1]) + len(part) > width: - lines.append(indent) - lines[-1] += part - return '\n'.join(x.rstrip() for x in lines).rstrip() - - # XXX: Dummy prefix to force TextWrapper() to wrap the first line. - if indent_only: - prefix = indent - - tw = textwrap.TextWrapper(break_long_words=False, - break_on_hyphens=False, - width=width, - initial_indent=prefix, - subsequent_indent=indent) - result = '\n'.join(tw.wrap(text.strip())) - - # XXX: Remove the dummy prefix. - if indent_only: - result = result[len(indent):] - - return result - - -def max_name(names): - if len(names) == 0: - return 0 - return max(len(name) for name in names) - - -def update_params_map(parent, ret_map, width=text_width - indentation): - """Updates `ret_map` with name:desc key-value pairs extracted - from Doxygen XML node `parent`. - """ - params = collections.OrderedDict() - for node in parent.childNodes: - if node.nodeType == node.TEXT_NODE: - continue - name_node = find_first(node, 'parametername') - if name_node.getAttribute('direction') == 'out': - continue - name = get_text(name_node) - if name in param_exclude: - continue - params[name.strip()] = node - max_name_len = max_name(params.keys()) + 8 - # `ret_map` is a name:desc map. - for name, node in params.items(): - desc = '' - desc_node = get_child(node, 'parameterdescription') - if desc_node: - desc = fmt_node_as_vimhelp( - desc_node, width=width, indent=(' ' * max_name_len)) - ret_map[name] = desc - return ret_map - - -def render_node(n: Element, text: str, prefix='', *, - indent: str = '', - width: int = (text_width - indentation), - fmt_vimhelp: bool = False): - """Renders a node as Vim help text, recursively traversing all descendants.""" - - def ind(s): - return s if fmt_vimhelp else '' - - # Get the current column offset from the last line of `text` - # (needed to appropriately wrap multiple and contiguous inline elements) - col_offset: int = len_lastline(text) - - text = '' - # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) - # text += (int(not space_preceding) * ' ') - - if n.nodeName == 'preformatted': - o = get_text(n) - ensure_nl = '' if o[-1] == '\n' else '\n' - if o[0:4] == 'lua\n': - text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1]) - elif o[0:4] == 'vim\n': - text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1]) - elif o[0:5] == 'help\n': - text += o[4:-1] - else: - text += '>{}{}\n<'.format(ensure_nl, o) - elif n.nodeName == 'programlisting': # codeblock (```) - o = get_text(n) - text += '>' - if 'filename' in n.attributes: - filename = n.attributes['filename'].value - text += filename.lstrip('.') - - text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4)) - elif is_inline(n): - o = get_text(n).strip() - if o: - DEL = chr(127) # a dummy character to pad for proper line wrap - assert len(DEL) == 1 - dummy_padding = DEL * max(0, col_offset - len(prefix)) - text += doc_wrap(dummy_padding + o, - prefix=prefix, indent=indent, width=width - ).replace(DEL, "") - elif n.nodeName == 'verbatim': - # TODO: currently we don't use this. The "[verbatim]" hint is there as - # a reminder that we must decide how to format this if we do use it. - text += ' [verbatim] {}'.format(get_text(n)) - elif n.nodeName == 'listitem': - for c in n.childNodes: - result = render_node( - c, - text, - indent=indent + (' ' * len(prefix)), - width=width - ) - if is_blank(result): - continue - text += indent + prefix + result - elif n.nodeName in ('para', 'heading'): - did_prefix = False - for c in n.childNodes: - c_text = render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width) - if (is_inline(c) - and '' != c_text.strip() - and text - and text[-1] not in (' ', '(', '|') - and not c_text.startswith(')')): - text += ' ' - text += c_text - did_prefix = True - elif n.nodeName == 'itemizedlist': - for c in n.childNodes: - text += '{}\n'.format(render_node(c, text, prefix='• ', - indent=indent, width=width)) - elif n.nodeName == 'orderedlist': - i = 1 - for c in n.childNodes: - if is_blank(get_text(c)): - text += '\n' - continue - text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i), - indent=indent, width=width)) - i = i + 1 - elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'): - text += ind(' ') - for c in n.childNodes: - if is_blank(render_node(c, text, prefix='• ', indent=' ', width=width)): - continue - text += render_node(c, text, prefix='• ', indent=' ', width=width) - # text += '\n' - elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'): - text += 'Warning:\n ' - for c in n.childNodes: - text += render_node(c, text, indent=' ', width=width) - text += '\n' - elif n.nodeName == 'simplesect' and 'see' == n.getAttribute('kind'): - text += ind(' ') - # Example: - # <simplesect kind="see"> - # <para>|autocommand|</para> - # </simplesect> - for c in n.childNodes: - text += render_node(c, text, prefix='• ', indent=' ', width=width) - elif n.nodeName == 'simplesect' and 'return' == n.getAttribute('kind'): - text += ind(' ') - for c in n.childNodes: - text += render_node(c, text, indent=' ', width=width) - elif n.nodeName == 'computeroutput': - return get_text(n) - else: - raise RuntimeError('unhandled node type: {}\n{}'.format( - n.nodeName, n.toprettyxml(indent=' ', newl='\n'))) - - return text - - -def para_as_map(parent: Element, - indent: str = '', - width: int = (text_width - indentation), - ): - """Extracts a Doxygen XML <para> node to a map. - - Keys: - 'text': Text from this <para> element - 'note': List of @note strings - 'params': <parameterlist> map - 'return': List of @return strings - 'seealso': List of @see strings - 'xrefs': ? - """ - chunks = { - 'text': '', - 'note': [], - 'params': collections.OrderedDict(), - 'return': [], - 'seealso': [], - 'prerelease': False, - 'xrefs': [] - } - - # Ordered dict of ordered lists. - groups = collections.OrderedDict([ - ('note', []), - ('params', []), - ('return', []), - ('seealso', []), - ('xrefs', []), - ]) - - # Gather nodes into groups. Mostly this is because we want "parameterlist" - # nodes to appear together. - text = '' - kind = '' - if is_inline(parent): - # Flatten inline text from a tree of non-block nodes. - text = doc_wrap(render_node(parent, ""), - indent=indent, width=width) - else: - prev = None # Previous node - for child in parent.childNodes: - if child.nodeName == 'parameterlist': - groups['params'].append(child) - elif child.nodeName == 'xrefsect': - groups['xrefs'].append(child) - elif child.nodeName == 'simplesect': - kind = child.getAttribute('kind') - if kind == 'note': - groups['note'].append(child) - elif kind == 'return': - groups['return'].append(child) - elif kind == 'see': - groups['seealso'].append(child) - elif kind == 'warning': - text += render_node(child, text, indent=indent, width=width) - elif kind == 'since': - since_match = re.match(r'^(\d+)', get_text(child)) - since = int(since_match.group(1)) if since_match else 0 - NVIM_API_LEVEL, NVIM_API_PRERELEASE = nvim_api_info() - if since > NVIM_API_LEVEL or ( - since == NVIM_API_LEVEL and NVIM_API_PRERELEASE - ): - chunks['prerelease'] = True - else: - raise RuntimeError('unhandled simplesect: {}\n{}'.format( - child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) - else: - child_text = render_node(child, text, indent=indent, width=width) - if (prev is not None - and is_inline(self_or_child(prev)) - and is_inline(self_or_child(child)) - and '' != get_text(self_or_child(child)).strip() - and text - and text[-1] not in (' ', '(', '|') - and not child_text.startswith(')')): - text += ' ' - - text += child_text - prev = child - - chunks['text'] += text - - # Generate map from the gathered items. - if len(groups['params']) > 0: - for child in groups['params']: - update_params_map(child, ret_map=chunks['params'], width=width) - for child in groups['note']: - chunks['note'].append(render_node( - child, '', indent=indent, width=width).rstrip()) - for child in groups['return']: - chunks['return'].append(render_node( - child, '', indent=indent, width=width)) - for child in groups['seealso']: - # Example: - # <simplesect kind="see"> - # <para>|autocommand|</para> - # </simplesect> - chunks['seealso'].append(render_node( - child, '', indent=indent, width=width)) - - xrefs = set() - for child in groups['xrefs']: - # XXX: Add a space (or any char) to `title` here, otherwise xrefs - # ("Deprecated" section) acts very weird... - title = get_text(get_child(child, 'xreftitle')) + ' ' - xrefs.add(title) - xrefdesc = get_text(get_child(child, 'xrefdescription')) - chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title), - width=width) + '\n') - - return chunks, xrefs - - -def is_program_listing(para): - """ - Return True if `para` contains a "programlisting" (i.e. a Markdown code - block ```). - - Sometimes a <para> element will have only a single "programlisting" child - node, but othertimes it will have extra whitespace around the - "programlisting" node. - - @param para XML <para> node - @return True if <para> is a programlisting - """ - - # Remove any child text nodes that are only whitespace - children = [ - n for n in para.childNodes - if n.nodeType != n.TEXT_NODE or n.data.strip() != '' - ] - - return len(children) == 1 and children[0].nodeName == 'programlisting' - - -FunctionParam = Tuple[ - str, # type - str, # parameter name -] - -@dataclasses.dataclass -class FunctionDoc: - """Data structure for function documentation. Also exported as msgpack.""" - - annotations: List[str] - """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map""" - - notes: List[Docstring] - """Notes: (@note strings)""" - - signature: str - """Function signature with *tags*.""" - - parameters: List[FunctionParam] - """Parameters: (type, name)""" - - parameters_doc: Dict[str, Docstring] - """Parameters documentation. Key is parameter name, value is doc.""" - - doc: List[Docstring] - """Main description for the function. Separated by paragraph.""" - - return_: List[Docstring] - """Return:, or Return (multiple): (@return strings)""" - - seealso: List[Docstring] - """See also: (@see strings)""" - - xrefs: List[Docstring] - """XRefs. Currently only used to track Deprecated functions.""" - - # for INCLUDE_C_DECL - c_decl: str | None = None - - prerelease: bool = False - - def export_mpack(self) -> Dict[str, Any]: - """Convert a dict to be exported as mpack data.""" - exported = self.__dict__.copy() - del exported['notes'] - del exported['c_decl'] - del exported['prerelease'] - del exported['xrefs'] - exported['return'] = exported.pop('return_') - return exported - - def doc_concatenated(self) -> Docstring: - """Concatenate all the paragraphs in `doc` into a single string, but - remove blank lines before 'programlisting' blocks. #25127 - - BEFORE (without programlisting processing): - ```vimdoc - Example: - - >vim - :echo nvim_get_color_by_name("Pink") - < - ``` - - AFTER: - ```vimdoc - Example: >vim - :echo nvim_get_color_by_name("Pink") - < - ``` - """ - def is_program_listing(paragraph: str) -> bool: - lines = paragraph.strip().split('\n') - return lines[0].startswith('>') and lines[-1] == '<' - - rendered = [] - for paragraph in self.doc: - if is_program_listing(paragraph): - rendered.append(' ') # Example: >vim - elif rendered: - rendered.append('\n\n') - rendered.append(paragraph) - return ''.join(rendered) - - def render(self) -> Docstring: - """Renders function documentation as Vim :help text.""" - rendered_blocks: List[Docstring] = [] - - def fmt_param_doc(m): - """Renders a params map as Vim :help text.""" - max_name_len = max_name(m.keys()) + 4 - out = '' - for name, desc in m.items(): - if name == 'self': - continue - name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) - out += '{}{}\n'.format(name, desc) - return out.rstrip() - - # Generate text from the gathered items. - chunks: List[Docstring] = [self.doc_concatenated()] - - notes = [] - if self.prerelease: - notes = [" This API is pre-release (unstable)."] - notes += self.notes - if len(notes) > 0: - chunks.append('\nNote: ~') - for s in notes: - chunks.append(' ' + s) - - if self.parameters_doc: - chunks.append('\nParameters: ~') - chunks.append(fmt_param_doc(self.parameters_doc)) - - if self.return_: - chunks.append('\nReturn (multiple): ~' if len(self.return_) > 1 - else '\nReturn: ~') - for s in self.return_: - chunks.append(' ' + s) - - if self.seealso: - chunks.append('\nSee also: ~') - for s in self.seealso: - chunks.append(' ' + s) - - # Note: xrefs are currently only used to remark "Deprecated: " - # for deprecated functions; visible when INCLUDE_DEPRECATED is set - for s in self.xrefs: - chunks.append('\n' + s) - - rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) - rendered_blocks.append('') - - return clean_lines('\n'.join(rendered_blocks).strip()) - - -def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=''): - """Renders (nested) Doxygen <para> nodes as Vim :help text. - - Only handles "text" nodes. Used for individual elements (see render_node()) - and in extract_defgroups(). - - NB: Blank lines in a docstring manifest as <para> tags. - """ - rendered_blocks = [] - - for child in parent.childNodes: - para, _ = para_as_map(child, indent, width) - - # 'programlisting' blocks are Markdown code blocks. Do not include - # these as a separate paragraph, but append to the last non-empty line - # in the text - if is_program_listing(child): - while rendered_blocks and rendered_blocks[-1] == '': - rendered_blocks.pop() - rendered_blocks[-1] += ' ' + para['text'] - continue - - # Generate text from the gathered items. - chunks = [para['text']] - - rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) - rendered_blocks.append('') - - return clean_lines('\n'.join(rendered_blocks).strip()) - - -def extract_from_xml(filename, target, *, - width: int, fmt_vimhelp: bool) -> Tuple[ - Dict[FunctionName, FunctionDoc], - Dict[FunctionName, FunctionDoc], -]: - """Extracts Doxygen info as maps without formatting the text. - - Returns two maps: - 1. Functions - 2. Deprecated functions - - The `fmt_vimhelp` variable controls some special cases for use by - fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) - """ - config: Config = CONFIG[target] - - fns: Dict[FunctionName, FunctionDoc] = {} - deprecated_fns: Dict[FunctionName, FunctionDoc] = {} - - dom = minidom.parse(filename) - compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) - for member in dom.getElementsByTagName('memberdef'): - if member.getAttribute('static') == 'yes' or \ - member.getAttribute('kind') != 'function' or \ - member.getAttribute('prot') == 'private' or \ - get_text(get_child(member, 'name')).startswith('_'): - continue - - loc = find_first(member, 'location') - if 'private' in loc.getAttribute('file'): - continue - - return_type = get_text(get_child(member, 'type')) - if return_type == '': - continue - - if 'local_function' in return_type: # Special from lua2dox.lua. - continue - - istbl = return_type.startswith('table') # Special from lua2dox.lua. - if istbl and not config.include_tables: - continue - - if return_type.startswith(('ArrayOf', 'DictionaryOf')): - parts = return_type.strip('_').split('_') - return_type = '{}({})'.format(parts[0], ', '.join(parts[1:])) - - name = get_text(get_child(member, 'name')) - - annotations = get_text(get_child(member, 'argsstring')) - if annotations and ')' in annotations: - annotations = annotations.rsplit(')', 1)[-1].strip() - # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of - # non-void functions. Special-case void functions here. - if name == 'nvim_get_mode' and len(annotations) == 0: - annotations += 'FUNC_API_FAST' - annotations = filter(None, map(lambda x: annotation_map.get(x), - annotations.split())) - - params = [] - type_length = 0 - - for param in iter_children(member, 'param'): - param_type = get_text(get_child(param, 'type')).strip() - param_name = '' - declname = get_child(param, 'declname') - if declname: - param_name = get_text(declname).strip() - elif config.mode == 'lua': - # XXX: this is what lua2dox gives us... - param_name = param_type - param_type = '' - - if param_name in param_exclude: - continue - - if fmt_vimhelp and param_type.endswith('*'): - param_type = param_type.strip('* ') - param_name = '*' + param_name - - type_length = max(type_length, len(param_type)) - params.append((param_type, param_name)) - - # Handle Object Oriented style functions here. - # We make sure they have "self" in the parameters, - # and a parent function - if return_type.startswith('function') \ - and len(return_type.split(' ')) >= 2 \ - and any(x[1] == 'self' for x in params): - split_return = return_type.split(' ') - name = f'{split_return[1]}:{name}' - params = [x for x in params if x[1] != 'self'] - - c_args = [] - for param_type, param_name in params: - c_args.append((' ' if fmt_vimhelp else '') + ( - '%s %s' % (param_type.ljust(type_length), param_name)).strip()) - - if not fmt_vimhelp: - pass - else: - fstem = '?' - if '.' in compoundname: - fstem = compoundname.split('.')[0] - fstem = config.module_override.get(fstem, fstem) - vimtag = config.fn_helptag_fmt(fstem, name, istbl) - - if config.fn_name_fmt: - name = config.fn_name_fmt(fstem, name) - - if istbl: - aopen, aclose = '', '' - else: - aopen, aclose = '(', ')' - - prefix = name + aopen - suffix = ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error', 'Arena', - 'lua_State')) + aclose - - if not fmt_vimhelp: - c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) - signature = prefix + suffix - else: - c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, - ',\n'.join(c_args)), - ' ') - - # Minimum 8 chars between signature and vimtag - lhs = (width - 8) - len(vimtag) - - if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(width) + '\n' - signature += doc_wrap(suffix, width=width, prefix=prefix, - func=True) - else: - signature = prefix + suffix - signature += vimtag.rjust(width - len(signature)) - - # Tracks `xrefsect` titles. As of this writing, used only for separating - # deprecated functions. - xrefs_all = set() - paras: List[Dict[str, Any]] = [] # paras means paragraphs! - brief_desc = find_first(member, 'briefdescription') - if brief_desc: - for child in brief_desc.childNodes: - para, xrefs = para_as_map(child) - paras.append(para) - xrefs_all.update(xrefs) - - desc = find_first(member, 'detaileddescription') - if desc: - paras_detail = [] # override briefdescription - for child in desc.childNodes: - para, xrefs = para_as_map(child) - paras_detail.append(para) - xrefs_all.update(xrefs) - log.debug( - textwrap.indent( - re.sub(r'\n\s*\n+', '\n', - desc.toprettyxml(indent=' ', newl='\n')), - ' ' * indentation)) - - # override briefdescription, if detaileddescription is not empty - # (note: briefdescription can contain some erroneous luadoc - # comments from preceding comments, this is a bug of lua2dox) - if any((para['text'] or para['note'] or para['params'] or - para['return'] or para['seealso'] - ) for para in paras_detail): - paras = paras_detail - - fn = FunctionDoc( - annotations=list(annotations), - notes=[], - signature=signature, - parameters=params, - parameters_doc=collections.OrderedDict(), - doc=[], - return_=[], - seealso=[], - xrefs=[], - ) - - for m in paras: - if m.get('text', ''): - fn.doc.append(m['text']) - if 'params' in m: - # Merge OrderedDicts. - fn.parameters_doc.update(m['params']) - if 'return' in m and len(m['return']) > 0: - fn.return_ += m['return'] - if 'seealso' in m and len(m['seealso']) > 0: - fn.seealso += m['seealso'] - if m.get('prerelease', False): - fn.prerelease = True - if 'note' in m: - fn.notes += m['note'] - if 'xrefs' in m: - fn.xrefs += m['xrefs'] - - if INCLUDE_C_DECL: - fn.c_decl = c_decl - - if 'Deprecated' in str(xrefs_all): - deprecated_fns[name] = fn - elif name.startswith(config.fn_name_prefix): - fns[name] = fn - - # sort functions by name (lexicographically) - fns = collections.OrderedDict(sorted( - fns.items(), - key=lambda key_item_tuple: key_item_tuple[0].lower(), - )) - deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items())) - return fns, deprecated_fns - - -def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: - """Entrypoint for generating Vim :help from from Doxygen XML. - - Returns 2 items: - 1. Vim help text for functions found in `filename`. - 2. Vim help text for deprecated functions. - """ - config: Config = CONFIG[target] - - fns_txt = {} # Map of func_name:vim-help-text. - deprecated_fns_txt = {} # Map of func_name:vim-help-text. - - fns: Dict[FunctionName, FunctionDoc] - deprecated_fns: Dict[FunctionName, FunctionDoc] - fns, deprecated_fns = extract_from_xml( - filename, target, width=text_width, fmt_vimhelp=True) - - def _handle_fn(fn_name: FunctionName, fn: FunctionDoc, - fns_txt: Dict[FunctionName, Docstring], deprecated=False): - # Generate Vim :help for parameters. - - # Generate body from FunctionDoc, not XML nodes - doc = fn.render() - if not doc and fn_name.startswith("nvim__"): - return - if not doc: - doc = ('TODO: Documentation' if not deprecated - else 'Deprecated.') - - # Annotations: put before Parameters - annotations: str = '\n'.join(fn.annotations) - if annotations: - annotations = ('\n\nAttributes: ~\n' + - textwrap.indent(annotations, ' ')) - i = doc.rfind('Parameters: ~') - if i == -1: - doc += annotations - else: - doc = doc[:i] + annotations + '\n\n' + doc[i:] - - # C Declaration: (debug only) - if INCLUDE_C_DECL: - doc += '\n\nC Declaration: ~\n>\n' - assert fn.c_decl is not None - doc += fn.c_decl - doc += '\n<' - - # Start of function documentations. e.g., - # nvim_cmd({*cmd}, {*opts}) *nvim_cmd()* - func_doc = fn.signature + '\n' - func_doc += textwrap.indent(clean_lines(doc), ' ' * indentation) - - # Verbatim handling. - func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) - - def process_helptags(func_doc: str) -> str: - lines: List[str] = func_doc.split('\n') - # skip ">lang ... <" regions - is_verbatim: bool = False - for i in range(len(lines)): - if re.search(' >([a-z])*$', lines[i]): - is_verbatim = True - elif is_verbatim and lines[i].strip() == '<': - is_verbatim = False - if not is_verbatim: - lines[i] = align_tags(lines[i]) - return "\n".join(lines) - - func_doc = process_helptags(func_doc) - - if (fn_name.startswith(config.fn_name_prefix) - and fn_name != "nvim_error_event"): - fns_txt[fn_name] = func_doc - - for fn_name, fn in fns.items(): - _handle_fn(fn_name, fn, fns_txt) - for fn_name, fn in deprecated_fns.items(): - _handle_fn(fn_name, fn, deprecated_fns_txt, deprecated=True) - - return ( - '\n\n'.join(list(fns_txt.values())), - '\n\n'.join(list(deprecated_fns_txt.values())), - ) - - -def delete_lines_below(filename, tokenstr): - """Deletes all lines below the line containing `tokenstr`, the line itself, - and one line above it. - """ - lines = open(filename).readlines() - i = 0 - found = False - for i, line in enumerate(lines, 1): - if tokenstr in line: - found = True - break - if not found: - raise RuntimeError(f'not found: "{tokenstr}"') - i = max(0, i - 2) - with open(filename, 'wt') as fp: - fp.writelines(lines[0:i]) - - -def extract_defgroups(base: str, dom: Document) -> Dict[SectionName, Docstring]: - '''Generate module-level (section) docs (@defgroup).''' - section_docs = {} - - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'group': - continue - - # Doxygen "@defgroup" directive. - groupname = get_text(find_first(compound, 'name')) - groupxml = os.path.join(base, '%s.xml' % - compound.getAttribute('refid')) - - group_parsed = minidom.parse(groupxml) - doc_list = [] - brief_desc = find_first(group_parsed, 'briefdescription') - if brief_desc: - for child in brief_desc.childNodes: - doc_list.append(fmt_node_as_vimhelp(child)) - - desc = find_first(group_parsed, 'detaileddescription') - if desc: - doc = fmt_node_as_vimhelp(desc) - - if doc: - doc_list.append(doc) - - # Can't use '.' in @defgroup, so convert to '--' - # "vim.json" => "vim-dot-json" - groupname = groupname.replace('-dot-', '.') - - section_docs[groupname] = "\n".join(doc_list) - - return section_docs - - -@dataclasses.dataclass -class Section: - """Represents a section. Includes section heading (defgroup) - and all the FunctionDoc that belongs to this section.""" - - name: str - '''Name of the section. Usually derived from basename of lua/c src file. - Example: "Autocmd".''' - - title: str - '''Formatted section config. see config.section_fmt(). - Example: "Autocmd Functions". ''' - - helptag: str - '''see config.helptag_fmt(). Example: *api-autocmd*''' - - @property - def id(self) -> str: - '''section id: Module/Section id matched against @defgroup. - e.g., "*api-autocmd*" => "api-autocmd" - ''' - return self.helptag.strip('*') - - doc: str = "" - '''Section heading docs extracted from @defgroup.''' - - # TODO: Do not carry rendered text, but handle FunctionDoc for better OOP - functions_text: Docstring | None = None - '''(Rendered) doc of all the functions that belong to this section.''' - - deprecated_functions_text: Docstring | None = None - '''(Rendered) doc of all the deprecated functions that belong to this - section.''' - - def __repr__(self): - return f"Section(title='{self.title}', helptag='{self.helptag}')" - - @classmethod - def make_from(cls, filename: str, config: Config, - section_docs: Dict[SectionName, str], - *, - functions_text: Docstring, - deprecated_functions_text: Docstring, - ): - # filename: e.g., 'autocmd.c' - # name: e.g. 'autocmd' - name = os.path.splitext(filename)[0].lower() - - # section name: e.g. "Autocmd" - sectname: SectionName - sectname = name.upper() if name == 'ui' else name.title() - sectname = config.section_name.get(filename, sectname) - - # Formatted (this is what's going to be written in the vimdoc) - # e.g., "Autocmd Functions" - title: str = config.section_fmt(sectname) - - # section tag: e.g., "*api-autocmd*" - section_tag: str = config.helptag_fmt(sectname) - - section = cls(name=sectname, title=title, helptag=section_tag, - functions_text=functions_text, - deprecated_functions_text=deprecated_functions_text, - ) - section.doc = section_docs.get(section.id) or '' - return section - - def render(self, add_header=True) -> str: - """Render as vimdoc.""" - doc = '' - - if add_header: - doc += SECTION_SEP - doc += '\n{}{}'.format( - self.title, - self.helptag.rjust(text_width - len(self.title)) - ) - - if self.doc: - doc += '\n\n' + self.doc - - if self.functions_text: - doc += '\n\n' + self.functions_text - - if INCLUDE_DEPRECATED and self.deprecated_functions_text: - doc += f'\n\n\nDeprecated {self.name} Functions: ~\n\n' - doc += self.deprecated_functions_text - - return doc - - def __bool__(self) -> bool: - """Whether this section has contents. Used for skipping empty ones.""" - return bool(self.doc or self.functions_text or - (INCLUDE_DEPRECATED and self.deprecated_functions_text)) - - -def main(doxygen_config, args): - """Generates: - - 1. Vim :help docs - 2. *.mpack files for use by API clients - - Doxygen is called and configured through stdin. - """ - for target in CONFIG: - if args.target is not None and target != args.target: - continue - - config: Config = CONFIG[target] - - mpack_file = os.path.join( - base_dir, 'runtime', 'doc', - config.filename.replace('.txt', '.mpack')) - if os.path.exists(mpack_file): - os.remove(mpack_file) - - output_dir = out_dir.format(target=target) - log.info("Generating documentation for %s in folder %s", - target, output_dir) - debug = args.log_level >= logging.DEBUG - p = subprocess.Popen( - ['doxygen', '-'], - stdin=subprocess.PIPE, - # silence warnings - # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found - stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL)) - p.communicate( - doxygen_config.format( - input=' '.join([f'"{file}"' for file in config.files]), - output=output_dir, - filter=filter_cmd, - file_patterns=config.file_patterns) - .encode('utf8') - ) - if p.returncode: - sys.exit(p.returncode) - - # Collects all functions as each module is processed. - fn_map_full: Dict[FunctionName, FunctionDoc] = {} - # key: filename (e.g. autocmd.c) - sections: Dict[str, Section] = {} - - base = os.path.join(output_dir, 'xml') - dom = minidom.parse(os.path.join(base, 'index.xml')) - - # Collect all @defgroups (section headings after the '===...' separator - section_docs: Dict[SectionName, Docstring] = extract_defgroups(base, dom) - - # Generate docs for all functions in the current module. - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'file': - continue - - filename = get_text(find_first(compound, 'name')) - if not ( - filename.endswith('.c') or - filename.endswith('.lua') - ): - continue - - xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) - - # Extract unformatted (*.mpack). - fn_map, _ = extract_from_xml( - xmlfile, target, width=9999, fmt_vimhelp=False) - - # Extract formatted (:help). - functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( - xmlfile, target) - - if not functions_text and not deprecated_text: - continue - - filename = os.path.basename(filename) - - section: Section = Section.make_from( - filename, config, section_docs, - functions_text=functions_text, - deprecated_functions_text=deprecated_text, - ) - - if section: # if not empty - sections[filename] = section - fn_map_full.update(fn_map) - else: - log.debug("Skipping empty section: %s", section) - - if len(sections) == 0: - fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') - if len(sections) > len(config.section_order): - raise RuntimeError( - '{}: found new modules {}; ' - 'update the "section_order" map'.format( - target, - set(sections).difference(config.section_order)) - ) - first_section_tag = sections[config.section_order[0]].helptag - - docs = '' - - for filename in config.section_order: - try: - section: Section = sections.pop(filename) - except KeyError: - msg(f'warning: empty docs, skipping (target={target}): {filename}') - msg(f' existing docs: {sections.keys()}') - continue - - add_sep_and_header = filename not in config.append_only - docs += section.render(add_header=add_sep_and_header) - docs += '\n\n\n' - - docs = docs.rstrip() + '\n\n' - docs += f' vim:tw=78:ts=8:sw={indentation}:sts={indentation}:et:ft=help:norl:\n' - - doc_file = os.path.join(base_dir, 'runtime', 'doc', config.filename) - - if os.path.exists(doc_file): - delete_lines_below(doc_file, first_section_tag) - with open(doc_file, 'ab') as fp: - fp.write(docs.encode('utf8')) - - fn_map_full_exported = collections.OrderedDict(sorted( - (name, fn_doc.export_mpack()) for (name, fn_doc) in fn_map_full.items() - )) - with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(fn_map_full_exported, use_bin_type=True)) # type: ignore - - if not args.keep_tmpfiles: - shutil.rmtree(output_dir) - - msg_report() - - -def filter_source(filename, keep_tmpfiles): - output_dir = out_dir.format(target='lua2dox') - name, extension = os.path.splitext(filename) - if extension == '.lua': - args = [str(nvim), '-l', lua2dox, filename] + (['--outdir', output_dir] if keep_tmpfiles else []) - p = subprocess.run(args, stdout=subprocess.PIPE) - op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8')) - print(op) - else: - """Filters the source to fix macros that confuse Doxygen.""" - with open(filename, 'rt') as fp: - print(re.sub(r'^(ArrayOf|DictionaryOf|Dict)(\(.*?\))', - lambda m: m.group(1)+'_'.join( - re.split(r'[^\w]+', m.group(2))), - fp.read(), flags=re.M)) - - -def parse_args(): - targets = ', '.join(CONFIG.keys()) - ap = argparse.ArgumentParser( - description="Generate helpdoc from source code") - ap.add_argument( - "--log-level", "-l", choices=LOG_LEVELS.keys(), - default=logging.getLevelName(logging.ERROR), help="Set log verbosity" - ) - ap.add_argument('source_filter', nargs='*', - help="Filter source file(s)") - ap.add_argument('-k', '--keep-tmpfiles', action='store_true', - help="Keep temporary files (tmp-xx-doc/ directories, including tmp-lua2dox-doc/ for lua2dox.lua quasi-C output)") - ap.add_argument('-t', '--target', - help=f'One of ({targets}), defaults to "all"') - return ap.parse_args() - - -Doxyfile = textwrap.dedent(''' - OUTPUT_DIRECTORY = {output} - INPUT = {input} - INPUT_ENCODING = UTF-8 - FILE_PATTERNS = {file_patterns} - RECURSIVE = YES - INPUT_FILTER = "{filter}" - EXCLUDE = - EXCLUDE_SYMLINKS = NO - EXCLUDE_PATTERNS = */private/* */health.lua */_*.lua - EXCLUDE_SYMBOLS = - EXTENSION_MAPPING = lua=C - EXTRACT_PRIVATE = NO - - GENERATE_HTML = NO - GENERATE_DOCSET = NO - GENERATE_HTMLHELP = NO - GENERATE_QHP = NO - GENERATE_TREEVIEW = NO - GENERATE_LATEX = NO - GENERATE_RTF = NO - GENERATE_MAN = NO - GENERATE_DOCBOOK = NO - GENERATE_AUTOGEN_DEF = NO - - GENERATE_XML = YES - XML_OUTPUT = xml - XML_PROGRAMLISTING = NO - - ENABLE_PREPROCESSING = YES - MACRO_EXPANSION = YES - EXPAND_ONLY_PREDEF = NO - MARKDOWN_SUPPORT = YES -''') - -if __name__ == "__main__": - args = parse_args() - print("Setting log level to %s" % args.log_level) - args.log_level = LOG_LEVELS[args.log_level] - log.setLevel(args.log_level) - log.addHandler(logging.StreamHandler()) - - # When invoked as a filter, args won't be passed, so use an env var. - if args.keep_tmpfiles: - os.environ['NVIM_KEEP_TMPFILES'] = '1' - keep_tmpfiles = ('NVIM_KEEP_TMPFILES' in os.environ) - - if len(args.source_filter) > 0: - filter_source(args.source_filter[0], keep_tmpfiles) - else: - main(Doxyfile, args) - -# vim: set ft=python ts=4 sw=4 tw=79 et : diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua deleted file mode 100644 index 0b3daa59b2..0000000000 --- a/scripts/lua2dox.lua +++ /dev/null @@ -1,544 +0,0 @@ ------------------------------------------------------------------------------ --- Copyright (C) 2012 by Simon Dales -- --- simon@purrsoft.co.uk -- --- -- --- This program is free software; you can redistribute it and/or modify -- --- it under the terms of the GNU General Public License as published by -- --- the Free Software Foundation; either version 2 of the License, or -- --- (at your option) any later version. -- --- -- --- This program is distributed in the hope that it will be useful, -- --- but WITHOUT ANY WARRANTY; without even the implied warranty of -- --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- --- GNU General Public License for more details. -- --- -- --- You should have received a copy of the GNU General Public License -- --- along with this program; if not, write to the -- --- Free Software Foundation, Inc., -- --- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -- ------------------------------------------------------------------------------ - ---[[! -Lua-to-Doxygen converter - -Partially from lua2dox -http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm - -RUNNING -------- - -This script "lua2dox.lua" gets called by "gen_vimdoc.py". - -DEBUGGING/DEVELOPING ---------------------- - -1. To debug, run gen_vimdoc.py with --keep-tmpfiles: - python3 scripts/gen_vimdoc.py -t treesitter --keep-tmpfiles -2. The filtered result will be written to ./tmp-lua2dox-doc/….lua.c - -Doxygen must be on your system. You can experiment like so: - -- Run "doxygen -g" to create a default Doxyfile. -- Then alter it to let it recognise lua. Add the following line: - FILE_PATTERNS = *.lua -- Then run "doxygen". - -The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. -It only has to be good enough for doxygen to see it as legal. - -One limitation is that each line is treated separately (except for long comments). -The implication is that class and function declarations must be on the same line. - -There is hack that will insert the "missing" close paren. -The effect is that you will get the function documented, but not with the parameter list you might expect. -]] - -local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' } - -local luacats_parser = require('src/nvim/generators/luacats_grammar') - -local debug_outfile = nil --- @type string? -local debug_output = {} - ---- write to stdout ---- @param str? string -local function write(str) - if not str then - return - end - - io.write(str) - if debug_outfile then - table.insert(debug_output, str) - end -end - ---- write to stdout ---- @param str? string -local function writeln(str) - write(str) - write('\n') -end - ---- an input file buffer ---- @class StreamRead ---- @field currentLine string? ---- @field contentsLen integer ---- @field currentLineNo integer ---- @field filecontents string[] -local StreamRead = {} - ---- @return StreamRead ---- @param filename string -function StreamRead.new(filename) - assert(filename, ('invalid file: %s'):format(filename)) - -- get lines from file - -- syphon lines to our table - local filecontents = {} --- @type string[] - for line in io.lines(filename) do - filecontents[#filecontents + 1] = line - end - - return setmetatable({ - filecontents = filecontents, - contentsLen = #filecontents, - currentLineNo = 1, - }, { __index = StreamRead }) -end - --- get a line -function StreamRead:getLine() - if self.currentLine then - self.currentLine = nil - return self.currentLine - end - - -- get line - if self.currentLineNo <= self.contentsLen then - local line = self.filecontents[self.currentLineNo] - self.currentLineNo = self.currentLineNo + 1 - return line - end - - return '' -end - --- save line fragment ---- @param line_fragment string -function StreamRead:ungetLine(line_fragment) - self.currentLine = line_fragment -end - --- is it eof? -function StreamRead:eof() - return not self.currentLine and self.currentLineNo > self.contentsLen -end - --- input filter ---- @class Lua2DoxFilter -local Lua2DoxFilter = { - generics = {}, --- @type table<string,string> - block_ignore = false, --- @type boolean -} -setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter }) - -function Lua2DoxFilter:reset() - self.generics = {} - self.block_ignore = false -end - ---- trim comment off end of string ---- ---- @param line string ---- @return string, string? -local function removeCommentFromLine(line) - local pos_comment = line:find('%-%-') - if not pos_comment then - return line - end - return line:sub(1, pos_comment - 1), line:sub(pos_comment) -end - ---- @param parsed luacats.Return ---- @return string -local function get_return_type(parsed) - local elems = {} --- @type string[] - for _, v in ipairs(parsed) do - local e = v.type --- @type string - if v.name then - e = e .. ' ' .. v.name --- @type string - end - elems[#elems + 1] = e - end - return '(' .. table.concat(elems, ', ') .. ')' -end - ---- @param name string ---- @return string -local function process_name(name, optional) - if optional then - name = name:sub(1, -2) --- @type string - end - return name -end - ---- @param ty string ---- @param generics table<string,string> ---- @return string -local function process_type(ty, generics, optional) - -- replace generic types - for k, v in pairs(generics) do - ty = ty:gsub(k, v) --- @type string - end - - -- strip parens - ty = ty:gsub('^%((.*)%)$', '%1') - - if optional and not ty:find('nil') then - ty = ty .. '?' - end - - -- remove whitespace in unions - ty = ty:gsub('%s*|%s*', '|') - - -- replace '|nil' with '?' - ty = ty:gsub('|nil', '?') - ty = ty:gsub('nil|(.*)', '%1?') - - return '(`' .. ty .. '`)' -end - ---- @param parsed luacats.Param ---- @param generics table<string,string> ---- @return string -local function process_param(parsed, generics) - local name, ty = parsed.name, parsed.type - local optional = vim.endswith(name, '?') - - return table.concat({ - '/// @param', - process_name(name, optional), - process_type(ty, generics, optional), - parsed.desc, - }, ' ') -end - ---- @param parsed luacats.Return ---- @param generics table<string,string> ---- @return string -local function process_return(parsed, generics) - local ty, name --- @type string, string - if #parsed == 1 then - ty, name = parsed[1].type, parsed[1].name or '' - else - ty, name = get_return_type(parsed), '' - end - - local optional = vim.endswith(name, '?') - - return table.concat({ - '/// @return', - process_type(ty, generics, optional), - process_name(name, optional), - parsed.desc, - }, ' ') -end - ---- Processes "@…" directives in a docstring line. ---- ---- @param line string ---- @return string? -function Lua2DoxFilter:process_magic(line) - line = line:gsub('^%s+@', '@') - line = line:gsub('@package', '@private') - line = line:gsub('@nodoc', '@private') - - if self.block_ignore then - return '// gg:" ' .. line .. '"' - end - - if not vim.startswith(line, '@') then -- it's a magic comment - return '/// ' .. line - end - - local magic_split = vim.split(line, ' ', { plain = true }) - local directive = magic_split[1] - - if - vim.list_contains({ - '@cast', - '@diagnostic', - '@overload', - '@meta', - '@type', - }, directive) - then - -- Ignore LSP directives - return '// gg:"' .. line .. '"' - elseif directive == '@defgroup' or directive == '@addtogroup' then - -- Can't use '.' in defgroup, so convert to '--' - return '/// ' .. line:gsub('%.', '-dot-') - end - - if directive == '@alias' then - -- this contiguous block should be all ignored. - self.block_ignore = true - return '// gg:"' .. line .. '"' - end - - -- preprocess line before parsing - if directive == '@param' or directive == '@return' then - for _, type in ipairs(TYPES) do - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') - line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') - - line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1') - line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1') - line = line:gsub('^@return%s+.*%((' .. type .. '%?)%)', '@return %1') - end - end - - local parsed = luacats_parser:match(line) - - if not parsed then - return '/// ' .. line - end - - local kind = parsed.kind - - if kind == 'generic' then - self.generics[parsed.name] = parsed.type or 'any' - return - elseif kind == 'param' then - return process_param(parsed --[[@as luacats.Param]], self.generics) - elseif kind == 'return' then - return process_return(parsed --[[@as luacats.Return]], self.generics) - end - - error(string.format('unhandled parsed line %q: %s', line, parsed)) -end - ---- @param line string ---- @param in_stream StreamRead ---- @return string -function Lua2DoxFilter:process_block_comment(line, in_stream) - local comment_parts = {} --- @type string[] - local done --- @type boolean? - - while not done and not in_stream:eof() do - local thisComment --- @type string? - local closeSquare = line:find(']]') - if not closeSquare then -- need to look on another line - thisComment = line .. '\n' - line = in_stream:getLine() - else - thisComment = line:sub(1, closeSquare - 1) - done = true - - -- unget the tail of the line - -- in most cases it's empty. This may make us less efficient but - -- easier to program - in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2))) - end - comment_parts[#comment_parts + 1] = thisComment - end - - local comment = table.concat(comment_parts) - - if comment:sub(1, 1) == '@' then -- it's a long magic comment - return '/*' .. comment .. '*/ ' - end - - -- discard - return '/* zz:' .. comment .. '*/ ' -end - ---- @param line string ---- @return string -function Lua2DoxFilter:process_function_header(line) - local pos_fn = assert(line:find('function')) - -- we've got a function - local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8))) - - if fn:sub(1, 1) == '(' then - -- it's an anonymous function - return '// ZZ: ' .. line - end - -- fn has a name, so is interesting - - -- want to fix for iffy declarations - if fn:find('[%({]') then - -- we might have a missing close paren - if not fn:find('%)') then - fn = fn .. ' ___MissingCloseParenHere___)' - end - end - - -- Big hax - if fn:find(':') then - fn = fn:gsub(':', '.', 1) - - local paren_start = fn:find('(', 1, true) - local paren_finish = fn:find(')', 1, true) - - -- Nothing in between the parens - local comma --- @type string - if paren_finish == paren_start + 1 then - comma = '' - else - comma = ', ' - end - - fn = fn:sub(1, paren_start) .. 'self' .. comma .. fn:sub(paren_start + 1) - end - - if line:match('local') then - -- Special: tell gen_vimdoc.py this is a local function. - return 'local_function ' .. fn .. '{}' - end - - -- add vanilla function - return 'function ' .. fn .. '{}' -end - ---- @param line string ---- @param in_stream StreamRead ---- @return string? -function Lua2DoxFilter:process_line(line, in_stream) - local line_raw = line - line = vim.trim(line) - - if vim.startswith(line, '---') then - return Lua2DoxFilter:process_magic(line:sub(4)) - end - - if vim.startswith(line, '--' .. '[[') then -- it's a long comment - return Lua2DoxFilter:process_block_comment(line:sub(5), in_stream) - end - - -- Hax... I'm sorry - -- M.fun = vim.memoize(function(...) - -- -> - -- function M.fun(...) - line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') - - if line:find('^function') or line:find('^local%s+function') then - return Lua2DoxFilter:process_function_header(line) - end - - if not line:match('^local') then - local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=') - if v and v:match('%.') then - -- Special: this lets gen_vimdoc.py handle tables. - return 'table ' .. v .. '() {}' - end - end - - if #line > 0 then -- we don't know what this line means, so just comment it out - return '// zz: ' .. line - end - - return '' -end - --- Processes the file and writes filtered output to stdout. ----@param filename string -function Lua2DoxFilter:filter(filename) - local in_stream = StreamRead.new(filename) - - local last_was_magic = false - - while not in_stream:eof() do - local line = in_stream:getLine() - - local out_line = self:process_line(line, in_stream) - - if not vim.startswith(vim.trim(line), '---') then - self:reset() - end - - if out_line then - -- Ensure all magic blocks associate with some object to prevent doxygen - -- from getting confused. - if vim.startswith(out_line, '///') then - last_was_magic = true - else - if last_was_magic and out_line:match('^// zz: [^-]+') then - writeln('local_function _ignore() {}') - end - last_was_magic = false - end - writeln(out_line) - end - end -end - ---- @class TApp ---- @field timestamp string|osdate ---- @field name string ---- @field version string ---- @field copyright string ---- this application -local TApp = { - timestamp = os.date('%c %Z', os.time()), - name = 'Lua2DoX', - version = '0.2 20130128', - copyright = 'Copyright (c) Simon Dales 2012-13', -} - -setmetatable(TApp, { __index = TApp }) - -function TApp:getRunStamp() - return self.name .. ' (' .. self.version .. ') ' .. self.timestamp -end - -function TApp:getVersion() - return self.name .. ' (' .. self.version .. ') ' -end - ---main - -if arg[1] == '--help' then - writeln(TApp:getVersion()) - writeln(TApp.copyright) - writeln([[ - run as: - nvim -l scripts/lua2dox.lua <param> - -------------- - Param: - <filename> : interprets filename - --version : show version/copyright info - --help : this help text]]) -elseif arg[1] == '--version' then - writeln(TApp:getVersion()) - writeln(TApp.copyright) -else -- It's a filter. - local filename = arg[1] - - if arg[2] == '--outdir' then - local outdir = arg[3] - if - type(outdir) ~= 'string' - or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) - then - error(('invalid --outdir: "%s"'):format(tostring(outdir))) - end - vim.fn.mkdir(outdir, 'p') - debug_outfile = string.format('%s/%s.c', outdir, vim.fs.basename(filename)) - end - - Lua2DoxFilter:filter(filename) - - -- output the tail - writeln('// #######################') - writeln('// app run:' .. TApp:getRunStamp()) - writeln('// #######################') - writeln() - - if debug_outfile then - local f = assert(io.open(debug_outfile, 'w')) - f:write(table.concat(debug_output)) - f:close() - end -end diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua new file mode 100644 index 0000000000..ee0f9d8e87 --- /dev/null +++ b/scripts/luacats_grammar.lua @@ -0,0 +1,218 @@ +--[[! +LPEG grammar for LuaCATS +]] + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local Ct, Cg = lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local nl = P('\r\n') + P('\n') +local ws = rep1(S(' \t') + nl) +local fill = opt(ws) + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local num = R('09') +local ident = letter * rep(letter + num + S '-.') +local string_single = P "'" * rep(any - P "'") * P "'" +local string_double = P '"' * rep(any - P '"') * P '"' + +local literal = (string_single + string_double + (opt(P '-') * num) + P 'false' + P 'true') + +local lname = (ident + P '...') * opt(P '?') + +--- @param x string +local function Pf(x) + return fill * P(x) * fill +end + +--- @param x string +local function Sf(x) + return fill * S(x) * fill +end + +--- @param x vim.lpeg.Pattern +local function comma(x) + return x * rep(Pf ',' * x) +end + +--- @param x vim.lpeg.Pattern +local function parenOpt(x) + return (Pf('(') * x * fill * P(')')) + x +end + +--- @type table<string,vim.lpeg.Pattern> +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +local desc_delim = Sf '#:' + ws + +--- @class nvim.luacats.Param +--- @field kind 'param' +--- @field name string +--- @field type string +--- @field desc? string + +--- @class nvim.luacats.Return +--- @field kind 'return' +--- @field [integer] { type: string, name?: string} +--- @field desc? string + +--- @class nvim.luacats.Generic +--- @field kind 'generic' +--- @field name string +--- @field type? string + +--- @class nvim.luacats.Class +--- @field kind 'class' +--- @field name string +--- @field parent? string + +--- @class nvim.luacats.Field +--- @field kind 'field' +--- @field name string +--- @field type string +--- @field desc? string +--- @field access? 'private'|'protected'|'package' + +--- @class nvim.luacats.Note +--- @field desc? string + +--- @alias nvim.luacats.grammar.result +--- | nvim.luacats.Param +--- | nvim.luacats.Return +--- | nvim.luacats.Generic +--- | nvim.luacats.Class +--- | nvim.luacats.Field +--- | nvim.luacats.Note + +--- @class nvim.luacats.grammar +--- @field match fun(self, input: string): nvim.luacats.grammar.result? + +local grammar = P { + rep1(P('@') * (v.ats + v.ext_ats)), + + ats = v.at_param + + v.at_return + + v.at_type + + v.at_cast + + v.at_generic + + v.at_class + + v.at_field + + v.at_access + + v.at_deprecated + + v.at_alias + + v.at_enum + + v.at_see + + v.at_diagnostic + + v.at_overload + + v.at_meta, + + ext_ats = v.ext_at_note + v.ext_at_since + v.ext_at_nodoc + v.ext_at_brief, + + at_param = Ct( + Cg(P('param'), 'kind') + * ws + * Cg(lname, 'name') + * ws + * parenOpt(Cg(v.ltype, 'type')) + * opt(desc_delim * Cg(rep(any), 'desc')) + ), + + at_return = Ct( + Cg(P('return'), 'kind') + * ws + * parenOpt(comma(Ct(Cg(v.ltype, 'type') * opt(ws * Cg(ident, 'name'))))) + * opt(desc_delim * Cg(rep(any), 'desc')) + ), + + at_type = Ct( + Cg(P('type'), 'kind') + * ws + * parenOpt(comma(Ct(Cg(v.ltype, 'type')))) + * opt(desc_delim * Cg(rep(any), 'desc')) + ), + + at_cast = Ct( + Cg(P('cast'), 'kind') * ws * Cg(lname, 'name') * ws * opt(Sf('+-')) * Cg(v.ltype, 'type') + ), + + at_generic = Ct( + Cg(P('generic'), 'kind') * ws * Cg(ident, 'name') * opt(Pf ':' * Cg(v.ltype, 'type')) + ), + + at_class = Ct( + Cg(P('class'), 'kind') + * ws + * opt(P('(exact)') * ws) + * Cg(lname, 'name') + * opt(Pf(':') * Cg(lname, 'parent')) + ), + + at_field = Ct( + Cg(P('field'), 'kind') + * ws + * opt(Cg(Pf('private') + Pf('package') + Pf('protected'), 'access')) + * Cg(lname, 'name') + * ws + * Cg(v.ltype, 'type') + * opt(desc_delim * Cg(rep(any), 'desc')) + ), + + at_access = Ct(Cg(P('private') + P('protected') + P('package'), 'kind')), + + at_deprecated = Ct(Cg(P('deprecated'), 'kind')), + + -- Types may be provided on subsequent lines + at_alias = Ct(Cg(P('alias'), 'kind') * ws * Cg(lname, 'name') * opt(ws * Cg(v.ltype, 'type'))), + + at_enum = Ct(Cg(P('enum'), 'kind') * ws * Cg(lname, 'name')), + + at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + at_diagnostic = Ct(Cg(P('diagnostic'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + at_overload = Ct(Cg(P('overload'), 'kind') * ws * Cg(v.ltype, 'type')), + at_meta = Ct(Cg(P('meta'), 'kind')), + + --- Custom extensions + ext_at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), + + -- TODO only consume 1 line + ext_at_since = Ct(Cg(P('since'), 'kind') * ws * Cg(rep(any), 'desc')), + + ext_at_nodoc = Ct(Cg(P('nodoc'), 'kind')), + ext_at_brief = Ct(Cg(P('brief'), 'kind') * opt(ws * Cg(rep(any), 'desc'))), + + ltype = v.ty_union + Pf '(' * v.ty_union * fill * P ')', + + ty_union = v.ty_opt * rep(Pf '|' * v.ty_opt), + ty = v.ty_fun + ident + v.ty_table + literal, + ty_param = Pf '<' * comma(v.ltype) * fill * P '>', + ty_opt = v.ty * opt(v.ty_param) * opt(P '[]') * opt(P '?'), + + table_key = (Pf '[' * literal * Pf ']') + lname, + table_elem = v.table_key * Pf ':' * v.ltype, + ty_table = Pf '{' * comma(v.table_elem) * Pf '}', + + fun_param = lname * opt(Pf ':' * v.ltype), + ty_fun = Pf 'fun(' * rep(comma(v.fun_param)) * fill * P ')' * opt(Pf ':' * comma(v.ltype)), +} + +return grammar --[[@as nvim.luacats.grammar]] diff --git a/scripts/luacats_parser.lua b/scripts/luacats_parser.lua new file mode 100644 index 0000000000..520272d1dc --- /dev/null +++ b/scripts/luacats_parser.lua @@ -0,0 +1,521 @@ +local luacats_grammar = require('scripts.luacats_grammar') + +--- @class nvim.luacats.parser.param +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.note +--- @field desc string + +--- @class nvim.luacats.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.luacats.parser.alias +--- @field kind 'alias' +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.fun +--- @field name string +--- @field params nvim.luacats.parser.param[] +--- @field returns nvim.luacats.parser.return[] +--- @field desc string +--- @field access? 'private'|'package'|'protected' +--- @field class? string +--- @field module? string +--- @field modvar? string +--- @field classvar? string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field generics? table<string,string> +--- @field table? true +--- @field notes? nvim.luacats.parser.note[] +--- @field see? nvim.luacats.parser.note[] + +--- @class nvim.luacats.parser.field +--- @field name string +--- @field type string +--- @field desc string +--- @field access? 'private'|'package'|'protected' + +--- @class nvim.luacats.parser.class +--- @field kind 'class' +--- @field name string +--- @field desc string +--- @field fields nvim.luacats.parser.field[] +--- @field notes? string[] + +--- @class nvim.luacats.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.luacats.parser.obj +--- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.luacats.parser.obj +--- | nvim.luacats.parser.class +--- | nvim.luacats.parser.fun +--- | nvim.luacats.parser.brief + +-- Remove this when we document classes properly +--- Some doc lines have the form: +--- param name some.complex.type (table) description +--- if so then transform the line to remove the complex type: +--- param name (table) description +--- @param line string +local function use_type_alt(line) + for _, type in ipairs({ 'table', 'function' }) do + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') + + line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1') + line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1') + line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1') + end + return line +end + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.luacats.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.luacats.parser.State +local function process_doc_line(line, state) + line = line:sub(4):gsub('^%s+@', '@') + line = use_type_alt(line) + + local parsed = luacats_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + local kind = parsed.kind + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'class' then + --- @cast parsed nvim.luacats.Class + state.cur_obj = { + kind = 'class', + name = parsed.name, + parent = parsed.parent, + desc = '', + fields = {}, + } + elseif kind == 'field' then + --- @cast parsed nvim.luacats.Field + if not parsed.access then + parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil + if parsed.desc then + parsed.desc = vim.trim(parsed.desc) + end + table.insert(cur_obj.fields, parsed) + end + state.doc_lines = nil + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + if vim.endswith(parsed.name, '?') then + parsed.name = parsed.name:sub(1, -2) + parsed.type = parsed.type .. '?' + end + state.last_doc_item = { + name = parsed.name, + type = parsed.type, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = cur_obj.returns or {} + for _, t in ipairs(parsed) do + table.insert(cur_obj.returns, { + name = t.name, + type = t.type, + desc = parsed.desc, + }) + end + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[#cur_obj.returns] + elseif kind == 'private' then + cur_obj.access = 'private' + elseif kind == 'package' then + cur_obj.access = 'package' + elseif kind == 'protected' then + cur_obj.access = 'protected' + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + elseif kind == 'type' then + cur_obj.desc = parsed.desc + parsed.desc = nil + parsed.kind = nil + cur_obj.type = parsed + elseif kind == 'alias' then + state.cur_obj = { + kind = 'alias', + desc = parsed.desc, + } + elseif kind == 'enum' then + -- TODO + state.doc_lines = nil + elseif + vim.tbl_contains({ + 'diagnostic', + 'cast', + 'overload', + 'meta', + }, kind) + then + -- Ignore + return + elseif kind == 'generic' then + cur_obj.generics = cur_obj.generics or {} + cur_obj.generics[parsed.name] = parsed.type or 'any' + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param fun nvim.luacats.parser.fun +--- @return nvim.luacats.parser.field +local function fun2field(fun) + local parts = { 'fun(' } + for _, p in ipairs(fun.params or {}) do + parts[#parts + 1] = string.format('%s: %s', p.name, p.type) + end + parts[#parts + 1] = ')' + if fun.returns then + parts[#parts + 1] = ': ' + local tys = {} --- @type string[] + for _, p in ipairs(fun.returns) do + tys[#tys + 1] = p.type + end + parts[#parts + 1] = table.concat(tys, ', ') + end + + return { + name = fun.name, + type = table.concat(parts, ''), + access = fun.access, + desc = fun.desc, + } +end + +--- Function to normalize known form for declaring functions and normalize into a more standard +--- form. +--- @param line string +--- @return string +local function filter_decl(line) + -- M.fun = vim._memoize(function(...) + -- -> + -- function M.fun(...) + line = line:gsub('^local (.+) = .*_memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)') + line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') + return line +end + +--- @param line string +--- @param state nvim.luacats.parser.State +--- @param classes table<string,nvim.luacats.parser.class> +--- @param classvars table<string,string> +--- @param has_indent boolean +local function process_lua_line(line, state, classes, classvars, has_indent) + line = filter_decl(line) + + if state.cur_obj and state.cur_obj.kind == 'class' then + local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=') + if nm then + classvars[nm] = state.cur_obj.name + end + return + end + + do + local parent_tbl, sep, fun_or_meth_nm = + line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(') + if parent_tbl then + -- Have a decl. Ensure cur_obj + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + -- Match `Class:foo` methods for defined classes + local class = classvars[parent_tbl] + if class then + --- @cast cur_obj nvim.luacats.parser.fun + cur_obj.name = fun_or_meth_nm + cur_obj.class = class + cur_obj.classvar = parent_tbl + -- Add self param to methods + if sep == ':' then + cur_obj.params = cur_obj.params or {} + table.insert(cur_obj.params, 1, { + name = 'self', + type = class, + }) + end + + -- Add method as the field to the class + table.insert(classes[class].fields, fun2field(cur_obj)) + return + end + + -- Match `M.foo` + if cur_obj and parent_tbl == cur_obj.modvar then + cur_obj.name = fun_or_meth_nm + return + end + end + end + + do + -- Handle: `function A.B.C.foo(...)` + local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(') + if fn_nm then + state.cur_obj = state.cur_obj or {} + state.cur_obj.name = fn_nm + return + end + end + + do + -- Handle: `M.foo = {...}` where `M` is the modvar + local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=') + if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + do + -- Handle: `foo = {...}` + local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=') + if tbl_nm and not has_indent then + state.cur_obj = state.cur_obj or {} + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + do + -- Handle: `vim.foo = {...}` + local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=') + if state.cur_obj and tbl_nm and not has_indent then + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + if state.cur_obj then + if line:find('^%s*%-%- luacheck:') then + state.cur_obj = nil + elseif line:find('^%s*local%s+') then + state.cur_obj = nil + elseif line:find('^%s*return%s+') then + state.cur_obj = nil + elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then + state.cur_obj = nil + end + end +end + +--- Determine the table name used to export functions of a module +--- Usually this is `M`. +--- @param filename string +--- @return string? +local function determine_modvar(filename) + local modvar --- @type string? + for line in io.lines(filename) do + do + --- @type string? + local m = line:match('^return%s+([a-zA-Z_]+)') + if m then + modvar = m + end + end + do + --- @type string? + local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),') + if m then + modvar = m + end + end + end + return modvar +end + +--- @param obj nvim.luacats.parser.obj +--- @param funs nvim.luacats.parser.fun[] +--- @param classes table<string,nvim.luacats.parser.class> +--- @param briefs string[] +--- @param uncommitted nvim.luacats.parser.obj[] +local function commit_obj(obj, classes, funs, briefs, uncommitted) + local commit = false + if obj.kind == 'class' then + --- @cast obj nvim.luacats.parser.class + if not classes[obj.name] then + classes[obj.name] = obj + commit = true + end + elseif obj.kind == 'alias' then + -- Just pretend + commit = true + elseif obj.kind == 'brief' then + --- @cast obj nvim.luacats.parser.brief` + briefs[#briefs + 1] = obj.desc + commit = true + else + --- @cast obj nvim.luacats.parser.fun` + if obj.name then + funs[#funs + 1] = obj + commit = true + end + end + if not commit then + table.insert(uncommitted, obj) + end + return commit +end + +--- @param filename string +--- @param uncommitted nvim.luacats.parser.obj[] +-- luacheck: no unused +local function dump_uncommitted(filename, uncommitted) + local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt' + if #uncommitted > 0 then + print(string.format('Could not commit %d objects in %s', #uncommitted, filename)) + vim.fn.mkdir(assert(vim.fs.dirname(out_path)), 'p') + local f = assert(io.open(out_path, 'w')) + for i, x in ipairs(uncommitted) do + f:write(i) + f:write(': ') + f:write(vim.inspect(x)) + f:write('\n') + end + f:close() + else + vim.fn.delete(out_path) + end +end + +local M = {} + +--- @param filename string +--- @return table<string,nvim.luacats.parser.class> classes +--- @return nvim.luacats.parser.fun[] funs +--- @return string[] briefs +--- @return nvim.luacats.parser.obj[] +function M.parse(filename) + local funs = {} --- @type nvim.luacats.parser.fun[] + local classes = {} --- @type table<string,nvim.luacats.parser.class> + local briefs = {} --- @type string[] + + local mod_return = determine_modvar(filename) + + --- @type string + local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename + module = module:gsub('/', '.') + + local classvars = {} --- @type table<string,string> + + local state = {} --- @type nvim.luacats.parser.State + + -- Keep track of any partial objects we don't commit + local uncommitted = {} --- @type nvim.luacats.parser.obj[] + + for line in io.lines(filename) do + local has_indent = line:match('^%s+') ~= nil + line = vim.trim(line) + if vim.startswith(line, '---') then + process_doc_line(line, state) + else + add_doc_lines_to_obj(state) + + if state.cur_obj then + state.cur_obj.modvar = mod_return + state.cur_obj.module = module + end + + process_lua_line(line, state, classes, classvars, has_indent) + + -- Commit the object + local cur_obj = state.cur_obj + if cur_obj then + if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then + --- @diagnostic disable-next-line:inject-field + cur_obj.line = line + end + end + + state = {} + end + end + + -- dump_uncommitted(filename, uncommitted) + + return classes, funs, briefs, uncommitted +end + +return M diff --git a/scripts/text_utils.lua b/scripts/text_utils.lua new file mode 100644 index 0000000000..5167ec42f2 --- /dev/null +++ b/scripts/text_utils.lua @@ -0,0 +1,239 @@ +local fmt = string.format + +--- @class nvim.text_utils.MDNode +--- @field [integer] nvim.text_utils.MDNode +--- @field type string +--- @field text? string + +local INDENTATION = 4 + +local M = {} + +local function contains(t, xs) + return vim.tbl_contains(xs, t) +end + +--- @param text string +--- @return nvim.text_utils.MDNode +local function parse_md(text) + local parser = vim.treesitter.languagetree.new(text, 'markdown', { + injections = { markdown = '' }, + }) + + local root = parser:parse(true)[1]:root() + + local EXCLUDE_TEXT_TYPE = { + list = true, + list_item = true, + section = true, + document = true, + fenced_code_block = true, + fenced_code_block_delimiter = true, + } + + --- @param node TSNode + --- @return nvim.text_utils.MDNode? + local function extract(node) + local ntype = node:type() + + if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then + return + end + + --- @type table<any,any> + local ret = { type = ntype } + + if not EXCLUDE_TEXT_TYPE[ntype] then + ret.text = vim.treesitter.get_node_text(node, text) + end + + for child, child_field in node:iter_children() do + local e = extract(child) + if child_field then + ret[child_field] = e + else + table.insert(ret, e) + end + end + + return ret + end + + return extract(root) or {} +end + +--- @param x string +--- @param start_indent integer +--- @param indent integer +--- @param text_width integer +--- @return string +function M.wrap(x, start_indent, indent, text_width) + local words = vim.split(vim.trim(x), '%s+') + local parts = { string.rep(' ', start_indent) } --- @type string[] + local count = indent + + for i, w in ipairs(words) do + if count > indent and count + #w > text_width - 1 then + parts[#parts + 1] = '\n' + parts[#parts + 1] = string.rep(' ', indent) + count = indent + elseif i ~= 1 then + parts[#parts + 1] = ' ' + count = count + 1 + end + count = count + #w + parts[#parts + 1] = w + end + + return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', '')) +end + +--- @param node nvim.text_utils.MDNode +--- @param start_indent integer +--- @param indent integer +--- @param text_width integer +--- @param level integer +--- @return string[] +local function render_md(node, start_indent, indent, text_width, level, is_list) + local parts = {} --- @type string[] + + -- For debugging + local add_tag = false + -- local add_tag = true + + if add_tag then + parts[#parts + 1] = '<' .. node.type .. '>' + end + + if node.type == 'paragraph' then + local text = assert(node.text) + text = text:gsub('(%s)%*(%w+)%*(%s)', '%1%2%3') + text = text:gsub('(%s)_(%w+)_(%s)', '%1%2%3') + text = text:gsub('\\|', '|') + text = text:gsub('\\%*', '*') + text = text:gsub('\\_', '_') + parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width) + parts[#parts + 1] = '\n' + elseif node.type == 'code_fence_content' then + local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n') + + local cindent = indent + INDENTATION + if level > 3 then + -- The tree-sitter markdown parser doesn't parse the code blocks indents + -- correctly in lists. Fudge it! + lines[1] = ' ' .. lines[1] -- ¯\_(ツ)_/¯ + cindent = indent - level + local _, initial_indent = lines[1]:find('^%s*') + initial_indent = initial_indent + cindent + if initial_indent < indent then + cindent = indent - INDENTATION + end + end + + for _, l in ipairs(lines) do + if #l > 0 then + parts[#parts + 1] = string.rep(' ', cindent) + parts[#parts + 1] = l + end + parts[#parts + 1] = '\n' + end + elseif node.type == 'fenced_code_block' then + parts[#parts + 1] = '>' + for _, child in ipairs(node) do + if child.type == 'info_string' then + parts[#parts + 1] = child.text + break + end + end + parts[#parts + 1] = '\n' + for _, child in ipairs(node) do + if child.type ~= 'info_string' then + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1)) + end + end + parts[#parts + 1] = '<\n' + elseif node.type == 'html_block' then + local text = node.text:gsub('^<pre>help', '') + text = text:gsub('</pre>%s*$', '') + parts[#parts + 1] = text + elseif node.type == 'list_marker_dot' then + parts[#parts + 1] = node.text + elseif contains(node.type, { 'list_marker_minus', 'list_marker_star' }) then + parts[#parts + 1] = '• ' + elseif node.type == 'list_item' then + parts[#parts + 1] = string.rep(' ', indent) + local offset = node[1].type == 'list_marker_dot' and 3 or 2 + for i, child in ipairs(node) do + local sindent = i <= 2 and 0 or (indent + offset) + vim.list_extend( + parts, + render_md(child, sindent, indent + offset, text_width, level + 1, true) + ) + end + else + if node.text then + error(fmt('cannot render:\n%s', vim.inspect(node))) + end + for i, child in ipairs(node) do + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1, is_list)) + if node.type ~= 'list' and i ~= #node then + if (node[i + 1] or {}).type ~= 'list' then + parts[#parts + 1] = '\n' + end + end + end + end + + if add_tag then + parts[#parts + 1] = '</' .. node.type .. '>' + end + + return parts +end + +--- @param text_width integer +local function align_tags(text_width) + --- @param line string + --- @return string + return function(line) + local tag_pat = '%s+(%*[^ ]+%*)%s*$' + local tags = {} + for m in line:gmatch(tag_pat) do + table.insert(tags, m) + end + + if #tags > 0 then + line = line:gsub(tag_pat, '') + local tags_str = ' ' .. table.concat(tags, ' ') + local pad = string.rep(' ', text_width - #line - #tags_str) + return line .. pad .. tags_str + end + + return line + end +end + +--- @param text string +--- @param start_indent integer +--- @param indent integer +--- @param is_list? boolean +--- @return string +function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list) + -- Add an extra newline so the parser can properly capture ending ``` + local parsed = parse_md(text .. '\n') + local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list) + + local lines = vim.split(table.concat(ret), '\n') + + lines = vim.tbl_map(align_tags(text_width), lines) + + local s = table.concat(lines, '\n') + + -- Reduce whitespace in code-blocks + s = s:gsub('\n+%s*>([a-z]+)\n?\n', ' >%1\n') + s = s:gsub('\n+%s*>\n?\n', ' >\n') + + return s +end + +return M |