aboutsummaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorLewis Russell <lewis6991@gmail.com>2024-02-15 17:16:04 +0000
committerLewis Russell <me@lewisr.dev>2024-02-27 14:41:17 +0000
commit9beb40a4db5613601fc1a4b828a44e5977eca046 (patch)
tree314096d28ccdf2a2b035091783baa35193887d6a /scripts
parent7ad2e3c64562bfb0ea2f7be305e4b0e6d2474d64 (diff)
downloadrneovim-9beb40a4db5613601fc1a4b828a44e5977eca046.tar.gz
rneovim-9beb40a4db5613601fc1a4b828a44e5977eca046.tar.bz2
rneovim-9beb40a4db5613601fc1a4b828a44e5977eca046.zip
feat(docs): replace lua2dox.lua
Problem: The documentation flow (`gen_vimdoc.py`) has several issues: - it's not very versatile - depends on doxygen - doesn't work well with Lua code as it requires an awkward filter script to convert it into pseudo-C. - The intermediate XML files and filters makes it too much like a rube goldberg machine. Solution: Re-implement the flow using Lua, LPEG and treesitter. - `gen_vimdoc.py` is now replaced with `gen_vimdoc.lua` and replicates a portion of the logic. - `lua2dox.lua` is gone! - No more XML files. - Doxygen is now longer used and instead we now use: - LPEG for comment parsing (see `scripts/luacats_grammar.lua` and `scripts/cdoc_grammar.lua`). - LPEG for C parsing (see `scripts/cdoc_parser.lua`) - Lua patterns for Lua parsing (see `scripts/luacats_parser.lua`). - Treesitter for Markdown parsing (see `scripts/text_utils.lua`). - The generated `runtime/doc/*.mpack` files have been removed. - `scripts/gen_eval_files.lua` now instead uses `scripts/cdoc_parser.lua` directly. - Text wrapping is implemented in `scripts/text_utils.lua` and appears to produce more consistent results (the main contributer to the diff of this change).
Diffstat (limited to 'scripts')
-rw-r--r--scripts/cdoc_grammar.lua87
-rw-r--r--scripts/cdoc_parser.lua223
-rwxr-xr-xscripts/gen_eval_files.lua100
-rwxr-xr-xscripts/gen_vimdoc.lua787
-rwxr-xr-xscripts/gen_vimdoc.py1766
-rw-r--r--scripts/lua2dox.lua544
-rw-r--r--scripts/luacats_grammar.lua218
-rw-r--r--scripts/luacats_parser.lua521
-rw-r--r--scripts/text_utils.lua239
9 files changed, 2135 insertions, 2350 deletions
diff --git a/scripts/cdoc_grammar.lua b/scripts/cdoc_grammar.lua
new file mode 100644
index 0000000000..6a7610883b
--- /dev/null
+++ b/scripts/cdoc_grammar.lua
@@ -0,0 +1,87 @@
+--[[!
+LPEG grammar for C doc comments
+]]
+
+--- @class nvim.cdoc.Param
+--- @field kind 'param'
+--- @field name string
+--- @field desc? string
+
+--- @class nvim.cdoc.Return
+--- @field kind 'return'
+--- @field desc string
+
+--- @class nvim.cdoc.Note
+--- @field desc? string
+
+--- @alias nvim.cdoc.grammar.result
+--- | nvim.cdoc.Param
+--- | nvim.cdoc.Return
+--- | nvim.cdoc.Note
+
+--- @class nvim.cdoc.grammar
+--- @field match fun(self, input: string): nvim.cdoc.grammar.result?
+
+local lpeg = vim.lpeg
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local Ct, Cg = lpeg.Ct, lpeg.Cg
+
+--- @param x vim.lpeg.Pattern
+local function rep(x)
+ return x ^ 0
+end
+
+--- @param x vim.lpeg.Pattern
+local function rep1(x)
+ return x ^ 1
+end
+
+--- @param x vim.lpeg.Pattern
+local function opt(x)
+ return x ^ -1
+end
+
+local nl = P('\r\n') + P('\n')
+local ws = rep1(S(' \t') + nl)
+
+local any = P(1) -- (consume one character)
+local letter = R('az', 'AZ') + S('_$')
+local ident = letter * rep(letter + R('09'))
+
+local io = P('[') * (P('in') + P('out') + P('inout')) * P(']')
+
+--- @param x string
+local function Pf(x)
+ return opt(ws) * P(x) * opt(ws)
+end
+
+--- @type table<string,vim.lpeg.Pattern>
+local v = setmetatable({}, {
+ __index = function(_, k)
+ return lpeg.V(k)
+ end,
+})
+
+local grammar = P {
+ rep1(P('@') * v.ats),
+
+ ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc,
+
+ at_param = Ct(
+ Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc'))
+ ),
+
+ at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))),
+
+ at_deprecated = Ct(Cg(P('deprecated'), 'kind')),
+
+ at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')),
+
+ at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')),
+
+ at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')),
+
+ at_nodoc = Ct(Cg(P('nodoc'), 'kind')),
+}
+
+return grammar --[[@as nvim.cdoc.grammar]]
diff --git a/scripts/cdoc_parser.lua b/scripts/cdoc_parser.lua
new file mode 100644
index 0000000000..5f0dc7be2c
--- /dev/null
+++ b/scripts/cdoc_parser.lua
@@ -0,0 +1,223 @@
+local cdoc_grammar = require('scripts.cdoc_grammar')
+local c_grammar = require('src.nvim.generators.c_grammar')
+
+--- @class nvim.cdoc.parser.param
+--- @field name string
+--- @field type string
+--- @field desc string
+
+--- @class nvim.cdoc.parser.return
+--- @field name string
+--- @field type string
+--- @field desc string
+
+--- @class nvim.cdoc.parser.note
+--- @field desc string
+
+--- @class nvim.cdoc.parser.brief
+--- @field kind 'brief'
+--- @field desc string
+
+--- @class nvim.cdoc.parser.fun
+--- @field name string
+--- @field params nvim.cdoc.parser.param[]
+--- @field returns nvim.cdoc.parser.return[]
+--- @field desc string
+--- @field deprecated? true
+--- @field since? string
+--- @field attrs? string[]
+--- @field nodoc? true
+--- @field notes? nvim.cdoc.parser.note[]
+--- @field see? nvim.cdoc.parser.note[]
+
+--- @class nvim.cdoc.parser.State
+--- @field doc_lines? string[]
+--- @field cur_obj? nvim.cdoc.parser.obj
+--- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note
+--- @field last_doc_item_indent? integer
+
+--- @alias nvim.cdoc.parser.obj
+--- | nvim.cdoc.parser.fun
+--- | nvim.cdoc.parser.brief
+
+--- If we collected any `---` lines. Add them to the existing (or new) object
+--- Used for function/class descriptions and multiline param descriptions.
+--- @param state nvim.cdoc.parser.State
+local function add_doc_lines_to_obj(state)
+ if state.doc_lines then
+ state.cur_obj = state.cur_obj or {}
+ local cur_obj = assert(state.cur_obj)
+ local txt = table.concat(state.doc_lines, '\n')
+ if cur_obj.desc then
+ cur_obj.desc = cur_obj.desc .. '\n' .. txt
+ else
+ cur_obj.desc = txt
+ end
+ state.doc_lines = nil
+ end
+end
+
+--- @param line string
+--- @param state nvim.cdoc.parser.State
+local function process_doc_line(line, state)
+ line = line:gsub('^%s+@', '@')
+
+ local parsed = cdoc_grammar:match(line)
+
+ if not parsed then
+ if line:match('^ ') then
+ line = line:sub(2)
+ end
+
+ if state.last_doc_item then
+ if not state.last_doc_item_indent then
+ state.last_doc_item_indent = #line:match('^%s*') + 1
+ end
+ state.last_doc_item.desc = (state.last_doc_item.desc or '')
+ .. '\n'
+ .. line:sub(state.last_doc_item_indent or 1)
+ else
+ state.doc_lines = state.doc_lines or {}
+ table.insert(state.doc_lines, line)
+ end
+ return
+ end
+
+ state.last_doc_item_indent = nil
+ state.last_doc_item = nil
+
+ local kind = parsed.kind
+
+ state.cur_obj = state.cur_obj or {}
+ local cur_obj = assert(state.cur_obj)
+
+ if kind == 'brief' then
+ state.cur_obj = {
+ kind = 'brief',
+ desc = parsed.desc,
+ }
+ elseif kind == 'param' then
+ state.last_doc_item_indent = nil
+ cur_obj.params = cur_obj.params or {}
+ state.last_doc_item = {
+ name = parsed.name,
+ desc = parsed.desc,
+ }
+ table.insert(cur_obj.params, state.last_doc_item)
+ elseif kind == 'return' then
+ cur_obj.returns = { {
+ desc = parsed.desc,
+ } }
+ state.last_doc_item_indent = nil
+ state.last_doc_item = cur_obj.returns[1]
+ elseif kind == 'deprecated' then
+ cur_obj.deprecated = true
+ elseif kind == 'nodoc' then
+ cur_obj.nodoc = true
+ elseif kind == 'since' then
+ cur_obj.since = parsed.desc
+ elseif kind == 'see' then
+ cur_obj.see = cur_obj.see or {}
+ table.insert(cur_obj.see, { desc = parsed.desc })
+ elseif kind == 'note' then
+ state.last_doc_item_indent = nil
+ state.last_doc_item = {
+ desc = parsed.desc,
+ }
+ cur_obj.notes = cur_obj.notes or {}
+ table.insert(cur_obj.notes, state.last_doc_item)
+ else
+ error('Unhandled' .. vim.inspect(parsed))
+ end
+end
+
+--- @param item table
+--- @param state nvim.cdoc.parser.State
+local function process_proto(item, state)
+ state.cur_obj = state.cur_obj or {}
+ local cur_obj = assert(state.cur_obj)
+ cur_obj.name = item.name
+ cur_obj.params = cur_obj.params or {}
+
+ for _, p in ipairs(item.parameters) do
+ local param = { name = p[2], type = p[1] }
+ local added = false
+ for _, cp in ipairs(cur_obj.params) do
+ if cp.name == param.name then
+ cp.type = param.type
+ added = true
+ break
+ end
+ end
+
+ if not added then
+ table.insert(cur_obj.params, param)
+ end
+ end
+
+ cur_obj.returns = cur_obj.returns or { {} }
+ cur_obj.returns[1].type = item.return_type
+
+ for _, a in ipairs({
+ 'fast',
+ 'remote_only',
+ 'lua_only',
+ 'textlock',
+ 'textlock_allow_cmdwin',
+ }) do
+ if item[a] then
+ cur_obj.attrs = cur_obj.attrs or {}
+ table.insert(cur_obj.attrs, a)
+ end
+ end
+
+ cur_obj.deprecated_since = item.deprecated_since
+
+ -- Remove some arguments
+ for i = #cur_obj.params, 1, -1 do
+ local p = cur_obj.params[i]
+ if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then
+ table.remove(cur_obj.params, i)
+ end
+ end
+end
+
+local M = {}
+
+--- @param filename string
+--- @return {} classes
+--- @return nvim.cdoc.parser.fun[] funs
+--- @return string[] briefs
+function M.parse(filename)
+ local funs = {} --- @type nvim.cdoc.parser.fun[]
+ local briefs = {} --- @type string[]
+ local state = {} --- @type nvim.cdoc.parser.State
+
+ local txt = assert(io.open(filename, 'r')):read('*all')
+
+ local parsed = c_grammar.grammar:match(txt)
+ for _, item in ipairs(parsed) do
+ if item.comment then
+ process_doc_line(item.comment, state)
+ else
+ add_doc_lines_to_obj(state)
+ if item[1] == 'proto' then
+ process_proto(item, state)
+ table.insert(funs, state.cur_obj)
+ end
+ local cur_obj = state.cur_obj
+ if cur_obj and not item.static then
+ if cur_obj.kind == 'brief' then
+ table.insert(briefs, cur_obj.desc)
+ end
+ end
+ state = {}
+ end
+ end
+
+ return {}, funs, briefs
+end
+
+-- M.parse('src/nvim/api/vim.c')
+
+return M
diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua
index b7f17a2d58..895033d5af 100755
--- a/scripts/gen_eval_files.lua
+++ b/scripts/gen_eval_files.lua
@@ -3,7 +3,6 @@
-- Generator for various vimdoc and Lua type files
local DEP_API_METADATA = 'build/api_metadata.mpack'
-local DEP_API_DOC = 'runtime/doc/api.mpack'
--- @class vim.api.metadata
--- @field name string
@@ -210,44 +209,65 @@ end
--- @return table<string, vim.EvalFn>
local function get_api_meta()
- local mpack_f = assert(io.open(DEP_API_METADATA, 'rb'))
- local metadata = vim.mpack.decode(mpack_f:read('*all')) --[[@as vim.api.metadata[] ]]
local ret = {} --- @type table<string, vim.EvalFn>
- local doc_mpack_f = assert(io.open(DEP_API_DOC, 'rb'))
- local doc_metadata = vim.mpack.decode(doc_mpack_f:read('*all')) --[[@as table<string,vim.gen_vim_doc_fun>]]
-
- for _, fun in ipairs(metadata) do
- if fun.lua then
- local fdoc = doc_metadata[fun.name]
-
- local params = {} --- @type {[1]:string,[2]:string}[]
- for _, p in ipairs(fun.parameters) do
- local ptype, pname = p[1], p[2]
- params[#params + 1] = {
- pname,
- api_type(ptype),
- fdoc and fdoc.parameters_doc[pname] or nil,
- }
- end
+ local cdoc_parser = require('scripts.cdoc_parser')
- local r = {
- signature = 'NA',
- name = fun.name,
- params = params,
- returns = api_type(fun.return_type),
- deprecated = fun.deprecated_since ~= nil,
- }
+ local f = 'src/nvim/api'
- if fdoc then
- if #fdoc.doc > 0 then
- r.desc = table.concat(fdoc.doc, '\n')
+ local function include(fun)
+ if not vim.startswith(fun.name, 'nvim_') then
+ return false
+ end
+ if vim.tbl_contains(fun.attrs or {}, 'lua_only') then
+ return true
+ end
+ if vim.tbl_contains(fun.attrs or {}, 'remote_only') then
+ return false
+ end
+ return true
+ end
+
+ --- @type table<string,nvim.cdoc.parser.fun>
+ local functions = {}
+ for path, ty in vim.fs.dir(f) do
+ if ty == 'file' then
+ local filename = vim.fs.joinpath(f, path)
+ local _, funs = cdoc_parser.parse(filename)
+ for _, fn in ipairs(funs) do
+ if include(fn) then
+ functions[fn.name] = fn
end
- r.return_desc = (fdoc['return'] or {})[1]
end
+ end
+ end
+
+ for _, fun in pairs(functions) do
+ local deprecated = fun.deprecated_since ~= nil
+
+ local params = {} --- @type {[1]:string,[2]:string}[]
+ for _, p in ipairs(fun.params) do
+ params[#params + 1] = {
+ p.name,
+ api_type(p.type),
+ not deprecated and p.desc or nil,
+ }
+ end
+
+ local r = {
+ signature = 'NA',
+ name = fun.name,
+ params = params,
+ returns = api_type(fun.returns[1].type),
+ deprecated = deprecated,
+ }
- ret[fun.name] = r
+ if not deprecated then
+ r.desc = fun.desc
+ r.return_desc = fun.returns[1].desc
end
+
+ ret[fun.name] = r
end
return ret
end
@@ -275,12 +295,10 @@ end
--- @param fun vim.EvalFn
--- @param write fun(line: string)
local function render_api_meta(_f, fun, write)
- if not vim.startswith(fun.name, 'nvim_') then
- return
- end
-
write('')
+ local text_utils = require('scripts.text_utils')
+
if vim.startswith(fun.name, 'nvim__') then
write('--- @private')
end
@@ -291,10 +309,10 @@ local function render_api_meta(_f, fun, write)
local desc = fun.desc
if desc then
+ desc = text_utils.md_to_vimdoc(desc, 0, 0, 74)
for _, l in ipairs(split(norm_text(desc))) do
write('--- ' .. l)
end
- write('---')
end
local param_names = {} --- @type string[]
@@ -303,8 +321,11 @@ local function render_api_meta(_f, fun, write)
param_names[#param_names + 1] = p[1]
local pdesc = p[3]
if pdesc then
- local pdesc_a = split(norm_text(pdesc))
- write('--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' .. pdesc_a[1])
+ local s = '--- @param ' .. p[1] .. ' ' .. p[2] .. ' '
+ local indent = #('@param ' .. p[1] .. ' ')
+ pdesc = text_utils.md_to_vimdoc(pdesc, #s, indent, 74, true)
+ local pdesc_a = split(vim.trim(norm_text(pdesc)))
+ write(s .. pdesc_a[1])
for i = 2, #pdesc_a do
if not pdesc_a[i] then
break
@@ -317,6 +338,7 @@ local function render_api_meta(_f, fun, write)
end
if fun.returns ~= '' then
local ret_desc = fun.returns_desc and ' : ' .. fun.returns_desc or ''
+ ret_desc = text_utils.md_to_vimdoc(ret_desc, 0, 0, 74)
local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns
write('--- @return ' .. ret .. ret_desc)
end
@@ -328,8 +350,6 @@ end
--- @return table<string, vim.EvalFn>
local function get_api_keysets_meta()
local mpack_f = assert(io.open(DEP_API_METADATA, 'rb'))
-
- --- @diagnostic disable-next-line:no-unknown
local metadata = assert(vim.mpack.decode(mpack_f:read('*all')))
local ret = {} --- @type table<string, vim.EvalFn>
diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua
new file mode 100755
index 0000000000..290cd83fbc
--- /dev/null
+++ b/scripts/gen_vimdoc.lua
@@ -0,0 +1,787 @@
+#!/usr/bin/env -S nvim -l
+--- Generates Nvim :help docs from Lua/C docstrings
+---
+--- The generated :help text for each function is formatted as follows:
+--- - Max width of 78 columns (`TEXT_WIDTH`).
+--- - Indent with spaces (not tabs).
+--- - Indent of 4 columns for body text (`INDENTATION`).
+--- - Function signature and helptag (right-aligned) on the same line.
+--- - Signature and helptag must have a minimum of 8 spaces between them.
+--- - If the signature is too long, it is placed on the line after the helptag.
+--- Signature wraps with subsequent lines indented to the open parenthesis.
+--- - Subsection bodies are indented an additional 4 spaces.
+--- - Body consists of function description, parameters, return description, and
+--- C declaration (`INCLUDE_C_DECL`).
+--- - Parameters are omitted for the `void` and `Error *` types, or if the
+--- parameter is marked as [out].
+--- - Each function documentation is separated by a single line.
+
+local luacats_parser = require('scripts.luacats_parser')
+local cdoc_parser = require('scripts.cdoc_parser')
+local text_utils = require('scripts.text_utils')
+
+local fmt = string.format
+
+local wrap = text_utils.wrap
+local md_to_vimdoc = text_utils.md_to_vimdoc
+
+local TEXT_WIDTH = 78
+local INDENTATION = 4
+
+--- @class (exact) nvim.gen_vimdoc.Config
+---
+--- Generated documentation target, e.g. api.txt
+--- @field filename string
+---
+--- @field section_order string[]
+---
+--- List of files/directories for doxygen to read, relative to `base_dir`.
+--- @field files string[]
+---
+--- @field exclude_types? true
+---
+--- Section name overrides. Key: filename (e.g., vim.c)
+--- @field section_name? table<string,string>
+---
+--- @field fn_name_pat? string
+---
+--- @field fn_xform? fun(fun: nvim.luacats.parser.fun)
+---
+--- For generated section names.
+--- @field section_fmt fun(name: string): string
+---
+--- @field helptag_fmt fun(name: string): string
+---
+--- Per-function helptag.
+--- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string
+---
+--- @field append_only? string[]
+
+local function contains(t, xs)
+ return vim.tbl_contains(xs, t)
+end
+
+--- @type {level:integer, prerelease:boolean}?
+local nvim_api_info_
+
+--- @return {level: integer, prerelease:boolean}
+local function nvim_api_info()
+ if not nvim_api_info_ then
+ --- @type integer?, boolean?
+ local level, prerelease
+ for l in io.lines('CMakeLists.txt') do
+ --- @cast l string
+ if level and prerelease then
+ break
+ end
+ local m1 = l:match('^set%(NVIM_API_LEVEL%s+(%d+)%)')
+ if m1 then
+ level = tonumber(m1) --[[@as integer]]
+ end
+ local m2 = l:match('^set%(NVIM_API_PRERELEASE%s+(%w+)%)')
+ if m2 then
+ prerelease = m2 == 'true'
+ end
+ end
+ nvim_api_info_ = { level = level, prerelease = prerelease }
+ end
+
+ return nvim_api_info_
+end
+
+--- @param fun nvim.luacats.parser.fun
+--- @return string
+local function fn_helptag_fmt_common(fun)
+ local fn_sfx = fun.table and '' or '()'
+ if fun.classvar then
+ return fmt('*%s:%s%s*', fun.classvar, fun.name, fn_sfx)
+ end
+ if fun.module then
+ return fmt('*%s.%s%s*', fun.module, fun.name, fn_sfx)
+ end
+ return fmt('*%s%s*', fun.name, fn_sfx)
+end
+
+--- @type table<string,nvim.gen_vimdoc.Config>
+local config = {
+ api = {
+ filename = 'api.txt',
+ section_order = {
+ 'vim.c',
+ 'vimscript.c',
+ 'command.c',
+ 'options.c',
+ 'buffer.c',
+ 'extmark.c',
+ 'window.c',
+ 'win_config.c',
+ 'tabpage.c',
+ 'autocmd.c',
+ 'ui.c',
+ },
+ exclude_types = true,
+ fn_name_pat = 'nvim_.*',
+ files = { 'src/nvim/api' },
+ section_name = {
+ ['vim.c'] = 'Global',
+ },
+ section_fmt = function(name)
+ return name .. ' Functions'
+ end,
+ helptag_fmt = function(name)
+ return fmt('*api-%s*', name:lower())
+ end,
+ },
+ lua = {
+ filename = 'lua.txt',
+ section_order = {
+ 'highlight.lua',
+ 'diff.lua',
+ 'mpack.lua',
+ 'json.lua',
+ 'base64.lua',
+ 'spell.lua',
+ 'builtin.lua',
+ '_options.lua',
+ '_editor.lua',
+ '_inspector.lua',
+ 'shared.lua',
+ 'loader.lua',
+ 'uri.lua',
+ 'ui.lua',
+ 'filetype.lua',
+ 'keymap.lua',
+ 'fs.lua',
+ 'glob.lua',
+ 'lpeg.lua',
+ 're.lua',
+ 'regex.lua',
+ 'secure.lua',
+ 'version.lua',
+ 'iter.lua',
+ 'snippet.lua',
+ 'text.lua',
+ },
+ files = {
+ 'runtime/lua/vim/iter.lua',
+ 'runtime/lua/vim/_editor.lua',
+ 'runtime/lua/vim/_options.lua',
+ 'runtime/lua/vim/shared.lua',
+ 'runtime/lua/vim/loader.lua',
+ 'runtime/lua/vim/uri.lua',
+ 'runtime/lua/vim/ui.lua',
+ 'runtime/lua/vim/filetype.lua',
+ 'runtime/lua/vim/keymap.lua',
+ 'runtime/lua/vim/fs.lua',
+ 'runtime/lua/vim/highlight.lua',
+ 'runtime/lua/vim/secure.lua',
+ 'runtime/lua/vim/version.lua',
+ 'runtime/lua/vim/_inspector.lua',
+ 'runtime/lua/vim/snippet.lua',
+ 'runtime/lua/vim/text.lua',
+ 'runtime/lua/vim/glob.lua',
+ 'runtime/lua/vim/_meta/builtin.lua',
+ 'runtime/lua/vim/_meta/diff.lua',
+ 'runtime/lua/vim/_meta/mpack.lua',
+ 'runtime/lua/vim/_meta/json.lua',
+ 'runtime/lua/vim/_meta/base64.lua',
+ 'runtime/lua/vim/_meta/regex.lua',
+ 'runtime/lua/vim/_meta/lpeg.lua',
+ 'runtime/lua/vim/_meta/re.lua',
+ 'runtime/lua/vim/_meta/spell.lua',
+ },
+ fn_xform = function(fun)
+ if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then
+ fun.module = 'vim'
+ end
+
+ if fun.module == 'vim' and contains(fun.name, { 'cmd', 'inspect' }) then
+ fun.table = nil
+ end
+
+ if fun.classvar or vim.startswith(fun.name, 'vim.') or fun.module == 'vim.iter' then
+ return
+ end
+
+ fun.name = fmt('%s.%s', fun.module, fun.name)
+ end,
+ section_name = {
+ ['_inspector.lua'] = 'inspector',
+ },
+ section_fmt = function(name)
+ name = name:lower()
+ if name == '_editor' then
+ return 'Lua module: vim'
+ elseif name == '_options' then
+ return 'LUA-VIMSCRIPT BRIDGE'
+ elseif name == 'builtin' then
+ return 'VIM'
+ end
+ if
+ contains(name, {
+ 'highlight',
+ 'mpack',
+ 'json',
+ 'base64',
+ 'diff',
+ 'spell',
+ 'regex',
+ 'lpeg',
+ 're',
+ })
+ then
+ return 'VIM.' .. name:upper()
+ end
+ return 'Lua module: vim.' .. name
+ end,
+ helptag_fmt = function(name)
+ if name == '_editor' then
+ return '*lua-vim*'
+ elseif name == '_options' then
+ return '*lua-vimscript*'
+ end
+ return '*vim.' .. name:lower() .. '*'
+ end,
+ fn_helptag_fmt = function(fun)
+ local name = fun.name
+
+ if vim.startswith(name, 'vim.') then
+ local fn_sfx = fun.table and '' or '()'
+ return fmt('*%s%s*', name, fn_sfx)
+ elseif fun.classvar == 'Option' then
+ return fmt('*vim.opt:%s()*', name)
+ end
+
+ return fn_helptag_fmt_common(fun)
+ end,
+ append_only = {
+ 'shared.lua',
+ },
+ },
+ lsp = {
+ filename = 'lsp.txt',
+ section_order = {
+ 'lsp.lua',
+ 'buf.lua',
+ 'diagnostic.lua',
+ 'codelens.lua',
+ 'inlay_hint.lua',
+ 'tagfunc.lua',
+ 'semantic_tokens.lua',
+ 'handlers.lua',
+ 'util.lua',
+ 'log.lua',
+ 'rpc.lua',
+ 'protocol.lua',
+ },
+ files = {
+ 'runtime/lua/vim/lsp',
+ 'runtime/lua/vim/lsp.lua',
+ },
+ fn_xform = function(fun)
+ fun.name = fun.name:gsub('result%.', '')
+ end,
+ section_fmt = function(name)
+ if name:lower() == 'lsp' then
+ return 'Lua module: vim.lsp'
+ end
+ return 'Lua module: vim.lsp.' .. name:lower()
+ end,
+ helptag_fmt = function(name)
+ if name:lower() == 'lsp' then
+ return '*lsp-core*'
+ end
+ return fmt('*lsp-%s*', name:lower())
+ end,
+ },
+ diagnostic = {
+ filename = 'diagnostic.txt',
+ section_order = {
+ 'diagnostic.lua',
+ },
+ files = { 'runtime/lua/vim/diagnostic.lua' },
+ section_fmt = function()
+ return 'Lua module: vim.diagnostic'
+ end,
+ helptag_fmt = function()
+ return '*diagnostic-api*'
+ end,
+ },
+ treesitter = {
+ filename = 'treesitter.txt',
+ section_order = {
+ 'treesitter.lua',
+ 'language.lua',
+ 'query.lua',
+ 'highlighter.lua',
+ 'languagetree.lua',
+ 'dev.lua',
+ },
+ files = {
+ 'runtime/lua/vim/treesitter.lua',
+ 'runtime/lua/vim/treesitter/',
+ },
+ section_fmt = function(name)
+ if name:lower() == 'treesitter' then
+ return 'Lua module: vim.treesitter'
+ end
+ return 'Lua module: vim.treesitter.' .. name:lower()
+ end,
+ helptag_fmt = function(name)
+ if name:lower() == 'treesitter' then
+ return '*lua-treesitter-core*'
+ end
+ return '*lua-treesitter-' .. name:lower() .. '*'
+ end,
+ },
+}
+
+--- @param ty string
+--- @param generics table<string,string>
+--- @return string
+local function replace_generics(ty, generics)
+ if ty:sub(-2) == '[]' then
+ local ty0 = ty:sub(1, -3)
+ if generics[ty0] then
+ return generics[ty0] .. '[]'
+ end
+ elseif ty:sub(-1) == '?' then
+ local ty0 = ty:sub(1, -2)
+ if generics[ty0] then
+ return generics[ty0] .. '?'
+ end
+ end
+
+ return generics[ty] or ty
+end
+
+--- @param ty string
+--- @param generics? table<string,string>
+local function render_type(ty, generics)
+ if generics then
+ ty = replace_generics(ty, generics)
+ end
+ ty = ty:gsub('%s*|%s*nil', '?')
+ ty = ty:gsub('nil%s*|%s*(.*)', '%1?')
+ ty = ty:gsub('%s*|%s*', '|')
+ return fmt('(`%s`)', ty)
+end
+
+--- @param p nvim.luacats.parser.param|nvim.luacats.parser.field
+local function should_render_param(p)
+ return not p.access and not contains(p.name, { '_', 'self' })
+end
+
+--- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[]
+--- @param generics? table<string,string>
+--- @param exclude_types? true
+local function render_fields_or_params(xs, generics, exclude_types)
+ local ret = {} --- @type string[]
+
+ xs = vim.tbl_filter(should_render_param, xs)
+
+ local indent = 0
+ for _, p in ipairs(xs) do
+ if p.type or p.desc then
+ indent = math.max(indent, #p.name + 3)
+ end
+ if exclude_types then
+ p.type = nil
+ end
+ end
+
+ for _, p in ipairs(xs) do
+ local nm, ty = p.name, p.type
+ local desc = p.desc
+ local pnm = fmt(' • %-' .. indent .. 's', '{' .. nm .. '}')
+ if ty then
+ local pty = render_type(ty, generics)
+ if desc then
+ desc = fmt('%s %s', pty, desc)
+ table.insert(ret, pnm)
+ table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true))
+ else
+ table.insert(ret, fmt('%s %s\n', pnm, pty))
+ end
+ else
+ if desc then
+ table.insert(ret, pnm)
+ table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true))
+ end
+ end
+ end
+
+ return table.concat(ret)
+end
+
+-- --- @param class lua2vimdoc.class
+-- local function render_class(class)
+-- writeln(fmt('*%s*', class.name))
+-- writeln()
+-- if #class.fields > 0 then
+-- writeln(' Fields: ~')
+-- render_fields_or_params(class.fields)
+-- end
+-- writeln()
+-- end
+
+-- --- @param cls table<string,lua2vimdoc.class>
+-- local function render_classes(cls)
+-- --- @diagnostic disable-next-line:no-unknown
+-- for _, class in vim.spairs(cls) do
+-- render_class(class)
+-- end
+-- end
+
+--- @param fun nvim.luacats.parser.fun
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_fun_header(fun, cfg)
+ local ret = {} --- @type string[]
+
+ local args = {} --- @type string[]
+ for _, p in ipairs(fun.params or {}) do
+ if p.name ~= 'self' then
+ args[#args + 1] = fmt('{%s}', p.name:gsub('%?$', ''))
+ end
+ end
+
+ local nm = fun.name
+ if fun.classvar then
+ nm = fmt('%s:%s', fun.classvar, nm)
+ end
+
+ local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')'
+
+ if not cfg.fn_helptag_fmt then
+ cfg.fn_helptag_fmt = fn_helptag_fmt_common
+ end
+
+ local tag = cfg.fn_helptag_fmt(fun)
+
+ if #proto + #tag > TEXT_WIDTH - 8 then
+ table.insert(ret, fmt('%78s\n', tag))
+ local name, pargs = proto:match('([^(]+%()(.*)')
+ table.insert(ret, name)
+ table.insert(ret, wrap(pargs, 0, #name, TEXT_WIDTH))
+ else
+ local pad = TEXT_WIDTH - #proto - #tag
+ table.insert(ret, proto .. string.rep(' ', pad) .. tag)
+ end
+
+ return table.concat(ret)
+end
+
+--- @param returns nvim.luacats.parser.return[]
+--- @param generics? table<string,string>
+--- @param exclude_types boolean
+local function render_returns(returns, generics, exclude_types)
+ local ret = {} --- @type string[]
+
+ returns = vim.deepcopy(returns)
+ if exclude_types then
+ for _, r in ipairs(returns) do
+ r.type = nil
+ end
+ end
+
+ if #returns > 1 then
+ table.insert(ret, ' Return (multiple): ~\n')
+ elseif #returns == 1 and next(returns[1]) then
+ table.insert(ret, ' Return: ~\n')
+ end
+
+ for _, p in ipairs(returns) do
+ local rnm, ty, desc = p.name, p.type, p.desc
+ local blk = ''
+ if ty then
+ blk = render_type(ty, generics)
+ end
+ if rnm then
+ blk = blk .. ' ' .. rnm
+ end
+ if desc then
+ blk = blk .. ' ' .. desc
+ end
+ table.insert(ret, md_to_vimdoc(blk, 8, 8, TEXT_WIDTH, true))
+ end
+
+ return table.concat(ret)
+end
+
+--- @param fun nvim.luacats.parser.fun
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_fun(fun, cfg)
+ if fun.access or fun.deprecated or fun.nodoc then
+ return
+ end
+
+ if cfg.fn_name_pat and not fun.name:match(cfg.fn_name_pat) then
+ return
+ end
+
+ if vim.startswith(fun.name, '_') or fun.name:find('[:.]_') then
+ return
+ end
+
+ local ret = {} --- @type string[]
+
+ table.insert(ret, render_fun_header(fun, cfg))
+ table.insert(ret, '\n')
+
+ if fun.desc then
+ table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH))
+ end
+
+ if fun.since then
+ local since = tonumber(fun.since)
+ local info = nvim_api_info()
+ if since and (since > info.level or since == info.level and info.prerelease) then
+ fun.notes = fun.notes or {}
+ table.insert(fun.notes, { desc = 'This API is pre-release (unstable).' })
+ end
+ end
+
+ if fun.notes then
+ table.insert(ret, '\n Note: ~\n')
+ for _, p in ipairs(fun.notes) do
+ table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true))
+ end
+ end
+
+ if fun.attrs then
+ table.insert(ret, '\n Attributes: ~\n')
+ for _, attr in ipairs(fun.attrs) do
+ local attr_str = ({
+ textlock = 'not allowed when |textlock| is active or in the |cmdwin|',
+ textlock_allow_cmdwin = 'not allowed when |textlock| is active',
+ fast = '|api-fast|',
+ remote_only = '|RPC| only',
+ lua_only = 'Lua |vim.api| only',
+ })[attr] or attr
+ table.insert(ret, fmt(' %s\n', attr_str))
+ end
+ end
+
+ if fun.params and #fun.params > 0 then
+ local param_txt = render_fields_or_params(fun.params, fun.generics, cfg.exclude_types)
+ if not param_txt:match('^%s*$') then
+ table.insert(ret, '\n Parameters: ~\n')
+ ret[#ret + 1] = param_txt
+ end
+ end
+
+ if fun.returns then
+ local txt = render_returns(fun.returns, fun.generics, cfg.exclude_types)
+ if not txt:match('^%s*$') then
+ table.insert(ret, '\n')
+ ret[#ret + 1] = txt
+ end
+ end
+
+ if fun.see then
+ table.insert(ret, '\n See also: ~\n')
+ for _, p in ipairs(fun.see) do
+ table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true))
+ end
+ end
+
+ table.insert(ret, '\n')
+ return table.concat(ret)
+end
+
+--- @param funs nvim.luacats.parser.fun[]
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_funs(funs, cfg)
+ local ret = {} --- @type string[]
+
+ for _, f in ipairs(funs) do
+ if cfg.fn_xform then
+ cfg.fn_xform(f)
+ end
+ ret[#ret + 1] = render_fun(f, cfg)
+ end
+
+ -- Sort via prototype
+ table.sort(ret, function(a, b)
+ local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n')
+ local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n')
+ return a1:lower() < b1:lower()
+ end)
+
+ return table.concat(ret)
+end
+
+--- @return string
+local function get_script_path()
+ local str = debug.getinfo(2, 'S').source:sub(2)
+ return str:match('(.*[/\\])') or './'
+end
+
+local script_path = get_script_path()
+local base_dir = vim.fs.dirname(assert(vim.fs.dirname(script_path)))
+
+local function delete_lines_below(doc_file, tokenstr)
+ local lines = {} --- @type string[]
+ local found = false
+ for line in io.lines(doc_file) do
+ if line:find(vim.pesc(tokenstr)) then
+ found = true
+ break
+ end
+ lines[#lines + 1] = line
+ end
+ if not found then
+ error(fmt('not found: %s in %s', tokenstr, doc_file))
+ end
+ lines[#lines] = nil
+ local fp = assert(io.open(doc_file, 'w'))
+ fp:write(table.concat(lines, '\n'))
+ fp:write('\n')
+ fp:close()
+end
+
+--- @param x string
+local function mktitle(x)
+ if x == 'ui' then
+ return 'UI'
+ end
+ return x:sub(1, 1):upper() .. x:sub(2)
+end
+
+--- @class nvim.gen_vimdoc.Section
+--- @field name string
+--- @field title string
+--- @field help_tag string
+--- @field funs_txt string
+--- @field doc? string[]
+
+--- @param filename string
+--- @param cfg nvim.gen_vimdoc.Config
+--- @param section_docs table<string,nvim.gen_vimdoc.Section>
+--- @param funs_txt string
+--- @return nvim.gen_vimdoc.Section?
+local function make_section(filename, cfg, section_docs, funs_txt)
+ -- filename: e.g., 'autocmd.c'
+ -- name: e.g. 'autocmd'
+ local name = filename:match('(.*)%.[a-z]+')
+
+ -- Formatted (this is what's going to be written in the vimdoc)
+ -- e.g., "Autocmd Functions"
+ local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name)
+
+ -- section tag: e.g., "*api-autocmd*"
+ local help_tag = cfg.helptag_fmt(sectname)
+
+ if funs_txt == '' and #section_docs == 0 then
+ return
+ end
+
+ return {
+ name = sectname,
+ title = cfg.section_fmt(sectname),
+ help_tag = help_tag,
+ funs_txt = funs_txt,
+ doc = section_docs,
+ }
+end
+
+--- @param section nvim.gen_vimdoc.Section
+--- @param add_header? boolean
+local function render_section(section, add_header)
+ local doc = {} --- @type string[]
+
+ if add_header ~= false then
+ vim.list_extend(doc, {
+ string.rep('=', TEXT_WIDTH),
+ '\n',
+ section.title,
+ fmt('%' .. (TEXT_WIDTH - section.title:len()) .. 's', section.help_tag),
+ })
+ end
+
+ if section.doc and #section.doc > 0 then
+ table.insert(doc, '\n\n')
+ vim.list_extend(doc, section.doc)
+ end
+
+ if section.funs_txt then
+ table.insert(doc, '\n\n')
+ table.insert(doc, section.funs_txt)
+ end
+
+ return table.concat(doc)
+end
+
+local parsers = {
+ lua = luacats_parser.parse,
+ c = cdoc_parser.parse,
+ h = cdoc_parser.parse,
+}
+
+--- @param files string[]
+local function expand_files(files)
+ for k, f in pairs(files) do
+ if vim.fn.isdirectory(f) == 1 then
+ table.remove(files, k)
+ for path, ty in vim.fs.dir(f) do
+ if ty == 'file' then
+ table.insert(files, vim.fs.joinpath(f, path))
+ end
+ end
+ end
+ end
+end
+
+--- @param cfg nvim.gen_vimdoc.Config
+local function gen_target(cfg)
+ local sections = {} --- @type table<string,nvim.gen_vimdoc.Section>
+
+ expand_files(cfg.files)
+
+ for _, f in pairs(cfg.files) do
+ local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']]
+ local parser = assert(parsers[ext])
+ local _, funs, briefs = parser(f)
+ local briefs_txt = {} --- @type string[]
+ for _, b in ipairs(briefs) do
+ briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH)
+ end
+ local funs_txt = render_funs(funs, cfg)
+ -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua`
+ local f_base = assert(vim.fs.basename(f))
+ sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt)
+ end
+
+ local first_section_tag = sections[cfg.section_order[1]].help_tag
+ local docs = {} --- @type string[]
+ for _, f in ipairs(cfg.section_order) do
+ local section = sections[f]
+ if section then
+ local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f)
+ table.insert(docs, render_section(section, add_sep_and_header))
+ end
+ end
+
+ table.insert(
+ docs,
+ fmt(' vim:tw=78:ts=8:sw=%d:sts=%d:et:ft=help:norl:\n', INDENTATION, INDENTATION)
+ )
+
+ local doc_file = vim.fs.joinpath(base_dir, 'runtime', 'doc', cfg.filename)
+
+ if vim.uv.fs_stat(doc_file) then
+ delete_lines_below(doc_file, first_section_tag)
+ end
+
+ local fp = assert(io.open(doc_file, 'a'))
+ fp:write(table.concat(docs, '\n'))
+ fp:close()
+end
+
+local function run()
+ for _, cfg in pairs(config) do
+ gen_target(cfg)
+ end
+end
+
+run()
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
deleted file mode 100755
index c1a2183f24..0000000000
--- a/scripts/gen_vimdoc.py
+++ /dev/null
@@ -1,1766 +0,0 @@
-#!/usr/bin/env python3
-
-r"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
-
-Also generates *.mpack files. To inspect the *.mpack structure:
- :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B')))
-
-Flow:
- main
- extract_from_xml
- fmt_node_as_vimhelp \
- para_as_map } recursive
- update_params_map /
- render_node
-
-TODO: eliminate this script and use Lua+treesitter (requires parsers for C and
-Lua markdown-style docstrings).
-
-The generated :help text for each function is formatted as follows:
-
- - Max width of 78 columns (`text_width`).
- - Indent with spaces (not tabs).
- - Indent of 4 columns for body text (`indentation`).
- - Function signature and helptag (right-aligned) on the same line.
- - Signature and helptag must have a minimum of 8 spaces between them.
- - If the signature is too long, it is placed on the line after the helptag.
- Signature wraps at `text_width - 8` characters with subsequent
- lines indented to the open parenthesis.
- - Subsection bodies are indented an additional 4 spaces.
- - Body consists of function description, parameters, return description, and
- C declaration (`INCLUDE_C_DECL`).
- - Parameters are omitted for the `void` and `Error *` types, or if the
- parameter is marked as [out].
- - Each function documentation is separated by a single line.
-"""
-
-from __future__ import annotations # PEP-563, python 3.7+
-
-import argparse
-import collections
-import dataclasses
-import logging
-import os
-import re
-import shutil
-import subprocess
-import sys
-import textwrap
-from pathlib import Path
-from typing import Any, Callable, Dict, List, Tuple
-from xml.dom import minidom
-
-if sys.version_info >= (3, 8):
- from typing import Literal
-
-import msgpack
-
-Element = minidom.Element
-Document = minidom.Document
-
-MIN_PYTHON_VERSION = (3, 7)
-MIN_DOXYGEN_VERSION = (1, 9, 0)
-
-if sys.version_info < MIN_PYTHON_VERSION:
- print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION))
- sys.exit(1)
-
-doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"],
- universal_newlines=True).split()[0].split('.')))
-
-if doxygen_version < MIN_DOXYGEN_VERSION:
- print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION))
- print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version))
- sys.exit(1)
-
-
-# Need a `nvim` that supports `-l`, try the local build
-nvim_path = Path(__file__).parent / "../build/bin/nvim"
-if nvim_path.exists():
- nvim = nvim_path.resolve()
-else:
- # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works.
- nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True)
- nvim_version = [line for line in nvim_out.split('\n')
- if '-l ' in line]
- if len(nvim_version) == 0:
- print((
- "\nYou need to have a local Neovim build or a `nvim` version 0.9 for `-l` "
- "support to build the documentation."))
- sys.exit(1)
- nvim = 'nvim'
-
-
-# DEBUG = ('DEBUG' in os.environ)
-INCLUDE_C_DECL = os.environ.get('INCLUDE_C_DECL', '0') != '0'
-INCLUDE_DEPRECATED = os.environ.get('INCLUDE_DEPRECATED', '0') != '0'
-
-log = logging.getLogger(__name__)
-
-LOG_LEVELS = {
- logging.getLevelName(level): level for level in [
- logging.DEBUG, logging.INFO, logging.ERROR
- ]
-}
-
-text_width = 78
-indentation = 4
-SECTION_SEP = '=' * text_width
-
-script_path = os.path.abspath(__file__)
-base_dir = os.path.dirname(os.path.dirname(script_path))
-out_dir = os.path.join(base_dir, 'tmp-{target}-doc')
-filter_cmd = '%s %s' % (sys.executable, script_path)
-msgs = [] # Messages to show on exit.
-lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua')
-
-
-SectionName = str
-
-Docstring = str # Represents (formatted) vimdoc string
-
-FunctionName = str
-
-
-@dataclasses.dataclass
-class Config:
- """Config for documentation."""
-
- mode: Literal['c', 'lua']
-
- filename: str
- """Generated documentation target, e.g. api.txt"""
-
- section_order: List[str]
- """Section ordering."""
-
- files: List[str]
- """List of files/directories for doxygen to read, relative to `base_dir`."""
-
- file_patterns: str
- """file patterns used by doxygen."""
-
- section_name: Dict[str, SectionName]
- """Section name overrides. Key: filename (e.g., vim.c)"""
-
- section_fmt: Callable[[SectionName], str]
- """For generated section names."""
-
- helptag_fmt: Callable[[SectionName], str]
- """Section helptag."""
-
- fn_helptag_fmt: Callable[[str, str, bool], str]
- """Per-function helptag."""
-
- module_override: Dict[str, str]
- """Module name overrides (for Lua)."""
-
- append_only: List[str]
- """Append the docs for these modules, do not start a new section."""
-
- fn_name_prefix: str
- """Only function with this prefix are considered"""
-
- fn_name_fmt: Callable[[str, str], str] | None = None
-
- include_tables: bool = True
-
-
-CONFIG: Dict[str, Config] = {
- 'api': Config(
- mode='c',
- filename = 'api.txt',
- # Section ordering.
- section_order=[x for x in [
- 'vim.c',
- 'vimscript.c',
- 'command.c',
- 'options.c',
- 'buffer.c',
- 'extmark.c',
- 'window.c',
- 'win_config.c',
- 'tabpage.c',
- 'autocmd.c',
- 'ui.c',
- 'deprecated.c' if INCLUDE_DEPRECATED else ''
- ] if x],
- files=['src/nvim/api'],
- file_patterns = '*.h *.c',
- fn_name_prefix = 'nvim_',
- section_name={
- 'vim.c': 'Global',
- },
- section_fmt=lambda name: f'{name} Functions',
- helptag_fmt=lambda name: f'*api-{name.lower()}*',
- fn_helptag_fmt=lambda fstem, name, istbl: f'*{name}()*',
- module_override={},
- append_only=[],
- ),
- 'lua': Config(
- mode='lua',
- filename='lua.txt',
- section_order=[
- 'highlight.lua',
- 'diff.lua',
- 'mpack.lua',
- 'json.lua',
- 'base64.lua',
- 'spell.lua',
- 'builtin.lua',
- '_options.lua',
- '_editor.lua',
- '_inspector.lua',
- 'shared.lua',
- 'loader.lua',
- 'uri.lua',
- 'ui.lua',
- 'filetype.lua',
- 'keymap.lua',
- 'fs.lua',
- 'glob.lua',
- 'lpeg.lua',
- 're.lua',
- 'regex.lua',
- 'secure.lua',
- 'version.lua',
- 'iter.lua',
- 'snippet.lua',
- 'text.lua',
- ],
- files=[
- 'runtime/lua/vim/iter.lua',
- 'runtime/lua/vim/_editor.lua',
- 'runtime/lua/vim/_options.lua',
- 'runtime/lua/vim/shared.lua',
- 'runtime/lua/vim/loader.lua',
- 'runtime/lua/vim/uri.lua',
- 'runtime/lua/vim/ui.lua',
- 'runtime/lua/vim/filetype.lua',
- 'runtime/lua/vim/keymap.lua',
- 'runtime/lua/vim/fs.lua',
- 'runtime/lua/vim/highlight.lua',
- 'runtime/lua/vim/secure.lua',
- 'runtime/lua/vim/version.lua',
- 'runtime/lua/vim/_inspector.lua',
- 'runtime/lua/vim/snippet.lua',
- 'runtime/lua/vim/text.lua',
- 'runtime/lua/vim/glob.lua',
- 'runtime/lua/vim/_meta/builtin.lua',
- 'runtime/lua/vim/_meta/diff.lua',
- 'runtime/lua/vim/_meta/mpack.lua',
- 'runtime/lua/vim/_meta/json.lua',
- 'runtime/lua/vim/_meta/base64.lua',
- 'runtime/lua/vim/_meta/regex.lua',
- 'runtime/lua/vim/_meta/lpeg.lua',
- 'runtime/lua/vim/_meta/re.lua',
- 'runtime/lua/vim/_meta/spell.lua',
- ],
- file_patterns='*.lua',
- fn_name_prefix='',
- fn_name_fmt=lambda fstem, name: (
- name if fstem in [ 'vim.iter' ] else
- f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else
- f'vim.{name}' if fstem == '_options' and not name[0].isupper() else
- f'{fstem}.{name}' if fstem.startswith('vim') else
- name
- ),
- section_name={
- 'lsp.lua': 'core',
- '_inspector.lua': 'inspector',
- },
- section_fmt=lambda name: (
- 'Lua module: vim' if name.lower() == '_editor' else
- 'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else
- f'VIM.{name.upper()}' if name.lower() in [
- 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell',
- 'regex', 'lpeg', 're',
- ] else
- 'VIM' if name.lower() == 'builtin' else
- f'Lua module: vim.{name.lower()}'),
- helptag_fmt=lambda name: (
- '*lua-vim*' if name.lower() == '_editor' else
- '*lua-vimscript*' if name.lower() == '_options' else
- f'*vim.{name.lower()}*'),
- fn_helptag_fmt=lambda fstem, name, istbl: (
- f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else
- # Exclude fstem for methods
- f'*{name}()*' if ':' in name else
- f'*vim.{name}()*' if fstem.lower() == '_editor' else
- f'*vim.{name}*' if fstem.lower() == '_options' and istbl else
- # Prevents vim.regex.regex
- f'*{fstem}()*' if fstem.endswith('.' + name) else
- f'*{fstem}.{name}{"" if istbl else "()"}*'
- ),
- module_override={
- # `shared` functions are exposed on the `vim` module.
- 'shared': 'vim',
- '_inspector': 'vim',
- 'uri': 'vim',
- 'ui': 'vim.ui',
- 'loader': 'vim.loader',
- 'filetype': 'vim.filetype',
- 'keymap': 'vim.keymap',
- 'fs': 'vim.fs',
- 'highlight': 'vim.highlight',
- 'secure': 'vim.secure',
- 'version': 'vim.version',
- 'iter': 'vim.iter',
- 'diff': 'vim',
- 'builtin': 'vim',
- 'mpack': 'vim.mpack',
- 'json': 'vim.json',
- 'base64': 'vim.base64',
- 'regex': 'vim.regex',
- 'lpeg': 'vim.lpeg',
- 're': 'vim.re',
- 'spell': 'vim.spell',
- 'snippet': 'vim.snippet',
- 'text': 'vim.text',
- 'glob': 'vim.glob',
- },
- append_only=[
- 'shared.lua',
- ],
- ),
- 'lsp': Config(
- mode='lua',
- filename='lsp.txt',
- section_order=[
- 'lsp.lua',
- 'buf.lua',
- 'diagnostic.lua',
- 'codelens.lua',
- 'inlay_hint.lua',
- 'tagfunc.lua',
- 'semantic_tokens.lua',
- 'handlers.lua',
- 'util.lua',
- 'log.lua',
- 'rpc.lua',
- 'protocol.lua',
- ],
- files=[
- 'runtime/lua/vim/lsp',
- 'runtime/lua/vim/lsp.lua',
- ],
- file_patterns='*.lua',
- fn_name_prefix='',
- section_name={'lsp.lua': 'lsp'},
- section_fmt=lambda name: (
- 'Lua module: vim.lsp'
- if name.lower() == 'lsp'
- else f'Lua module: vim.lsp.{name.lower()}'),
- helptag_fmt=lambda name: (
- '*lsp-core*'
- if name.lower() == 'lsp'
- else f'*lsp-{name.lower()}*'),
- fn_helptag_fmt=lambda fstem, name, istbl: (
- f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else
- # HACK. TODO(justinmk): class/structure support in lua2dox
- '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else
- f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'),
- module_override={},
- append_only=[],
- ),
- 'diagnostic': Config(
- mode='lua',
- filename='diagnostic.txt',
- section_order=[
- 'diagnostic.lua',
- ],
- files=['runtime/lua/vim/diagnostic.lua'],
- file_patterns='*.lua',
- fn_name_prefix='',
- include_tables=False,
- section_name={'diagnostic.lua': 'diagnostic'},
- section_fmt=lambda _: 'Lua module: vim.diagnostic',
- helptag_fmt=lambda _: '*diagnostic-api*',
- fn_helptag_fmt=lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*',
- module_override={},
- append_only=[],
- ),
- 'treesitter': Config(
- mode='lua',
- filename='treesitter.txt',
- section_order=[
- 'treesitter.lua',
- 'language.lua',
- 'query.lua',
- 'highlighter.lua',
- 'languagetree.lua',
- 'dev.lua',
- ],
- files=[
- 'runtime/lua/vim/treesitter.lua',
- 'runtime/lua/vim/treesitter/',
- ],
- file_patterns='*.lua',
- fn_name_prefix='',
- section_name={},
- section_fmt=lambda name: (
- 'Lua module: vim.treesitter'
- if name.lower() == 'treesitter'
- else f'Lua module: vim.treesitter.{name.lower()}'),
- helptag_fmt=lambda name: (
- '*lua-treesitter-core*'
- if name.lower() == 'treesitter'
- else f'*lua-treesitter-{name.lower()}*'),
- fn_helptag_fmt=lambda fstem, name, istbl: (
- f'*vim.{fstem}.{name}()*'
- if fstem == 'treesitter'
- else f'*{name}()*'
- if name[0].isupper()
- else f'*vim.treesitter.{fstem}.{name}()*'),
- module_override={},
- append_only=[],
- ),
-}
-
-param_exclude = (
- 'channel_id',
-)
-
-# Annotations are displayed as line items after API function descriptions.
-annotation_map = {
- 'FUNC_API_FAST': '|api-fast|',
- 'FUNC_API_TEXTLOCK': 'not allowed when |textlock| is active or in the |cmdwin|',
- 'FUNC_API_TEXTLOCK_ALLOW_CMDWIN': 'not allowed when |textlock| is active',
- 'FUNC_API_REMOTE_ONLY': '|RPC| only',
- 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only',
-}
-
-
-def nvim_api_info() -> Tuple[int, bool]:
- """Returns NVIM_API_LEVEL, NVIM_API_PRERELEASE from CMakeLists.txt"""
- if not hasattr(nvim_api_info, 'LEVEL'):
- script_dir = os.path.dirname(os.path.abspath(__file__))
- cmake_file_path = os.path.join(script_dir, '..', 'CMakeLists.txt')
- with open(cmake_file_path, 'r') as cmake_file:
- cmake_content = cmake_file.read()
-
- api_level_match = re.search(r'set\(NVIM_API_LEVEL (\d+)\)', cmake_content)
- api_prerelease_match = re.search(
- r'set\(NVIM_API_PRERELEASE (\w+)\)', cmake_content
- )
-
- if not api_level_match or not api_prerelease_match:
- raise RuntimeError(
- 'Could not find NVIM_API_LEVEL or NVIM_API_PRERELEASE in CMakeLists.txt'
- )
-
- nvim_api_info.LEVEL = int(api_level_match.group(1))
- nvim_api_info.PRERELEASE = api_prerelease_match.group(1).lower() == 'true'
-
- return nvim_api_info.LEVEL, nvim_api_info.PRERELEASE
-
-
-# Raises an error with details about `o`, if `cond` is in object `o`,
-# or if `cond()` is callable and returns True.
-def debug_this(o, cond=True):
- name = ''
- if cond is False:
- return
- if not isinstance(o, str):
- try:
- name = o.nodeName
- o = o.toprettyxml(indent=' ', newl='\n')
- except Exception:
- pass
- if (cond is True
- or (callable(cond) and cond())
- or (not callable(cond) and cond in o)):
- raise RuntimeError('xxx: {}\n{}'.format(name, o))
-
-
-# Appends a message to a list which will be printed on exit.
-def msg(s):
- msgs.append(s)
-
-
-# Print all collected messages.
-def msg_report():
- for m in msgs:
- print(f' {m}')
-
-
-# Print collected messages, then throw an exception.
-def fail(s):
- msg_report()
- raise RuntimeError(s)
-
-
-def find_first(parent, name):
- """Finds the first matching node within parent."""
- sub = parent.getElementsByTagName(name)
- if not sub:
- return None
- return sub[0]
-
-
-def iter_children(parent, name):
- """Yields matching child nodes within parent."""
- for child in parent.childNodes:
- if child.nodeType == child.ELEMENT_NODE and child.nodeName == name:
- yield child
-
-
-def get_child(parent, name):
- """Gets the first matching child node."""
- for child in iter_children(parent, name):
- return child
- return None
-
-
-def self_or_child(n):
- """Gets the first child node, or self."""
- if len(n.childNodes) == 0:
- return n
- return n.childNodes[0]
-
-
-def align_tags(line):
- tag_regex = r"\s(\*.+?\*)(?:\s|$)"
- tags = re.findall(tag_regex, line)
-
- if len(tags) > 0:
- line = re.sub(tag_regex, "", line)
- tags = " " + " ".join(tags)
- line = line + (" " * (78 - len(line) - len(tags))) + tags
- return line
-
-
-def clean_lines(text):
- """Removes superfluous lines.
-
- The beginning and end of the string is trimmed. Empty lines are collapsed.
- """
- return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text))
-
-
-def is_blank(text):
- return '' == clean_lines(text)
-
-
-def get_text(n):
- """Recursively concatenates all text in a node tree."""
- text = ''
- if n.nodeType == n.TEXT_NODE:
- return n.data
- if n.nodeName == 'computeroutput':
- for node in n.childNodes:
- text += get_text(node)
- return '`{}`'.format(text)
- if n.nodeName == 'sp': # space, used in "programlisting" nodes
- return ' '
- for node in n.childNodes:
- if node.nodeType == node.TEXT_NODE:
- text += node.data
- elif node.nodeType == node.ELEMENT_NODE:
- text += get_text(node)
- return text
-
-
-# Gets the length of the last line in `text`, excluding newline ("\n") char.
-def len_lastline(text):
- lastnl = text.rfind('\n')
- if -1 == lastnl:
- return len(text)
- if '\n' == text[-1]:
- return lastnl - (1 + text.rfind('\n', 0, lastnl))
- return len(text) - (1 + lastnl)
-
-
-def len_lastline_withoutindent(text, indent):
- n = len_lastline(text)
- return (n - len(indent)) if n > len(indent) else 0
-
-
-# Returns True if node `n` contains only inline (not block-level) elements.
-def is_inline(n):
- # if len(n.childNodes) == 0:
- # return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput'
- for c in n.childNodes:
- if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput':
- return False
- if not is_inline(c):
- return False
- return True
-
-
-def doc_wrap(text, prefix='', width=70, func=False, indent=None) -> str:
- """Wraps text to `width`.
-
- First line is prefixed with `prefix`, subsequent lines are aligned.
- If `func` is True, only wrap at commas.
- """
- if not width:
- # return prefix + text
- return text
-
- # Whitespace used to indent all lines except the first line.
- indent = ' ' * len(prefix) if indent is None else indent
- indent_only = (prefix == '' and indent is not None)
-
- if func:
- lines = [prefix]
- for part in text.split(', '):
- if part[-1] not in ');':
- part += ', '
- if len(lines[-1]) + len(part) > width:
- lines.append(indent)
- lines[-1] += part
- return '\n'.join(x.rstrip() for x in lines).rstrip()
-
- # XXX: Dummy prefix to force TextWrapper() to wrap the first line.
- if indent_only:
- prefix = indent
-
- tw = textwrap.TextWrapper(break_long_words=False,
- break_on_hyphens=False,
- width=width,
- initial_indent=prefix,
- subsequent_indent=indent)
- result = '\n'.join(tw.wrap(text.strip()))
-
- # XXX: Remove the dummy prefix.
- if indent_only:
- result = result[len(indent):]
-
- return result
-
-
-def max_name(names):
- if len(names) == 0:
- return 0
- return max(len(name) for name in names)
-
-
-def update_params_map(parent, ret_map, width=text_width - indentation):
- """Updates `ret_map` with name:desc key-value pairs extracted
- from Doxygen XML node `parent`.
- """
- params = collections.OrderedDict()
- for node in parent.childNodes:
- if node.nodeType == node.TEXT_NODE:
- continue
- name_node = find_first(node, 'parametername')
- if name_node.getAttribute('direction') == 'out':
- continue
- name = get_text(name_node)
- if name in param_exclude:
- continue
- params[name.strip()] = node
- max_name_len = max_name(params.keys()) + 8
- # `ret_map` is a name:desc map.
- for name, node in params.items():
- desc = ''
- desc_node = get_child(node, 'parameterdescription')
- if desc_node:
- desc = fmt_node_as_vimhelp(
- desc_node, width=width, indent=(' ' * max_name_len))
- ret_map[name] = desc
- return ret_map
-
-
-def render_node(n: Element, text: str, prefix='', *,
- indent: str = '',
- width: int = (text_width - indentation),
- fmt_vimhelp: bool = False):
- """Renders a node as Vim help text, recursively traversing all descendants."""
-
- def ind(s):
- return s if fmt_vimhelp else ''
-
- # Get the current column offset from the last line of `text`
- # (needed to appropriately wrap multiple and contiguous inline elements)
- col_offset: int = len_lastline(text)
-
- text = ''
- # space_preceding = (len(text) > 0 and ' ' == text[-1][-1])
- # text += (int(not space_preceding) * ' ')
-
- if n.nodeName == 'preformatted':
- o = get_text(n)
- ensure_nl = '' if o[-1] == '\n' else '\n'
- if o[0:4] == 'lua\n':
- text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1])
- elif o[0:4] == 'vim\n':
- text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1])
- elif o[0:5] == 'help\n':
- text += o[4:-1]
- else:
- text += '>{}{}\n<'.format(ensure_nl, o)
- elif n.nodeName == 'programlisting': # codeblock (```)
- o = get_text(n)
- text += '>'
- if 'filename' in n.attributes:
- filename = n.attributes['filename'].value
- text += filename.lstrip('.')
-
- text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4))
- elif is_inline(n):
- o = get_text(n).strip()
- if o:
- DEL = chr(127) # a dummy character to pad for proper line wrap
- assert len(DEL) == 1
- dummy_padding = DEL * max(0, col_offset - len(prefix))
- text += doc_wrap(dummy_padding + o,
- prefix=prefix, indent=indent, width=width
- ).replace(DEL, "")
- elif n.nodeName == 'verbatim':
- # TODO: currently we don't use this. The "[verbatim]" hint is there as
- # a reminder that we must decide how to format this if we do use it.
- text += ' [verbatim] {}'.format(get_text(n))
- elif n.nodeName == 'listitem':
- for c in n.childNodes:
- result = render_node(
- c,
- text,
- indent=indent + (' ' * len(prefix)),
- width=width
- )
- if is_blank(result):
- continue
- text += indent + prefix + result
- elif n.nodeName in ('para', 'heading'):
- did_prefix = False
- for c in n.childNodes:
- c_text = render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width)
- if (is_inline(c)
- and '' != c_text.strip()
- and text
- and text[-1] not in (' ', '(', '|')
- and not c_text.startswith(')')):
- text += ' '
- text += c_text
- did_prefix = True
- elif n.nodeName == 'itemizedlist':
- for c in n.childNodes:
- text += '{}\n'.format(render_node(c, text, prefix='• ',
- indent=indent, width=width))
- elif n.nodeName == 'orderedlist':
- i = 1
- for c in n.childNodes:
- if is_blank(get_text(c)):
- text += '\n'
- continue
- text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i),
- indent=indent, width=width))
- i = i + 1
- elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'):
- text += ind(' ')
- for c in n.childNodes:
- if is_blank(render_node(c, text, prefix='• ', indent=' ', width=width)):
- continue
- text += render_node(c, text, prefix='• ', indent=' ', width=width)
- # text += '\n'
- elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'):
- text += 'Warning:\n '
- for c in n.childNodes:
- text += render_node(c, text, indent=' ', width=width)
- text += '\n'
- elif n.nodeName == 'simplesect' and 'see' == n.getAttribute('kind'):
- text += ind(' ')
- # Example:
- # <simplesect kind="see">
- # <para>|autocommand|</para>
- # </simplesect>
- for c in n.childNodes:
- text += render_node(c, text, prefix='• ', indent=' ', width=width)
- elif n.nodeName == 'simplesect' and 'return' == n.getAttribute('kind'):
- text += ind(' ')
- for c in n.childNodes:
- text += render_node(c, text, indent=' ', width=width)
- elif n.nodeName == 'computeroutput':
- return get_text(n)
- else:
- raise RuntimeError('unhandled node type: {}\n{}'.format(
- n.nodeName, n.toprettyxml(indent=' ', newl='\n')))
-
- return text
-
-
-def para_as_map(parent: Element,
- indent: str = '',
- width: int = (text_width - indentation),
- ):
- """Extracts a Doxygen XML <para> node to a map.
-
- Keys:
- 'text': Text from this <para> element
- 'note': List of @note strings
- 'params': <parameterlist> map
- 'return': List of @return strings
- 'seealso': List of @see strings
- 'xrefs': ?
- """
- chunks = {
- 'text': '',
- 'note': [],
- 'params': collections.OrderedDict(),
- 'return': [],
- 'seealso': [],
- 'prerelease': False,
- 'xrefs': []
- }
-
- # Ordered dict of ordered lists.
- groups = collections.OrderedDict([
- ('note', []),
- ('params', []),
- ('return', []),
- ('seealso', []),
- ('xrefs', []),
- ])
-
- # Gather nodes into groups. Mostly this is because we want "parameterlist"
- # nodes to appear together.
- text = ''
- kind = ''
- if is_inline(parent):
- # Flatten inline text from a tree of non-block nodes.
- text = doc_wrap(render_node(parent, ""),
- indent=indent, width=width)
- else:
- prev = None # Previous node
- for child in parent.childNodes:
- if child.nodeName == 'parameterlist':
- groups['params'].append(child)
- elif child.nodeName == 'xrefsect':
- groups['xrefs'].append(child)
- elif child.nodeName == 'simplesect':
- kind = child.getAttribute('kind')
- if kind == 'note':
- groups['note'].append(child)
- elif kind == 'return':
- groups['return'].append(child)
- elif kind == 'see':
- groups['seealso'].append(child)
- elif kind == 'warning':
- text += render_node(child, text, indent=indent, width=width)
- elif kind == 'since':
- since_match = re.match(r'^(\d+)', get_text(child))
- since = int(since_match.group(1)) if since_match else 0
- NVIM_API_LEVEL, NVIM_API_PRERELEASE = nvim_api_info()
- if since > NVIM_API_LEVEL or (
- since == NVIM_API_LEVEL and NVIM_API_PRERELEASE
- ):
- chunks['prerelease'] = True
- else:
- raise RuntimeError('unhandled simplesect: {}\n{}'.format(
- child.nodeName, child.toprettyxml(indent=' ', newl='\n')))
- else:
- child_text = render_node(child, text, indent=indent, width=width)
- if (prev is not None
- and is_inline(self_or_child(prev))
- and is_inline(self_or_child(child))
- and '' != get_text(self_or_child(child)).strip()
- and text
- and text[-1] not in (' ', '(', '|')
- and not child_text.startswith(')')):
- text += ' '
-
- text += child_text
- prev = child
-
- chunks['text'] += text
-
- # Generate map from the gathered items.
- if len(groups['params']) > 0:
- for child in groups['params']:
- update_params_map(child, ret_map=chunks['params'], width=width)
- for child in groups['note']:
- chunks['note'].append(render_node(
- child, '', indent=indent, width=width).rstrip())
- for child in groups['return']:
- chunks['return'].append(render_node(
- child, '', indent=indent, width=width))
- for child in groups['seealso']:
- # Example:
- # <simplesect kind="see">
- # <para>|autocommand|</para>
- # </simplesect>
- chunks['seealso'].append(render_node(
- child, '', indent=indent, width=width))
-
- xrefs = set()
- for child in groups['xrefs']:
- # XXX: Add a space (or any char) to `title` here, otherwise xrefs
- # ("Deprecated" section) acts very weird...
- title = get_text(get_child(child, 'xreftitle')) + ' '
- xrefs.add(title)
- xrefdesc = get_text(get_child(child, 'xrefdescription'))
- chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title),
- width=width) + '\n')
-
- return chunks, xrefs
-
-
-def is_program_listing(para):
- """
- Return True if `para` contains a "programlisting" (i.e. a Markdown code
- block ```).
-
- Sometimes a <para> element will have only a single "programlisting" child
- node, but othertimes it will have extra whitespace around the
- "programlisting" node.
-
- @param para XML <para> node
- @return True if <para> is a programlisting
- """
-
- # Remove any child text nodes that are only whitespace
- children = [
- n for n in para.childNodes
- if n.nodeType != n.TEXT_NODE or n.data.strip() != ''
- ]
-
- return len(children) == 1 and children[0].nodeName == 'programlisting'
-
-
-FunctionParam = Tuple[
- str, # type
- str, # parameter name
-]
-
-@dataclasses.dataclass
-class FunctionDoc:
- """Data structure for function documentation. Also exported as msgpack."""
-
- annotations: List[str]
- """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map"""
-
- notes: List[Docstring]
- """Notes: (@note strings)"""
-
- signature: str
- """Function signature with *tags*."""
-
- parameters: List[FunctionParam]
- """Parameters: (type, name)"""
-
- parameters_doc: Dict[str, Docstring]
- """Parameters documentation. Key is parameter name, value is doc."""
-
- doc: List[Docstring]
- """Main description for the function. Separated by paragraph."""
-
- return_: List[Docstring]
- """Return:, or Return (multiple): (@return strings)"""
-
- seealso: List[Docstring]
- """See also: (@see strings)"""
-
- xrefs: List[Docstring]
- """XRefs. Currently only used to track Deprecated functions."""
-
- # for INCLUDE_C_DECL
- c_decl: str | None = None
-
- prerelease: bool = False
-
- def export_mpack(self) -> Dict[str, Any]:
- """Convert a dict to be exported as mpack data."""
- exported = self.__dict__.copy()
- del exported['notes']
- del exported['c_decl']
- del exported['prerelease']
- del exported['xrefs']
- exported['return'] = exported.pop('return_')
- return exported
-
- def doc_concatenated(self) -> Docstring:
- """Concatenate all the paragraphs in `doc` into a single string, but
- remove blank lines before 'programlisting' blocks. #25127
-
- BEFORE (without programlisting processing):
- ```vimdoc
- Example:
-
- >vim
- :echo nvim_get_color_by_name("Pink")
- <
- ```
-
- AFTER:
- ```vimdoc
- Example: >vim
- :echo nvim_get_color_by_name("Pink")
- <
- ```
- """
- def is_program_listing(paragraph: str) -> bool:
- lines = paragraph.strip().split('\n')
- return lines[0].startswith('>') and lines[-1] == '<'
-
- rendered = []
- for paragraph in self.doc:
- if is_program_listing(paragraph):
- rendered.append(' ') # Example: >vim
- elif rendered:
- rendered.append('\n\n')
- rendered.append(paragraph)
- return ''.join(rendered)
-
- def render(self) -> Docstring:
- """Renders function documentation as Vim :help text."""
- rendered_blocks: List[Docstring] = []
-
- def fmt_param_doc(m):
- """Renders a params map as Vim :help text."""
- max_name_len = max_name(m.keys()) + 4
- out = ''
- for name, desc in m.items():
- if name == 'self':
- continue
- name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len))
- out += '{}{}\n'.format(name, desc)
- return out.rstrip()
-
- # Generate text from the gathered items.
- chunks: List[Docstring] = [self.doc_concatenated()]
-
- notes = []
- if self.prerelease:
- notes = [" This API is pre-release (unstable)."]
- notes += self.notes
- if len(notes) > 0:
- chunks.append('\nNote: ~')
- for s in notes:
- chunks.append(' ' + s)
-
- if self.parameters_doc:
- chunks.append('\nParameters: ~')
- chunks.append(fmt_param_doc(self.parameters_doc))
-
- if self.return_:
- chunks.append('\nReturn (multiple): ~' if len(self.return_) > 1
- else '\nReturn: ~')
- for s in self.return_:
- chunks.append(' ' + s)
-
- if self.seealso:
- chunks.append('\nSee also: ~')
- for s in self.seealso:
- chunks.append(' ' + s)
-
- # Note: xrefs are currently only used to remark "Deprecated: "
- # for deprecated functions; visible when INCLUDE_DEPRECATED is set
- for s in self.xrefs:
- chunks.append('\n' + s)
-
- rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
- rendered_blocks.append('')
-
- return clean_lines('\n'.join(rendered_blocks).strip())
-
-
-def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=''):
- """Renders (nested) Doxygen <para> nodes as Vim :help text.
-
- Only handles "text" nodes. Used for individual elements (see render_node())
- and in extract_defgroups().
-
- NB: Blank lines in a docstring manifest as <para> tags.
- """
- rendered_blocks = []
-
- for child in parent.childNodes:
- para, _ = para_as_map(child, indent, width)
-
- # 'programlisting' blocks are Markdown code blocks. Do not include
- # these as a separate paragraph, but append to the last non-empty line
- # in the text
- if is_program_listing(child):
- while rendered_blocks and rendered_blocks[-1] == '':
- rendered_blocks.pop()
- rendered_blocks[-1] += ' ' + para['text']
- continue
-
- # Generate text from the gathered items.
- chunks = [para['text']]
-
- rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
- rendered_blocks.append('')
-
- return clean_lines('\n'.join(rendered_blocks).strip())
-
-
-def extract_from_xml(filename, target, *,
- width: int, fmt_vimhelp: bool) -> Tuple[
- Dict[FunctionName, FunctionDoc],
- Dict[FunctionName, FunctionDoc],
-]:
- """Extracts Doxygen info as maps without formatting the text.
-
- Returns two maps:
- 1. Functions
- 2. Deprecated functions
-
- The `fmt_vimhelp` variable controls some special cases for use by
- fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :)
- """
- config: Config = CONFIG[target]
-
- fns: Dict[FunctionName, FunctionDoc] = {}
- deprecated_fns: Dict[FunctionName, FunctionDoc] = {}
-
- dom = minidom.parse(filename)
- compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
- for member in dom.getElementsByTagName('memberdef'):
- if member.getAttribute('static') == 'yes' or \
- member.getAttribute('kind') != 'function' or \
- member.getAttribute('prot') == 'private' or \
- get_text(get_child(member, 'name')).startswith('_'):
- continue
-
- loc = find_first(member, 'location')
- if 'private' in loc.getAttribute('file'):
- continue
-
- return_type = get_text(get_child(member, 'type'))
- if return_type == '':
- continue
-
- if 'local_function' in return_type: # Special from lua2dox.lua.
- continue
-
- istbl = return_type.startswith('table') # Special from lua2dox.lua.
- if istbl and not config.include_tables:
- continue
-
- if return_type.startswith(('ArrayOf', 'DictionaryOf')):
- parts = return_type.strip('_').split('_')
- return_type = '{}({})'.format(parts[0], ', '.join(parts[1:]))
-
- name = get_text(get_child(member, 'name'))
-
- annotations = get_text(get_child(member, 'argsstring'))
- if annotations and ')' in annotations:
- annotations = annotations.rsplit(')', 1)[-1].strip()
- # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of
- # non-void functions. Special-case void functions here.
- if name == 'nvim_get_mode' and len(annotations) == 0:
- annotations += 'FUNC_API_FAST'
- annotations = filter(None, map(lambda x: annotation_map.get(x),
- annotations.split()))
-
- params = []
- type_length = 0
-
- for param in iter_children(member, 'param'):
- param_type = get_text(get_child(param, 'type')).strip()
- param_name = ''
- declname = get_child(param, 'declname')
- if declname:
- param_name = get_text(declname).strip()
- elif config.mode == 'lua':
- # XXX: this is what lua2dox gives us...
- param_name = param_type
- param_type = ''
-
- if param_name in param_exclude:
- continue
-
- if fmt_vimhelp and param_type.endswith('*'):
- param_type = param_type.strip('* ')
- param_name = '*' + param_name
-
- type_length = max(type_length, len(param_type))
- params.append((param_type, param_name))
-
- # Handle Object Oriented style functions here.
- # We make sure they have "self" in the parameters,
- # and a parent function
- if return_type.startswith('function') \
- and len(return_type.split(' ')) >= 2 \
- and any(x[1] == 'self' for x in params):
- split_return = return_type.split(' ')
- name = f'{split_return[1]}:{name}'
- params = [x for x in params if x[1] != 'self']
-
- c_args = []
- for param_type, param_name in params:
- c_args.append((' ' if fmt_vimhelp else '') + (
- '%s %s' % (param_type.ljust(type_length), param_name)).strip())
-
- if not fmt_vimhelp:
- pass
- else:
- fstem = '?'
- if '.' in compoundname:
- fstem = compoundname.split('.')[0]
- fstem = config.module_override.get(fstem, fstem)
- vimtag = config.fn_helptag_fmt(fstem, name, istbl)
-
- if config.fn_name_fmt:
- name = config.fn_name_fmt(fstem, name)
-
- if istbl:
- aopen, aclose = '', ''
- else:
- aopen, aclose = '(', ')'
-
- prefix = name + aopen
- suffix = ', '.join('{%s}' % a[1] for a in params
- if a[0] not in ('void', 'Error', 'Arena',
- 'lua_State')) + aclose
-
- if not fmt_vimhelp:
- c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
- signature = prefix + suffix
- else:
- c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name,
- ',\n'.join(c_args)),
- ' ')
-
- # Minimum 8 chars between signature and vimtag
- lhs = (width - 8) - len(vimtag)
-
- if len(prefix) + len(suffix) > lhs:
- signature = vimtag.rjust(width) + '\n'
- signature += doc_wrap(suffix, width=width, prefix=prefix,
- func=True)
- else:
- signature = prefix + suffix
- signature += vimtag.rjust(width - len(signature))
-
- # Tracks `xrefsect` titles. As of this writing, used only for separating
- # deprecated functions.
- xrefs_all = set()
- paras: List[Dict[str, Any]] = [] # paras means paragraphs!
- brief_desc = find_first(member, 'briefdescription')
- if brief_desc:
- for child in brief_desc.childNodes:
- para, xrefs = para_as_map(child)
- paras.append(para)
- xrefs_all.update(xrefs)
-
- desc = find_first(member, 'detaileddescription')
- if desc:
- paras_detail = [] # override briefdescription
- for child in desc.childNodes:
- para, xrefs = para_as_map(child)
- paras_detail.append(para)
- xrefs_all.update(xrefs)
- log.debug(
- textwrap.indent(
- re.sub(r'\n\s*\n+', '\n',
- desc.toprettyxml(indent=' ', newl='\n')),
- ' ' * indentation))
-
- # override briefdescription, if detaileddescription is not empty
- # (note: briefdescription can contain some erroneous luadoc
- # comments from preceding comments, this is a bug of lua2dox)
- if any((para['text'] or para['note'] or para['params'] or
- para['return'] or para['seealso']
- ) for para in paras_detail):
- paras = paras_detail
-
- fn = FunctionDoc(
- annotations=list(annotations),
- notes=[],
- signature=signature,
- parameters=params,
- parameters_doc=collections.OrderedDict(),
- doc=[],
- return_=[],
- seealso=[],
- xrefs=[],
- )
-
- for m in paras:
- if m.get('text', ''):
- fn.doc.append(m['text'])
- if 'params' in m:
- # Merge OrderedDicts.
- fn.parameters_doc.update(m['params'])
- if 'return' in m and len(m['return']) > 0:
- fn.return_ += m['return']
- if 'seealso' in m and len(m['seealso']) > 0:
- fn.seealso += m['seealso']
- if m.get('prerelease', False):
- fn.prerelease = True
- if 'note' in m:
- fn.notes += m['note']
- if 'xrefs' in m:
- fn.xrefs += m['xrefs']
-
- if INCLUDE_C_DECL:
- fn.c_decl = c_decl
-
- if 'Deprecated' in str(xrefs_all):
- deprecated_fns[name] = fn
- elif name.startswith(config.fn_name_prefix):
- fns[name] = fn
-
- # sort functions by name (lexicographically)
- fns = collections.OrderedDict(sorted(
- fns.items(),
- key=lambda key_item_tuple: key_item_tuple[0].lower(),
- ))
- deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items()))
- return fns, deprecated_fns
-
-
-def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]:
- """Entrypoint for generating Vim :help from from Doxygen XML.
-
- Returns 2 items:
- 1. Vim help text for functions found in `filename`.
- 2. Vim help text for deprecated functions.
- """
- config: Config = CONFIG[target]
-
- fns_txt = {} # Map of func_name:vim-help-text.
- deprecated_fns_txt = {} # Map of func_name:vim-help-text.
-
- fns: Dict[FunctionName, FunctionDoc]
- deprecated_fns: Dict[FunctionName, FunctionDoc]
- fns, deprecated_fns = extract_from_xml(
- filename, target, width=text_width, fmt_vimhelp=True)
-
- def _handle_fn(fn_name: FunctionName, fn: FunctionDoc,
- fns_txt: Dict[FunctionName, Docstring], deprecated=False):
- # Generate Vim :help for parameters.
-
- # Generate body from FunctionDoc, not XML nodes
- doc = fn.render()
- if not doc and fn_name.startswith("nvim__"):
- return
- if not doc:
- doc = ('TODO: Documentation' if not deprecated
- else 'Deprecated.')
-
- # Annotations: put before Parameters
- annotations: str = '\n'.join(fn.annotations)
- if annotations:
- annotations = ('\n\nAttributes: ~\n' +
- textwrap.indent(annotations, ' '))
- i = doc.rfind('Parameters: ~')
- if i == -1:
- doc += annotations
- else:
- doc = doc[:i] + annotations + '\n\n' + doc[i:]
-
- # C Declaration: (debug only)
- if INCLUDE_C_DECL:
- doc += '\n\nC Declaration: ~\n>\n'
- assert fn.c_decl is not None
- doc += fn.c_decl
- doc += '\n<'
-
- # Start of function documentations. e.g.,
- # nvim_cmd({*cmd}, {*opts}) *nvim_cmd()*
- func_doc = fn.signature + '\n'
- func_doc += textwrap.indent(clean_lines(doc), ' ' * indentation)
-
- # Verbatim handling.
- func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)
-
- def process_helptags(func_doc: str) -> str:
- lines: List[str] = func_doc.split('\n')
- # skip ">lang ... <" regions
- is_verbatim: bool = False
- for i in range(len(lines)):
- if re.search(' >([a-z])*$', lines[i]):
- is_verbatim = True
- elif is_verbatim and lines[i].strip() == '<':
- is_verbatim = False
- if not is_verbatim:
- lines[i] = align_tags(lines[i])
- return "\n".join(lines)
-
- func_doc = process_helptags(func_doc)
-
- if (fn_name.startswith(config.fn_name_prefix)
- and fn_name != "nvim_error_event"):
- fns_txt[fn_name] = func_doc
-
- for fn_name, fn in fns.items():
- _handle_fn(fn_name, fn, fns_txt)
- for fn_name, fn in deprecated_fns.items():
- _handle_fn(fn_name, fn, deprecated_fns_txt, deprecated=True)
-
- return (
- '\n\n'.join(list(fns_txt.values())),
- '\n\n'.join(list(deprecated_fns_txt.values())),
- )
-
-
-def delete_lines_below(filename, tokenstr):
- """Deletes all lines below the line containing `tokenstr`, the line itself,
- and one line above it.
- """
- lines = open(filename).readlines()
- i = 0
- found = False
- for i, line in enumerate(lines, 1):
- if tokenstr in line:
- found = True
- break
- if not found:
- raise RuntimeError(f'not found: "{tokenstr}"')
- i = max(0, i - 2)
- with open(filename, 'wt') as fp:
- fp.writelines(lines[0:i])
-
-
-def extract_defgroups(base: str, dom: Document) -> Dict[SectionName, Docstring]:
- '''Generate module-level (section) docs (@defgroup).'''
- section_docs = {}
-
- for compound in dom.getElementsByTagName('compound'):
- if compound.getAttribute('kind') != 'group':
- continue
-
- # Doxygen "@defgroup" directive.
- groupname = get_text(find_first(compound, 'name'))
- groupxml = os.path.join(base, '%s.xml' %
- compound.getAttribute('refid'))
-
- group_parsed = minidom.parse(groupxml)
- doc_list = []
- brief_desc = find_first(group_parsed, 'briefdescription')
- if brief_desc:
- for child in brief_desc.childNodes:
- doc_list.append(fmt_node_as_vimhelp(child))
-
- desc = find_first(group_parsed, 'detaileddescription')
- if desc:
- doc = fmt_node_as_vimhelp(desc)
-
- if doc:
- doc_list.append(doc)
-
- # Can't use '.' in @defgroup, so convert to '--'
- # "vim.json" => "vim-dot-json"
- groupname = groupname.replace('-dot-', '.')
-
- section_docs[groupname] = "\n".join(doc_list)
-
- return section_docs
-
-
-@dataclasses.dataclass
-class Section:
- """Represents a section. Includes section heading (defgroup)
- and all the FunctionDoc that belongs to this section."""
-
- name: str
- '''Name of the section. Usually derived from basename of lua/c src file.
- Example: "Autocmd".'''
-
- title: str
- '''Formatted section config. see config.section_fmt().
- Example: "Autocmd Functions". '''
-
- helptag: str
- '''see config.helptag_fmt(). Example: *api-autocmd*'''
-
- @property
- def id(self) -> str:
- '''section id: Module/Section id matched against @defgroup.
- e.g., "*api-autocmd*" => "api-autocmd"
- '''
- return self.helptag.strip('*')
-
- doc: str = ""
- '''Section heading docs extracted from @defgroup.'''
-
- # TODO: Do not carry rendered text, but handle FunctionDoc for better OOP
- functions_text: Docstring | None = None
- '''(Rendered) doc of all the functions that belong to this section.'''
-
- deprecated_functions_text: Docstring | None = None
- '''(Rendered) doc of all the deprecated functions that belong to this
- section.'''
-
- def __repr__(self):
- return f"Section(title='{self.title}', helptag='{self.helptag}')"
-
- @classmethod
- def make_from(cls, filename: str, config: Config,
- section_docs: Dict[SectionName, str],
- *,
- functions_text: Docstring,
- deprecated_functions_text: Docstring,
- ):
- # filename: e.g., 'autocmd.c'
- # name: e.g. 'autocmd'
- name = os.path.splitext(filename)[0].lower()
-
- # section name: e.g. "Autocmd"
- sectname: SectionName
- sectname = name.upper() if name == 'ui' else name.title()
- sectname = config.section_name.get(filename, sectname)
-
- # Formatted (this is what's going to be written in the vimdoc)
- # e.g., "Autocmd Functions"
- title: str = config.section_fmt(sectname)
-
- # section tag: e.g., "*api-autocmd*"
- section_tag: str = config.helptag_fmt(sectname)
-
- section = cls(name=sectname, title=title, helptag=section_tag,
- functions_text=functions_text,
- deprecated_functions_text=deprecated_functions_text,
- )
- section.doc = section_docs.get(section.id) or ''
- return section
-
- def render(self, add_header=True) -> str:
- """Render as vimdoc."""
- doc = ''
-
- if add_header:
- doc += SECTION_SEP
- doc += '\n{}{}'.format(
- self.title,
- self.helptag.rjust(text_width - len(self.title))
- )
-
- if self.doc:
- doc += '\n\n' + self.doc
-
- if self.functions_text:
- doc += '\n\n' + self.functions_text
-
- if INCLUDE_DEPRECATED and self.deprecated_functions_text:
- doc += f'\n\n\nDeprecated {self.name} Functions: ~\n\n'
- doc += self.deprecated_functions_text
-
- return doc
-
- def __bool__(self) -> bool:
- """Whether this section has contents. Used for skipping empty ones."""
- return bool(self.doc or self.functions_text or
- (INCLUDE_DEPRECATED and self.deprecated_functions_text))
-
-
-def main(doxygen_config, args):
- """Generates:
-
- 1. Vim :help docs
- 2. *.mpack files for use by API clients
-
- Doxygen is called and configured through stdin.
- """
- for target in CONFIG:
- if args.target is not None and target != args.target:
- continue
-
- config: Config = CONFIG[target]
-
- mpack_file = os.path.join(
- base_dir, 'runtime', 'doc',
- config.filename.replace('.txt', '.mpack'))
- if os.path.exists(mpack_file):
- os.remove(mpack_file)
-
- output_dir = out_dir.format(target=target)
- log.info("Generating documentation for %s in folder %s",
- target, output_dir)
- debug = args.log_level >= logging.DEBUG
- p = subprocess.Popen(
- ['doxygen', '-'],
- stdin=subprocess.PIPE,
- # silence warnings
- # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found
- stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL))
- p.communicate(
- doxygen_config.format(
- input=' '.join([f'"{file}"' for file in config.files]),
- output=output_dir,
- filter=filter_cmd,
- file_patterns=config.file_patterns)
- .encode('utf8')
- )
- if p.returncode:
- sys.exit(p.returncode)
-
- # Collects all functions as each module is processed.
- fn_map_full: Dict[FunctionName, FunctionDoc] = {}
- # key: filename (e.g. autocmd.c)
- sections: Dict[str, Section] = {}
-
- base = os.path.join(output_dir, 'xml')
- dom = minidom.parse(os.path.join(base, 'index.xml'))
-
- # Collect all @defgroups (section headings after the '===...' separator
- section_docs: Dict[SectionName, Docstring] = extract_defgroups(base, dom)
-
- # Generate docs for all functions in the current module.
- for compound in dom.getElementsByTagName('compound'):
- if compound.getAttribute('kind') != 'file':
- continue
-
- filename = get_text(find_first(compound, 'name'))
- if not (
- filename.endswith('.c') or
- filename.endswith('.lua')
- ):
- continue
-
- xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid')))
-
- # Extract unformatted (*.mpack).
- fn_map, _ = extract_from_xml(
- xmlfile, target, width=9999, fmt_vimhelp=False)
-
- # Extract formatted (:help).
- functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
- xmlfile, target)
-
- if not functions_text and not deprecated_text:
- continue
-
- filename = os.path.basename(filename)
-
- section: Section = Section.make_from(
- filename, config, section_docs,
- functions_text=functions_text,
- deprecated_functions_text=deprecated_text,
- )
-
- if section: # if not empty
- sections[filename] = section
- fn_map_full.update(fn_map)
- else:
- log.debug("Skipping empty section: %s", section)
-
- if len(sections) == 0:
- fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)')
- if len(sections) > len(config.section_order):
- raise RuntimeError(
- '{}: found new modules {}; '
- 'update the "section_order" map'.format(
- target,
- set(sections).difference(config.section_order))
- )
- first_section_tag = sections[config.section_order[0]].helptag
-
- docs = ''
-
- for filename in config.section_order:
- try:
- section: Section = sections.pop(filename)
- except KeyError:
- msg(f'warning: empty docs, skipping (target={target}): {filename}')
- msg(f' existing docs: {sections.keys()}')
- continue
-
- add_sep_and_header = filename not in config.append_only
- docs += section.render(add_header=add_sep_and_header)
- docs += '\n\n\n'
-
- docs = docs.rstrip() + '\n\n'
- docs += f' vim:tw=78:ts=8:sw={indentation}:sts={indentation}:et:ft=help:norl:\n'
-
- doc_file = os.path.join(base_dir, 'runtime', 'doc', config.filename)
-
- if os.path.exists(doc_file):
- delete_lines_below(doc_file, first_section_tag)
- with open(doc_file, 'ab') as fp:
- fp.write(docs.encode('utf8'))
-
- fn_map_full_exported = collections.OrderedDict(sorted(
- (name, fn_doc.export_mpack()) for (name, fn_doc) in fn_map_full.items()
- ))
- with open(mpack_file, 'wb') as fp:
- fp.write(msgpack.packb(fn_map_full_exported, use_bin_type=True)) # type: ignore
-
- if not args.keep_tmpfiles:
- shutil.rmtree(output_dir)
-
- msg_report()
-
-
-def filter_source(filename, keep_tmpfiles):
- output_dir = out_dir.format(target='lua2dox')
- name, extension = os.path.splitext(filename)
- if extension == '.lua':
- args = [str(nvim), '-l', lua2dox, filename] + (['--outdir', output_dir] if keep_tmpfiles else [])
- p = subprocess.run(args, stdout=subprocess.PIPE)
- op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8'))
- print(op)
- else:
- """Filters the source to fix macros that confuse Doxygen."""
- with open(filename, 'rt') as fp:
- print(re.sub(r'^(ArrayOf|DictionaryOf|Dict)(\(.*?\))',
- lambda m: m.group(1)+'_'.join(
- re.split(r'[^\w]+', m.group(2))),
- fp.read(), flags=re.M))
-
-
-def parse_args():
- targets = ', '.join(CONFIG.keys())
- ap = argparse.ArgumentParser(
- description="Generate helpdoc from source code")
- ap.add_argument(
- "--log-level", "-l", choices=LOG_LEVELS.keys(),
- default=logging.getLevelName(logging.ERROR), help="Set log verbosity"
- )
- ap.add_argument('source_filter', nargs='*',
- help="Filter source file(s)")
- ap.add_argument('-k', '--keep-tmpfiles', action='store_true',
- help="Keep temporary files (tmp-xx-doc/ directories, including tmp-lua2dox-doc/ for lua2dox.lua quasi-C output)")
- ap.add_argument('-t', '--target',
- help=f'One of ({targets}), defaults to "all"')
- return ap.parse_args()
-
-
-Doxyfile = textwrap.dedent('''
- OUTPUT_DIRECTORY = {output}
- INPUT = {input}
- INPUT_ENCODING = UTF-8
- FILE_PATTERNS = {file_patterns}
- RECURSIVE = YES
- INPUT_FILTER = "{filter}"
- EXCLUDE =
- EXCLUDE_SYMLINKS = NO
- EXCLUDE_PATTERNS = */private/* */health.lua */_*.lua
- EXCLUDE_SYMBOLS =
- EXTENSION_MAPPING = lua=C
- EXTRACT_PRIVATE = NO
-
- GENERATE_HTML = NO
- GENERATE_DOCSET = NO
- GENERATE_HTMLHELP = NO
- GENERATE_QHP = NO
- GENERATE_TREEVIEW = NO
- GENERATE_LATEX = NO
- GENERATE_RTF = NO
- GENERATE_MAN = NO
- GENERATE_DOCBOOK = NO
- GENERATE_AUTOGEN_DEF = NO
-
- GENERATE_XML = YES
- XML_OUTPUT = xml
- XML_PROGRAMLISTING = NO
-
- ENABLE_PREPROCESSING = YES
- MACRO_EXPANSION = YES
- EXPAND_ONLY_PREDEF = NO
- MARKDOWN_SUPPORT = YES
-''')
-
-if __name__ == "__main__":
- args = parse_args()
- print("Setting log level to %s" % args.log_level)
- args.log_level = LOG_LEVELS[args.log_level]
- log.setLevel(args.log_level)
- log.addHandler(logging.StreamHandler())
-
- # When invoked as a filter, args won't be passed, so use an env var.
- if args.keep_tmpfiles:
- os.environ['NVIM_KEEP_TMPFILES'] = '1'
- keep_tmpfiles = ('NVIM_KEEP_TMPFILES' in os.environ)
-
- if len(args.source_filter) > 0:
- filter_source(args.source_filter[0], keep_tmpfiles)
- else:
- main(Doxyfile, args)
-
-# vim: set ft=python ts=4 sw=4 tw=79 et :
diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua
deleted file mode 100644
index 0b3daa59b2..0000000000
--- a/scripts/lua2dox.lua
+++ /dev/null
@@ -1,544 +0,0 @@
------------------------------------------------------------------------------
--- Copyright (C) 2012 by Simon Dales --
--- simon@purrsoft.co.uk --
--- --
--- This program is free software; you can redistribute it and/or modify --
--- it under the terms of the GNU General Public License as published by --
--- the Free Software Foundation; either version 2 of the License, or --
--- (at your option) any later version. --
--- --
--- This program is distributed in the hope that it will be useful, --
--- but WITHOUT ANY WARRANTY; without even the implied warranty of --
--- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --
--- GNU General Public License for more details. --
--- --
--- You should have received a copy of the GNU General Public License --
--- along with this program; if not, write to the --
--- Free Software Foundation, Inc., --
--- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. --
------------------------------------------------------------------------------
-
---[[!
-Lua-to-Doxygen converter
-
-Partially from lua2dox
-http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm
-
-RUNNING
--------
-
-This script "lua2dox.lua" gets called by "gen_vimdoc.py".
-
-DEBUGGING/DEVELOPING
----------------------
-
-1. To debug, run gen_vimdoc.py with --keep-tmpfiles:
- python3 scripts/gen_vimdoc.py -t treesitter --keep-tmpfiles
-2. The filtered result will be written to ./tmp-lua2dox-doc/….lua.c
-
-Doxygen must be on your system. You can experiment like so:
-
-- Run "doxygen -g" to create a default Doxyfile.
-- Then alter it to let it recognise lua. Add the following line:
- FILE_PATTERNS = *.lua
-- Then run "doxygen".
-
-The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language.
-It only has to be good enough for doxygen to see it as legal.
-
-One limitation is that each line is treated separately (except for long comments).
-The implication is that class and function declarations must be on the same line.
-
-There is hack that will insert the "missing" close paren.
-The effect is that you will get the function documented, but not with the parameter list you might expect.
-]]
-
-local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' }
-
-local luacats_parser = require('src/nvim/generators/luacats_grammar')
-
-local debug_outfile = nil --- @type string?
-local debug_output = {}
-
---- write to stdout
---- @param str? string
-local function write(str)
- if not str then
- return
- end
-
- io.write(str)
- if debug_outfile then
- table.insert(debug_output, str)
- end
-end
-
---- write to stdout
---- @param str? string
-local function writeln(str)
- write(str)
- write('\n')
-end
-
---- an input file buffer
---- @class StreamRead
---- @field currentLine string?
---- @field contentsLen integer
---- @field currentLineNo integer
---- @field filecontents string[]
-local StreamRead = {}
-
---- @return StreamRead
---- @param filename string
-function StreamRead.new(filename)
- assert(filename, ('invalid file: %s'):format(filename))
- -- get lines from file
- -- syphon lines to our table
- local filecontents = {} --- @type string[]
- for line in io.lines(filename) do
- filecontents[#filecontents + 1] = line
- end
-
- return setmetatable({
- filecontents = filecontents,
- contentsLen = #filecontents,
- currentLineNo = 1,
- }, { __index = StreamRead })
-end
-
--- get a line
-function StreamRead:getLine()
- if self.currentLine then
- self.currentLine = nil
- return self.currentLine
- end
-
- -- get line
- if self.currentLineNo <= self.contentsLen then
- local line = self.filecontents[self.currentLineNo]
- self.currentLineNo = self.currentLineNo + 1
- return line
- end
-
- return ''
-end
-
--- save line fragment
---- @param line_fragment string
-function StreamRead:ungetLine(line_fragment)
- self.currentLine = line_fragment
-end
-
--- is it eof?
-function StreamRead:eof()
- return not self.currentLine and self.currentLineNo > self.contentsLen
-end
-
--- input filter
---- @class Lua2DoxFilter
-local Lua2DoxFilter = {
- generics = {}, --- @type table<string,string>
- block_ignore = false, --- @type boolean
-}
-setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter })
-
-function Lua2DoxFilter:reset()
- self.generics = {}
- self.block_ignore = false
-end
-
---- trim comment off end of string
----
---- @param line string
---- @return string, string?
-local function removeCommentFromLine(line)
- local pos_comment = line:find('%-%-')
- if not pos_comment then
- return line
- end
- return line:sub(1, pos_comment - 1), line:sub(pos_comment)
-end
-
---- @param parsed luacats.Return
---- @return string
-local function get_return_type(parsed)
- local elems = {} --- @type string[]
- for _, v in ipairs(parsed) do
- local e = v.type --- @type string
- if v.name then
- e = e .. ' ' .. v.name --- @type string
- end
- elems[#elems + 1] = e
- end
- return '(' .. table.concat(elems, ', ') .. ')'
-end
-
---- @param name string
---- @return string
-local function process_name(name, optional)
- if optional then
- name = name:sub(1, -2) --- @type string
- end
- return name
-end
-
---- @param ty string
---- @param generics table<string,string>
---- @return string
-local function process_type(ty, generics, optional)
- -- replace generic types
- for k, v in pairs(generics) do
- ty = ty:gsub(k, v) --- @type string
- end
-
- -- strip parens
- ty = ty:gsub('^%((.*)%)$', '%1')
-
- if optional and not ty:find('nil') then
- ty = ty .. '?'
- end
-
- -- remove whitespace in unions
- ty = ty:gsub('%s*|%s*', '|')
-
- -- replace '|nil' with '?'
- ty = ty:gsub('|nil', '?')
- ty = ty:gsub('nil|(.*)', '%1?')
-
- return '(`' .. ty .. '`)'
-end
-
---- @param parsed luacats.Param
---- @param generics table<string,string>
---- @return string
-local function process_param(parsed, generics)
- local name, ty = parsed.name, parsed.type
- local optional = vim.endswith(name, '?')
-
- return table.concat({
- '/// @param',
- process_name(name, optional),
- process_type(ty, generics, optional),
- parsed.desc,
- }, ' ')
-end
-
---- @param parsed luacats.Return
---- @param generics table<string,string>
---- @return string
-local function process_return(parsed, generics)
- local ty, name --- @type string, string
- if #parsed == 1 then
- ty, name = parsed[1].type, parsed[1].name or ''
- else
- ty, name = get_return_type(parsed), ''
- end
-
- local optional = vim.endswith(name, '?')
-
- return table.concat({
- '/// @return',
- process_type(ty, generics, optional),
- process_name(name, optional),
- parsed.desc,
- }, ' ')
-end
-
---- Processes "@…" directives in a docstring line.
----
---- @param line string
---- @return string?
-function Lua2DoxFilter:process_magic(line)
- line = line:gsub('^%s+@', '@')
- line = line:gsub('@package', '@private')
- line = line:gsub('@nodoc', '@private')
-
- if self.block_ignore then
- return '// gg:" ' .. line .. '"'
- end
-
- if not vim.startswith(line, '@') then -- it's a magic comment
- return '/// ' .. line
- end
-
- local magic_split = vim.split(line, ' ', { plain = true })
- local directive = magic_split[1]
-
- if
- vim.list_contains({
- '@cast',
- '@diagnostic',
- '@overload',
- '@meta',
- '@type',
- }, directive)
- then
- -- Ignore LSP directives
- return '// gg:"' .. line .. '"'
- elseif directive == '@defgroup' or directive == '@addtogroup' then
- -- Can't use '.' in defgroup, so convert to '--'
- return '/// ' .. line:gsub('%.', '-dot-')
- end
-
- if directive == '@alias' then
- -- this contiguous block should be all ignored.
- self.block_ignore = true
- return '// gg:"' .. line .. '"'
- end
-
- -- preprocess line before parsing
- if directive == '@param' or directive == '@return' then
- for _, type in ipairs(TYPES) do
- line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2')
- line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2')
- line = line:gsub('^@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2')
-
- line = line:gsub('^@return%s+.*%((' .. type .. ')%)', '@return %1')
- line = line:gsub('^@return%s+.*%((' .. type .. '|nil)%)', '@return %1')
- line = line:gsub('^@return%s+.*%((' .. type .. '%?)%)', '@return %1')
- end
- end
-
- local parsed = luacats_parser:match(line)
-
- if not parsed then
- return '/// ' .. line
- end
-
- local kind = parsed.kind
-
- if kind == 'generic' then
- self.generics[parsed.name] = parsed.type or 'any'
- return
- elseif kind == 'param' then
- return process_param(parsed --[[@as luacats.Param]], self.generics)
- elseif kind == 'return' then
- return process_return(parsed --[[@as luacats.Return]], self.generics)
- end
-
- error(string.format('unhandled parsed line %q: %s', line, parsed))
-end
-
---- @param line string
---- @param in_stream StreamRead
---- @return string
-function Lua2DoxFilter:process_block_comment(line, in_stream)
- local comment_parts = {} --- @type string[]
- local done --- @type boolean?
-
- while not done and not in_stream:eof() do
- local thisComment --- @type string?
- local closeSquare = line:find(']]')
- if not closeSquare then -- need to look on another line
- thisComment = line .. '\n'
- line = in_stream:getLine()
- else
- thisComment = line:sub(1, closeSquare - 1)
- done = true
-
- -- unget the tail of the line
- -- in most cases it's empty. This may make us less efficient but
- -- easier to program
- in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2)))
- end
- comment_parts[#comment_parts + 1] = thisComment
- end
-
- local comment = table.concat(comment_parts)
-
- if comment:sub(1, 1) == '@' then -- it's a long magic comment
- return '/*' .. comment .. '*/ '
- end
-
- -- discard
- return '/* zz:' .. comment .. '*/ '
-end
-
---- @param line string
---- @return string
-function Lua2DoxFilter:process_function_header(line)
- local pos_fn = assert(line:find('function'))
- -- we've got a function
- local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8)))
-
- if fn:sub(1, 1) == '(' then
- -- it's an anonymous function
- return '// ZZ: ' .. line
- end
- -- fn has a name, so is interesting
-
- -- want to fix for iffy declarations
- if fn:find('[%({]') then
- -- we might have a missing close paren
- if not fn:find('%)') then
- fn = fn .. ' ___MissingCloseParenHere___)'
- end
- end
-
- -- Big hax
- if fn:find(':') then
- fn = fn:gsub(':', '.', 1)
-
- local paren_start = fn:find('(', 1, true)
- local paren_finish = fn:find(')', 1, true)
-
- -- Nothing in between the parens
- local comma --- @type string
- if paren_finish == paren_start + 1 then
- comma = ''
- else
- comma = ', '
- end
-
- fn = fn:sub(1, paren_start) .. 'self' .. comma .. fn:sub(paren_start + 1)
- end
-
- if line:match('local') then
- -- Special: tell gen_vimdoc.py this is a local function.
- return 'local_function ' .. fn .. '{}'
- end
-
- -- add vanilla function
- return 'function ' .. fn .. '{}'
-end
-
---- @param line string
---- @param in_stream StreamRead
---- @return string?
-function Lua2DoxFilter:process_line(line, in_stream)
- local line_raw = line
- line = vim.trim(line)
-
- if vim.startswith(line, '---') then
- return Lua2DoxFilter:process_magic(line:sub(4))
- end
-
- if vim.startswith(line, '--' .. '[[') then -- it's a long comment
- return Lua2DoxFilter:process_block_comment(line:sub(5), in_stream)
- end
-
- -- Hax... I'm sorry
- -- M.fun = vim.memoize(function(...)
- -- ->
- -- function M.fun(...)
- line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)')
-
- if line:find('^function') or line:find('^local%s+function') then
- return Lua2DoxFilter:process_function_header(line)
- end
-
- if not line:match('^local') then
- local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=')
- if v and v:match('%.') then
- -- Special: this lets gen_vimdoc.py handle tables.
- return 'table ' .. v .. '() {}'
- end
- end
-
- if #line > 0 then -- we don't know what this line means, so just comment it out
- return '// zz: ' .. line
- end
-
- return ''
-end
-
--- Processes the file and writes filtered output to stdout.
----@param filename string
-function Lua2DoxFilter:filter(filename)
- local in_stream = StreamRead.new(filename)
-
- local last_was_magic = false
-
- while not in_stream:eof() do
- local line = in_stream:getLine()
-
- local out_line = self:process_line(line, in_stream)
-
- if not vim.startswith(vim.trim(line), '---') then
- self:reset()
- end
-
- if out_line then
- -- Ensure all magic blocks associate with some object to prevent doxygen
- -- from getting confused.
- if vim.startswith(out_line, '///') then
- last_was_magic = true
- else
- if last_was_magic and out_line:match('^// zz: [^-]+') then
- writeln('local_function _ignore() {}')
- end
- last_was_magic = false
- end
- writeln(out_line)
- end
- end
-end
-
---- @class TApp
---- @field timestamp string|osdate
---- @field name string
---- @field version string
---- @field copyright string
---- this application
-local TApp = {
- timestamp = os.date('%c %Z', os.time()),
- name = 'Lua2DoX',
- version = '0.2 20130128',
- copyright = 'Copyright (c) Simon Dales 2012-13',
-}
-
-setmetatable(TApp, { __index = TApp })
-
-function TApp:getRunStamp()
- return self.name .. ' (' .. self.version .. ') ' .. self.timestamp
-end
-
-function TApp:getVersion()
- return self.name .. ' (' .. self.version .. ') '
-end
-
---main
-
-if arg[1] == '--help' then
- writeln(TApp:getVersion())
- writeln(TApp.copyright)
- writeln([[
- run as:
- nvim -l scripts/lua2dox.lua <param>
- --------------
- Param:
- <filename> : interprets filename
- --version : show version/copyright info
- --help : this help text]])
-elseif arg[1] == '--version' then
- writeln(TApp:getVersion())
- writeln(TApp.copyright)
-else -- It's a filter.
- local filename = arg[1]
-
- if arg[2] == '--outdir' then
- local outdir = arg[3]
- if
- type(outdir) ~= 'string'
- or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir))
- then
- error(('invalid --outdir: "%s"'):format(tostring(outdir)))
- end
- vim.fn.mkdir(outdir, 'p')
- debug_outfile = string.format('%s/%s.c', outdir, vim.fs.basename(filename))
- end
-
- Lua2DoxFilter:filter(filename)
-
- -- output the tail
- writeln('// #######################')
- writeln('// app run:' .. TApp:getRunStamp())
- writeln('// #######################')
- writeln()
-
- if debug_outfile then
- local f = assert(io.open(debug_outfile, 'w'))
- f:write(table.concat(debug_output))
- f:close()
- end
-end
diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua
new file mode 100644
index 0000000000..ee0f9d8e87
--- /dev/null
+++ b/scripts/luacats_grammar.lua
@@ -0,0 +1,218 @@
+--[[!
+LPEG grammar for LuaCATS
+]]
+
+local lpeg = vim.lpeg
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local Ct, Cg = lpeg.Ct, lpeg.Cg
+
+--- @param x vim.lpeg.Pattern
+local function rep(x)
+ return x ^ 0
+end
+
+--- @param x vim.lpeg.Pattern
+local function rep1(x)
+ return x ^ 1
+end
+
+--- @param x vim.lpeg.Pattern
+local function opt(x)
+ return x ^ -1
+end
+
+local nl = P('\r\n') + P('\n')
+local ws = rep1(S(' \t') + nl)
+local fill = opt(ws)
+
+local any = P(1) -- (consume one character)
+local letter = R('az', 'AZ') + S('_$')
+local num = R('09')
+local ident = letter * rep(letter + num + S '-.')
+local string_single = P "'" * rep(any - P "'") * P "'"
+local string_double = P '"' * rep(any - P '"') * P '"'
+
+local literal = (string_single + string_double + (opt(P '-') * num) + P 'false' + P 'true')
+
+local lname = (ident + P '...') * opt(P '?')
+
+--- @param x string
+local function Pf(x)
+ return fill * P(x) * fill
+end
+
+--- @param x string
+local function Sf(x)
+ return fill * S(x) * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function comma(x)
+ return x * rep(Pf ',' * x)
+end
+
+--- @param x vim.lpeg.Pattern
+local function parenOpt(x)
+ return (Pf('(') * x * fill * P(')')) + x
+end
+
+--- @type table<string,vim.lpeg.Pattern>
+local v = setmetatable({}, {
+ __index = function(_, k)
+ return lpeg.V(k)
+ end,
+})
+
+local desc_delim = Sf '#:' + ws
+
+--- @class nvim.luacats.Param
+--- @field kind 'param'
+--- @field name string
+--- @field type string
+--- @field desc? string
+
+--- @class nvim.luacats.Return
+--- @field kind 'return'
+--- @field [integer] { type: string, name?: string}
+--- @field desc? string
+
+--- @class nvim.luacats.Generic
+--- @field kind 'generic'
+--- @field name string
+--- @field type? string
+
+--- @class nvim.luacats.Class
+--- @field kind 'class'
+--- @field name string
+--- @field parent? string
+
+--- @class nvim.luacats.Field
+--- @field kind 'field'
+--- @field name string
+--- @field type string
+--- @field desc? string
+--- @field access? 'private'|'protected'|'package'
+
+--- @class nvim.luacats.Note
+--- @field desc? string
+
+--- @alias nvim.luacats.grammar.result
+--- | nvim.luacats.Param
+--- | nvim.luacats.Return
+--- | nvim.luacats.Generic
+--- | nvim.luacats.Class
+--- | nvim.luacats.Field
+--- | nvim.luacats.Note
+
+--- @class nvim.luacats.grammar
+--- @field match fun(self, input: string): nvim.luacats.grammar.result?
+
+local grammar = P {
+ rep1(P('@') * (v.ats + v.ext_ats)),
+
+ ats = v.at_param
+ + v.at_return
+ + v.at_type
+ + v.at_cast
+ + v.at_generic
+ + v.at_class
+ + v.at_field
+ + v.at_access
+ + v.at_deprecated
+ + v.at_alias
+ + v.at_enum
+ + v.at_see
+ + v.at_diagnostic
+ + v.at_overload
+ + v.at_meta,
+
+ ext_ats = v.ext_at_note + v.ext_at_since + v.ext_at_nodoc + v.ext_at_brief,
+
+ at_param = Ct(
+ Cg(P('param'), 'kind')
+ * ws
+ * Cg(lname, 'name')
+ * ws
+ * parenOpt(Cg(v.ltype, 'type'))
+ * opt(desc_delim * Cg(rep(any), 'desc'))
+ ),
+
+ at_return = Ct(
+ Cg(P('return'), 'kind')
+ * ws
+ * parenOpt(comma(Ct(Cg(v.ltype, 'type') * opt(ws * Cg(ident, 'name')))))
+ * opt(desc_delim * Cg(rep(any), 'desc'))
+ ),
+
+ at_type = Ct(
+ Cg(P('type'), 'kind')
+ * ws
+ * parenOpt(comma(Ct(Cg(v.ltype, 'type'))))
+ * opt(desc_delim * Cg(rep(any), 'desc'))
+ ),
+
+ at_cast = Ct(
+ Cg(P('cast'), 'kind') * ws * Cg(lname, 'name') * ws * opt(Sf('+-')) * Cg(v.ltype, 'type')
+ ),
+
+ at_generic = Ct(
+ Cg(P('generic'), 'kind') * ws * Cg(ident, 'name') * opt(Pf ':' * Cg(v.ltype, 'type'))
+ ),
+
+ at_class = Ct(
+ Cg(P('class'), 'kind')
+ * ws
+ * opt(P('(exact)') * ws)
+ * Cg(lname, 'name')
+ * opt(Pf(':') * Cg(lname, 'parent'))
+ ),
+
+ at_field = Ct(
+ Cg(P('field'), 'kind')
+ * ws
+ * opt(Cg(Pf('private') + Pf('package') + Pf('protected'), 'access'))
+ * Cg(lname, 'name')
+ * ws
+ * Cg(v.ltype, 'type')
+ * opt(desc_delim * Cg(rep(any), 'desc'))
+ ),
+
+ at_access = Ct(Cg(P('private') + P('protected') + P('package'), 'kind')),
+
+ at_deprecated = Ct(Cg(P('deprecated'), 'kind')),
+
+ -- Types may be provided on subsequent lines
+ at_alias = Ct(Cg(P('alias'), 'kind') * ws * Cg(lname, 'name') * opt(ws * Cg(v.ltype, 'type'))),
+
+ at_enum = Ct(Cg(P('enum'), 'kind') * ws * Cg(lname, 'name')),
+
+ at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')),
+ at_diagnostic = Ct(Cg(P('diagnostic'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')),
+ at_overload = Ct(Cg(P('overload'), 'kind') * ws * Cg(v.ltype, 'type')),
+ at_meta = Ct(Cg(P('meta'), 'kind')),
+
+ --- Custom extensions
+ ext_at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')),
+
+ -- TODO only consume 1 line
+ ext_at_since = Ct(Cg(P('since'), 'kind') * ws * Cg(rep(any), 'desc')),
+
+ ext_at_nodoc = Ct(Cg(P('nodoc'), 'kind')),
+ ext_at_brief = Ct(Cg(P('brief'), 'kind') * opt(ws * Cg(rep(any), 'desc'))),
+
+ ltype = v.ty_union + Pf '(' * v.ty_union * fill * P ')',
+
+ ty_union = v.ty_opt * rep(Pf '|' * v.ty_opt),
+ ty = v.ty_fun + ident + v.ty_table + literal,
+ ty_param = Pf '<' * comma(v.ltype) * fill * P '>',
+ ty_opt = v.ty * opt(v.ty_param) * opt(P '[]') * opt(P '?'),
+
+ table_key = (Pf '[' * literal * Pf ']') + lname,
+ table_elem = v.table_key * Pf ':' * v.ltype,
+ ty_table = Pf '{' * comma(v.table_elem) * Pf '}',
+
+ fun_param = lname * opt(Pf ':' * v.ltype),
+ ty_fun = Pf 'fun(' * rep(comma(v.fun_param)) * fill * P ')' * opt(Pf ':' * comma(v.ltype)),
+}
+
+return grammar --[[@as nvim.luacats.grammar]]
diff --git a/scripts/luacats_parser.lua b/scripts/luacats_parser.lua
new file mode 100644
index 0000000000..520272d1dc
--- /dev/null
+++ b/scripts/luacats_parser.lua
@@ -0,0 +1,521 @@
+local luacats_grammar = require('scripts.luacats_grammar')
+
+--- @class nvim.luacats.parser.param
+--- @field name string
+--- @field type string
+--- @field desc string
+
+--- @class nvim.luacats.parser.return
+--- @field name string
+--- @field type string
+--- @field desc string
+
+--- @class nvim.luacats.parser.note
+--- @field desc string
+
+--- @class nvim.luacats.parser.brief
+--- @field kind 'brief'
+--- @field desc string
+
+--- @class nvim.luacats.parser.alias
+--- @field kind 'alias'
+--- @field type string
+--- @field desc string
+
+--- @class nvim.luacats.parser.fun
+--- @field name string
+--- @field params nvim.luacats.parser.param[]
+--- @field returns nvim.luacats.parser.return[]
+--- @field desc string
+--- @field access? 'private'|'package'|'protected'
+--- @field class? string
+--- @field module? string
+--- @field modvar? string
+--- @field classvar? string
+--- @field deprecated? true
+--- @field since? string
+--- @field attrs? string[]
+--- @field nodoc? true
+--- @field generics? table<string,string>
+--- @field table? true
+--- @field notes? nvim.luacats.parser.note[]
+--- @field see? nvim.luacats.parser.note[]
+
+--- @class nvim.luacats.parser.field
+--- @field name string
+--- @field type string
+--- @field desc string
+--- @field access? 'private'|'package'|'protected'
+
+--- @class nvim.luacats.parser.class
+--- @field kind 'class'
+--- @field name string
+--- @field desc string
+--- @field fields nvim.luacats.parser.field[]
+--- @field notes? string[]
+
+--- @class nvim.luacats.parser.State
+--- @field doc_lines? string[]
+--- @field cur_obj? nvim.luacats.parser.obj
+--- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note
+--- @field last_doc_item_indent? integer
+
+--- @alias nvim.luacats.parser.obj
+--- | nvim.luacats.parser.class
+--- | nvim.luacats.parser.fun
+--- | nvim.luacats.parser.brief
+
+-- Remove this when we document classes properly
+--- Some doc lines have the form:
+--- param name some.complex.type (table) description
+--- if so then transform the line to remove the complex type:
+--- param name (table) description
+--- @param line string
+local function use_type_alt(line)
+ for _, type in ipairs({ 'table', 'function' }) do
+ line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2')
+ line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2')
+ line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2')
+
+ line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1')
+ line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1')
+ line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1')
+ end
+ return line
+end
+
+--- If we collected any `---` lines. Add them to the existing (or new) object
+--- Used for function/class descriptions and multiline param descriptions.
+--- @param state nvim.luacats.parser.State
+local function add_doc_lines_to_obj(state)
+ if state.doc_lines then
+ state.cur_obj = state.cur_obj or {}
+ local cur_obj = assert(state.cur_obj)
+ local txt = table.concat(state.doc_lines, '\n')
+ if cur_obj.desc then
+ cur_obj.desc = cur_obj.desc .. '\n' .. txt
+ else
+ cur_obj.desc = txt
+ end
+ state.doc_lines = nil
+ end
+end
+
+--- @param line string
+--- @param state nvim.luacats.parser.State
+local function process_doc_line(line, state)
+ line = line:sub(4):gsub('^%s+@', '@')
+ line = use_type_alt(line)
+
+ local parsed = luacats_grammar:match(line)
+
+ if not parsed then
+ if line:match('^ ') then
+ line = line:sub(2)
+ end
+
+ if state.last_doc_item then
+ if not state.last_doc_item_indent then
+ state.last_doc_item_indent = #line:match('^%s*') + 1
+ end
+ state.last_doc_item.desc = (state.last_doc_item.desc or '')
+ .. '\n'
+ .. line:sub(state.last_doc_item_indent or 1)
+ else
+ state.doc_lines = state.doc_lines or {}
+ table.insert(state.doc_lines, line)
+ end
+ return
+ end
+
+ state.last_doc_item_indent = nil
+ state.last_doc_item = nil
+ state.cur_obj = state.cur_obj or {}
+ local cur_obj = assert(state.cur_obj)
+
+ local kind = parsed.kind
+
+ if kind == 'brief' then
+ state.cur_obj = {
+ kind = 'brief',
+ desc = parsed.desc,
+ }
+ elseif kind == 'class' then
+ --- @cast parsed nvim.luacats.Class
+ state.cur_obj = {
+ kind = 'class',
+ name = parsed.name,
+ parent = parsed.parent,
+ desc = '',
+ fields = {},
+ }
+ elseif kind == 'field' then
+ --- @cast parsed nvim.luacats.Field
+ if not parsed.access then
+ parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil
+ if parsed.desc then
+ parsed.desc = vim.trim(parsed.desc)
+ end
+ table.insert(cur_obj.fields, parsed)
+ end
+ state.doc_lines = nil
+ elseif kind == 'param' then
+ state.last_doc_item_indent = nil
+ cur_obj.params = cur_obj.params or {}
+ if vim.endswith(parsed.name, '?') then
+ parsed.name = parsed.name:sub(1, -2)
+ parsed.type = parsed.type .. '?'
+ end
+ state.last_doc_item = {
+ name = parsed.name,
+ type = parsed.type,
+ desc = parsed.desc,
+ }
+ table.insert(cur_obj.params, state.last_doc_item)
+ elseif kind == 'return' then
+ cur_obj.returns = cur_obj.returns or {}
+ for _, t in ipairs(parsed) do
+ table.insert(cur_obj.returns, {
+ name = t.name,
+ type = t.type,
+ desc = parsed.desc,
+ })
+ end
+ state.last_doc_item_indent = nil
+ state.last_doc_item = cur_obj.returns[#cur_obj.returns]
+ elseif kind == 'private' then
+ cur_obj.access = 'private'
+ elseif kind == 'package' then
+ cur_obj.access = 'package'
+ elseif kind == 'protected' then
+ cur_obj.access = 'protected'
+ elseif kind == 'deprecated' then
+ cur_obj.deprecated = true
+ elseif kind == 'nodoc' then
+ cur_obj.nodoc = true
+ elseif kind == 'since' then
+ cur_obj.since = parsed.desc
+ elseif kind == 'see' then
+ cur_obj.see = cur_obj.see or {}
+ table.insert(cur_obj.see, { desc = parsed.desc })
+ elseif kind == 'note' then
+ state.last_doc_item_indent = nil
+ state.last_doc_item = {
+ desc = parsed.desc,
+ }
+ cur_obj.notes = cur_obj.notes or {}
+ table.insert(cur_obj.notes, state.last_doc_item)
+ elseif kind == 'type' then
+ cur_obj.desc = parsed.desc
+ parsed.desc = nil
+ parsed.kind = nil
+ cur_obj.type = parsed
+ elseif kind == 'alias' then
+ state.cur_obj = {
+ kind = 'alias',
+ desc = parsed.desc,
+ }
+ elseif kind == 'enum' then
+ -- TODO
+ state.doc_lines = nil
+ elseif
+ vim.tbl_contains({
+ 'diagnostic',
+ 'cast',
+ 'overload',
+ 'meta',
+ }, kind)
+ then
+ -- Ignore
+ return
+ elseif kind == 'generic' then
+ cur_obj.generics = cur_obj.generics or {}
+ cur_obj.generics[parsed.name] = parsed.type or 'any'
+ else
+ error('Unhandled' .. vim.inspect(parsed))
+ end
+end
+
+--- @param fun nvim.luacats.parser.fun
+--- @return nvim.luacats.parser.field
+local function fun2field(fun)
+ local parts = { 'fun(' }
+ for _, p in ipairs(fun.params or {}) do
+ parts[#parts + 1] = string.format('%s: %s', p.name, p.type)
+ end
+ parts[#parts + 1] = ')'
+ if fun.returns then
+ parts[#parts + 1] = ': '
+ local tys = {} --- @type string[]
+ for _, p in ipairs(fun.returns) do
+ tys[#tys + 1] = p.type
+ end
+ parts[#parts + 1] = table.concat(tys, ', ')
+ end
+
+ return {
+ name = fun.name,
+ type = table.concat(parts, ''),
+ access = fun.access,
+ desc = fun.desc,
+ }
+end
+
+--- Function to normalize known form for declaring functions and normalize into a more standard
+--- form.
+--- @param line string
+--- @return string
+local function filter_decl(line)
+ -- M.fun = vim._memoize(function(...)
+ -- ->
+ -- function M.fun(...)
+ line = line:gsub('^local (.+) = .*_memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)')
+ line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)')
+ return line
+end
+
+--- @param line string
+--- @param state nvim.luacats.parser.State
+--- @param classes table<string,nvim.luacats.parser.class>
+--- @param classvars table<string,string>
+--- @param has_indent boolean
+local function process_lua_line(line, state, classes, classvars, has_indent)
+ line = filter_decl(line)
+
+ if state.cur_obj and state.cur_obj.kind == 'class' then
+ local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=')
+ if nm then
+ classvars[nm] = state.cur_obj.name
+ end
+ return
+ end
+
+ do
+ local parent_tbl, sep, fun_or_meth_nm =
+ line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(')
+ if parent_tbl then
+ -- Have a decl. Ensure cur_obj
+ state.cur_obj = state.cur_obj or {}
+ local cur_obj = assert(state.cur_obj)
+
+ -- Match `Class:foo` methods for defined classes
+ local class = classvars[parent_tbl]
+ if class then
+ --- @cast cur_obj nvim.luacats.parser.fun
+ cur_obj.name = fun_or_meth_nm
+ cur_obj.class = class
+ cur_obj.classvar = parent_tbl
+ -- Add self param to methods
+ if sep == ':' then
+ cur_obj.params = cur_obj.params or {}
+ table.insert(cur_obj.params, 1, {
+ name = 'self',
+ type = class,
+ })
+ end
+
+ -- Add method as the field to the class
+ table.insert(classes[class].fields, fun2field(cur_obj))
+ return
+ end
+
+ -- Match `M.foo`
+ if cur_obj and parent_tbl == cur_obj.modvar then
+ cur_obj.name = fun_or_meth_nm
+ return
+ end
+ end
+ end
+
+ do
+ -- Handle: `function A.B.C.foo(...)`
+ local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(')
+ if fn_nm then
+ state.cur_obj = state.cur_obj or {}
+ state.cur_obj.name = fn_nm
+ return
+ end
+ end
+
+ do
+ -- Handle: `M.foo = {...}` where `M` is the modvar
+ local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=')
+ if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then
+ state.cur_obj.name = tbl_nm
+ state.cur_obj.table = true
+ return
+ end
+ end
+
+ do
+ -- Handle: `foo = {...}`
+ local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=')
+ if tbl_nm and not has_indent then
+ state.cur_obj = state.cur_obj or {}
+ state.cur_obj.name = tbl_nm
+ state.cur_obj.table = true
+ return
+ end
+ end
+
+ do
+ -- Handle: `vim.foo = {...}`
+ local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=')
+ if state.cur_obj and tbl_nm and not has_indent then
+ state.cur_obj.name = tbl_nm
+ state.cur_obj.table = true
+ return
+ end
+ end
+
+ if state.cur_obj then
+ if line:find('^%s*%-%- luacheck:') then
+ state.cur_obj = nil
+ elseif line:find('^%s*local%s+') then
+ state.cur_obj = nil
+ elseif line:find('^%s*return%s+') then
+ state.cur_obj = nil
+ elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then
+ state.cur_obj = nil
+ end
+ end
+end
+
+--- Determine the table name used to export functions of a module
+--- Usually this is `M`.
+--- @param filename string
+--- @return string?
+local function determine_modvar(filename)
+ local modvar --- @type string?
+ for line in io.lines(filename) do
+ do
+ --- @type string?
+ local m = line:match('^return%s+([a-zA-Z_]+)')
+ if m then
+ modvar = m
+ end
+ end
+ do
+ --- @type string?
+ local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),')
+ if m then
+ modvar = m
+ end
+ end
+ end
+ return modvar
+end
+
+--- @param obj nvim.luacats.parser.obj
+--- @param funs nvim.luacats.parser.fun[]
+--- @param classes table<string,nvim.luacats.parser.class>
+--- @param briefs string[]
+--- @param uncommitted nvim.luacats.parser.obj[]
+local function commit_obj(obj, classes, funs, briefs, uncommitted)
+ local commit = false
+ if obj.kind == 'class' then
+ --- @cast obj nvim.luacats.parser.class
+ if not classes[obj.name] then
+ classes[obj.name] = obj
+ commit = true
+ end
+ elseif obj.kind == 'alias' then
+ -- Just pretend
+ commit = true
+ elseif obj.kind == 'brief' then
+ --- @cast obj nvim.luacats.parser.brief`
+ briefs[#briefs + 1] = obj.desc
+ commit = true
+ else
+ --- @cast obj nvim.luacats.parser.fun`
+ if obj.name then
+ funs[#funs + 1] = obj
+ commit = true
+ end
+ end
+ if not commit then
+ table.insert(uncommitted, obj)
+ end
+ return commit
+end
+
+--- @param filename string
+--- @param uncommitted nvim.luacats.parser.obj[]
+-- luacheck: no unused
+local function dump_uncommitted(filename, uncommitted)
+ local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt'
+ if #uncommitted > 0 then
+ print(string.format('Could not commit %d objects in %s', #uncommitted, filename))
+ vim.fn.mkdir(assert(vim.fs.dirname(out_path)), 'p')
+ local f = assert(io.open(out_path, 'w'))
+ for i, x in ipairs(uncommitted) do
+ f:write(i)
+ f:write(': ')
+ f:write(vim.inspect(x))
+ f:write('\n')
+ end
+ f:close()
+ else
+ vim.fn.delete(out_path)
+ end
+end
+
+local M = {}
+
+--- @param filename string
+--- @return table<string,nvim.luacats.parser.class> classes
+--- @return nvim.luacats.parser.fun[] funs
+--- @return string[] briefs
+--- @return nvim.luacats.parser.obj[]
+function M.parse(filename)
+ local funs = {} --- @type nvim.luacats.parser.fun[]
+ local classes = {} --- @type table<string,nvim.luacats.parser.class>
+ local briefs = {} --- @type string[]
+
+ local mod_return = determine_modvar(filename)
+
+ --- @type string
+ local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename
+ module = module:gsub('/', '.')
+
+ local classvars = {} --- @type table<string,string>
+
+ local state = {} --- @type nvim.luacats.parser.State
+
+ -- Keep track of any partial objects we don't commit
+ local uncommitted = {} --- @type nvim.luacats.parser.obj[]
+
+ for line in io.lines(filename) do
+ local has_indent = line:match('^%s+') ~= nil
+ line = vim.trim(line)
+ if vim.startswith(line, '---') then
+ process_doc_line(line, state)
+ else
+ add_doc_lines_to_obj(state)
+
+ if state.cur_obj then
+ state.cur_obj.modvar = mod_return
+ state.cur_obj.module = module
+ end
+
+ process_lua_line(line, state, classes, classvars, has_indent)
+
+ -- Commit the object
+ local cur_obj = state.cur_obj
+ if cur_obj then
+ if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then
+ --- @diagnostic disable-next-line:inject-field
+ cur_obj.line = line
+ end
+ end
+
+ state = {}
+ end
+ end
+
+ -- dump_uncommitted(filename, uncommitted)
+
+ return classes, funs, briefs, uncommitted
+end
+
+return M
diff --git a/scripts/text_utils.lua b/scripts/text_utils.lua
new file mode 100644
index 0000000000..5167ec42f2
--- /dev/null
+++ b/scripts/text_utils.lua
@@ -0,0 +1,239 @@
+local fmt = string.format
+
+--- @class nvim.text_utils.MDNode
+--- @field [integer] nvim.text_utils.MDNode
+--- @field type string
+--- @field text? string
+
+local INDENTATION = 4
+
+local M = {}
+
+local function contains(t, xs)
+ return vim.tbl_contains(xs, t)
+end
+
+--- @param text string
+--- @return nvim.text_utils.MDNode
+local function parse_md(text)
+ local parser = vim.treesitter.languagetree.new(text, 'markdown', {
+ injections = { markdown = '' },
+ })
+
+ local root = parser:parse(true)[1]:root()
+
+ local EXCLUDE_TEXT_TYPE = {
+ list = true,
+ list_item = true,
+ section = true,
+ document = true,
+ fenced_code_block = true,
+ fenced_code_block_delimiter = true,
+ }
+
+ --- @param node TSNode
+ --- @return nvim.text_utils.MDNode?
+ local function extract(node)
+ local ntype = node:type()
+
+ if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then
+ return
+ end
+
+ --- @type table<any,any>
+ local ret = { type = ntype }
+
+ if not EXCLUDE_TEXT_TYPE[ntype] then
+ ret.text = vim.treesitter.get_node_text(node, text)
+ end
+
+ for child, child_field in node:iter_children() do
+ local e = extract(child)
+ if child_field then
+ ret[child_field] = e
+ else
+ table.insert(ret, e)
+ end
+ end
+
+ return ret
+ end
+
+ return extract(root) or {}
+end
+
+--- @param x string
+--- @param start_indent integer
+--- @param indent integer
+--- @param text_width integer
+--- @return string
+function M.wrap(x, start_indent, indent, text_width)
+ local words = vim.split(vim.trim(x), '%s+')
+ local parts = { string.rep(' ', start_indent) } --- @type string[]
+ local count = indent
+
+ for i, w in ipairs(words) do
+ if count > indent and count + #w > text_width - 1 then
+ parts[#parts + 1] = '\n'
+ parts[#parts + 1] = string.rep(' ', indent)
+ count = indent
+ elseif i ~= 1 then
+ parts[#parts + 1] = ' '
+ count = count + 1
+ end
+ count = count + #w
+ parts[#parts + 1] = w
+ end
+
+ return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', ''))
+end
+
+--- @param node nvim.text_utils.MDNode
+--- @param start_indent integer
+--- @param indent integer
+--- @param text_width integer
+--- @param level integer
+--- @return string[]
+local function render_md(node, start_indent, indent, text_width, level, is_list)
+ local parts = {} --- @type string[]
+
+ -- For debugging
+ local add_tag = false
+ -- local add_tag = true
+
+ if add_tag then
+ parts[#parts + 1] = '<' .. node.type .. '>'
+ end
+
+ if node.type == 'paragraph' then
+ local text = assert(node.text)
+ text = text:gsub('(%s)%*(%w+)%*(%s)', '%1%2%3')
+ text = text:gsub('(%s)_(%w+)_(%s)', '%1%2%3')
+ text = text:gsub('\\|', '|')
+ text = text:gsub('\\%*', '*')
+ text = text:gsub('\\_', '_')
+ parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width)
+ parts[#parts + 1] = '\n'
+ elseif node.type == 'code_fence_content' then
+ local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n')
+
+ local cindent = indent + INDENTATION
+ if level > 3 then
+ -- The tree-sitter markdown parser doesn't parse the code blocks indents
+ -- correctly in lists. Fudge it!
+ lines[1] = ' ' .. lines[1] -- ¯\_(ツ)_/¯
+ cindent = indent - level
+ local _, initial_indent = lines[1]:find('^%s*')
+ initial_indent = initial_indent + cindent
+ if initial_indent < indent then
+ cindent = indent - INDENTATION
+ end
+ end
+
+ for _, l in ipairs(lines) do
+ if #l > 0 then
+ parts[#parts + 1] = string.rep(' ', cindent)
+ parts[#parts + 1] = l
+ end
+ parts[#parts + 1] = '\n'
+ end
+ elseif node.type == 'fenced_code_block' then
+ parts[#parts + 1] = '>'
+ for _, child in ipairs(node) do
+ if child.type == 'info_string' then
+ parts[#parts + 1] = child.text
+ break
+ end
+ end
+ parts[#parts + 1] = '\n'
+ for _, child in ipairs(node) do
+ if child.type ~= 'info_string' then
+ vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1))
+ end
+ end
+ parts[#parts + 1] = '<\n'
+ elseif node.type == 'html_block' then
+ local text = node.text:gsub('^<pre>help', '')
+ text = text:gsub('</pre>%s*$', '')
+ parts[#parts + 1] = text
+ elseif node.type == 'list_marker_dot' then
+ parts[#parts + 1] = node.text
+ elseif contains(node.type, { 'list_marker_minus', 'list_marker_star' }) then
+ parts[#parts + 1] = '• '
+ elseif node.type == 'list_item' then
+ parts[#parts + 1] = string.rep(' ', indent)
+ local offset = node[1].type == 'list_marker_dot' and 3 or 2
+ for i, child in ipairs(node) do
+ local sindent = i <= 2 and 0 or (indent + offset)
+ vim.list_extend(
+ parts,
+ render_md(child, sindent, indent + offset, text_width, level + 1, true)
+ )
+ end
+ else
+ if node.text then
+ error(fmt('cannot render:\n%s', vim.inspect(node)))
+ end
+ for i, child in ipairs(node) do
+ vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1, is_list))
+ if node.type ~= 'list' and i ~= #node then
+ if (node[i + 1] or {}).type ~= 'list' then
+ parts[#parts + 1] = '\n'
+ end
+ end
+ end
+ end
+
+ if add_tag then
+ parts[#parts + 1] = '</' .. node.type .. '>'
+ end
+
+ return parts
+end
+
+--- @param text_width integer
+local function align_tags(text_width)
+ --- @param line string
+ --- @return string
+ return function(line)
+ local tag_pat = '%s+(%*[^ ]+%*)%s*$'
+ local tags = {}
+ for m in line:gmatch(tag_pat) do
+ table.insert(tags, m)
+ end
+
+ if #tags > 0 then
+ line = line:gsub(tag_pat, '')
+ local tags_str = ' ' .. table.concat(tags, ' ')
+ local pad = string.rep(' ', text_width - #line - #tags_str)
+ return line .. pad .. tags_str
+ end
+
+ return line
+ end
+end
+
+--- @param text string
+--- @param start_indent integer
+--- @param indent integer
+--- @param is_list? boolean
+--- @return string
+function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list)
+ -- Add an extra newline so the parser can properly capture ending ```
+ local parsed = parse_md(text .. '\n')
+ local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list)
+
+ local lines = vim.split(table.concat(ret), '\n')
+
+ lines = vim.tbl_map(align_tags(text_width), lines)
+
+ local s = table.concat(lines, '\n')
+
+ -- Reduce whitespace in code-blocks
+ s = s:gsub('\n+%s*>([a-z]+)\n?\n', ' >%1\n')
+ s = s:gsub('\n+%s*>\n?\n', ' >\n')
+
+ return s
+end
+
+return M