From 0f24b0826a27b7868a3aacc25199787e7453d4cc Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Wed, 26 Feb 2025 11:38:07 +0000 Subject: build: move all generator scripts to `src/gen/` - Move all generator Lua scripts to the `src/gen/` - Add a `.luarc.json` to `src/gen/` - Add a `preload.lua` to `src/gen/` - Add `src` to `package.path` so it aligns with `.luarc.json' - Fix all `require` statements in `src/gen/` so they are consistent: - `require('scripts.foo')` -> `require('gen.foo')` - `require('src.nvim.options')` -> `require('nvim.options')` - `require('api.dispatch_deprecated')` -> `require('nvim.api.dispatch_deprecated')` --- scripts/cdoc_grammar.lua | 87 --- scripts/cdoc_parser.lua | 223 ------- scripts/gen_eval_files.lua | 1090 ------------------------------- scripts/gen_filetype.lua | 209 ------ scripts/gen_help_html.lua | 1491 ------------------------------------------- scripts/gen_lsp.lua | 514 --------------- scripts/gen_vimdoc.lua | 1041 ------------------------------ scripts/lintdoc.lua | 4 +- scripts/luacats_grammar.lua | 207 ------ scripts/luacats_parser.lua | 535 ---------------- scripts/release.sh | 2 +- scripts/util.lua | 399 ------------ 12 files changed, 3 insertions(+), 5799 deletions(-) delete mode 100644 scripts/cdoc_grammar.lua delete mode 100644 scripts/cdoc_parser.lua delete mode 100755 scripts/gen_eval_files.lua delete mode 100644 scripts/gen_filetype.lua delete mode 100644 scripts/gen_help_html.lua delete mode 100644 scripts/gen_lsp.lua delete mode 100755 scripts/gen_vimdoc.lua delete mode 100644 scripts/luacats_grammar.lua delete mode 100644 scripts/luacats_parser.lua delete mode 100644 scripts/util.lua (limited to 'scripts') diff --git a/scripts/cdoc_grammar.lua b/scripts/cdoc_grammar.lua deleted file mode 100644 index 6a7610883b..0000000000 --- a/scripts/cdoc_grammar.lua +++ /dev/null @@ -1,87 +0,0 @@ ---[[! -LPEG grammar for C doc comments -]] - ---- @class nvim.cdoc.Param ---- @field kind 'param' ---- @field name string ---- @field desc? string - ---- @class nvim.cdoc.Return ---- @field kind 'return' ---- @field desc string - ---- @class nvim.cdoc.Note ---- @field desc? string - ---- @alias nvim.cdoc.grammar.result ---- | nvim.cdoc.Param ---- | nvim.cdoc.Return ---- | nvim.cdoc.Note - ---- @class nvim.cdoc.grammar ---- @field match fun(self, input: string): nvim.cdoc.grammar.result? - -local lpeg = vim.lpeg -local P, R, S = lpeg.P, lpeg.R, lpeg.S -local Ct, Cg = lpeg.Ct, lpeg.Cg - ---- @param x vim.lpeg.Pattern -local function rep(x) - return x ^ 0 -end - ---- @param x vim.lpeg.Pattern -local function rep1(x) - return x ^ 1 -end - ---- @param x vim.lpeg.Pattern -local function opt(x) - return x ^ -1 -end - -local nl = P('\r\n') + P('\n') -local ws = rep1(S(' \t') + nl) - -local any = P(1) -- (consume one character) -local letter = R('az', 'AZ') + S('_$') -local ident = letter * rep(letter + R('09')) - -local io = P('[') * (P('in') + P('out') + P('inout')) * P(']') - ---- @param x string -local function Pf(x) - return opt(ws) * P(x) * opt(ws) -end - ---- @type table -local v = setmetatable({}, { - __index = function(_, k) - return lpeg.V(k) - end, -}) - -local grammar = P { - rep1(P('@') * v.ats), - - ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc, - - at_param = Ct( - Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc')) - ), - - at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))), - - at_deprecated = Ct(Cg(P('deprecated'), 'kind')), - - at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), - - at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')), - - at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), - - at_nodoc = Ct(Cg(P('nodoc'), 'kind')), -} - -return grammar --[[@as nvim.cdoc.grammar]] diff --git a/scripts/cdoc_parser.lua b/scripts/cdoc_parser.lua deleted file mode 100644 index 5f0dc7be2c..0000000000 --- a/scripts/cdoc_parser.lua +++ /dev/null @@ -1,223 +0,0 @@ -local cdoc_grammar = require('scripts.cdoc_grammar') -local c_grammar = require('src.nvim.generators.c_grammar') - ---- @class nvim.cdoc.parser.param ---- @field name string ---- @field type string ---- @field desc string - ---- @class nvim.cdoc.parser.return ---- @field name string ---- @field type string ---- @field desc string - ---- @class nvim.cdoc.parser.note ---- @field desc string - ---- @class nvim.cdoc.parser.brief ---- @field kind 'brief' ---- @field desc string - ---- @class nvim.cdoc.parser.fun ---- @field name string ---- @field params nvim.cdoc.parser.param[] ---- @field returns nvim.cdoc.parser.return[] ---- @field desc string ---- @field deprecated? true ---- @field since? string ---- @field attrs? string[] ---- @field nodoc? true ---- @field notes? nvim.cdoc.parser.note[] ---- @field see? nvim.cdoc.parser.note[] - ---- @class nvim.cdoc.parser.State ---- @field doc_lines? string[] ---- @field cur_obj? nvim.cdoc.parser.obj ---- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note ---- @field last_doc_item_indent? integer - ---- @alias nvim.cdoc.parser.obj ---- | nvim.cdoc.parser.fun ---- | nvim.cdoc.parser.brief - ---- If we collected any `---` lines. Add them to the existing (or new) object ---- Used for function/class descriptions and multiline param descriptions. ---- @param state nvim.cdoc.parser.State -local function add_doc_lines_to_obj(state) - if state.doc_lines then - state.cur_obj = state.cur_obj or {} - local cur_obj = assert(state.cur_obj) - local txt = table.concat(state.doc_lines, '\n') - if cur_obj.desc then - cur_obj.desc = cur_obj.desc .. '\n' .. txt - else - cur_obj.desc = txt - end - state.doc_lines = nil - end -end - ---- @param line string ---- @param state nvim.cdoc.parser.State -local function process_doc_line(line, state) - line = line:gsub('^%s+@', '@') - - local parsed = cdoc_grammar:match(line) - - if not parsed then - if line:match('^ ') then - line = line:sub(2) - end - - if state.last_doc_item then - if not state.last_doc_item_indent then - state.last_doc_item_indent = #line:match('^%s*') + 1 - end - state.last_doc_item.desc = (state.last_doc_item.desc or '') - .. '\n' - .. line:sub(state.last_doc_item_indent or 1) - else - state.doc_lines = state.doc_lines or {} - table.insert(state.doc_lines, line) - end - return - end - - state.last_doc_item_indent = nil - state.last_doc_item = nil - - local kind = parsed.kind - - state.cur_obj = state.cur_obj or {} - local cur_obj = assert(state.cur_obj) - - if kind == 'brief' then - state.cur_obj = { - kind = 'brief', - desc = parsed.desc, - } - elseif kind == 'param' then - state.last_doc_item_indent = nil - cur_obj.params = cur_obj.params or {} - state.last_doc_item = { - name = parsed.name, - desc = parsed.desc, - } - table.insert(cur_obj.params, state.last_doc_item) - elseif kind == 'return' then - cur_obj.returns = { { - desc = parsed.desc, - } } - state.last_doc_item_indent = nil - state.last_doc_item = cur_obj.returns[1] - elseif kind == 'deprecated' then - cur_obj.deprecated = true - elseif kind == 'nodoc' then - cur_obj.nodoc = true - elseif kind == 'since' then - cur_obj.since = parsed.desc - elseif kind == 'see' then - cur_obj.see = cur_obj.see or {} - table.insert(cur_obj.see, { desc = parsed.desc }) - elseif kind == 'note' then - state.last_doc_item_indent = nil - state.last_doc_item = { - desc = parsed.desc, - } - cur_obj.notes = cur_obj.notes or {} - table.insert(cur_obj.notes, state.last_doc_item) - else - error('Unhandled' .. vim.inspect(parsed)) - end -end - ---- @param item table ---- @param state nvim.cdoc.parser.State -local function process_proto(item, state) - state.cur_obj = state.cur_obj or {} - local cur_obj = assert(state.cur_obj) - cur_obj.name = item.name - cur_obj.params = cur_obj.params or {} - - for _, p in ipairs(item.parameters) do - local param = { name = p[2], type = p[1] } - local added = false - for _, cp in ipairs(cur_obj.params) do - if cp.name == param.name then - cp.type = param.type - added = true - break - end - end - - if not added then - table.insert(cur_obj.params, param) - end - end - - cur_obj.returns = cur_obj.returns or { {} } - cur_obj.returns[1].type = item.return_type - - for _, a in ipairs({ - 'fast', - 'remote_only', - 'lua_only', - 'textlock', - 'textlock_allow_cmdwin', - }) do - if item[a] then - cur_obj.attrs = cur_obj.attrs or {} - table.insert(cur_obj.attrs, a) - end - end - - cur_obj.deprecated_since = item.deprecated_since - - -- Remove some arguments - for i = #cur_obj.params, 1, -1 do - local p = cur_obj.params[i] - if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then - table.remove(cur_obj.params, i) - end - end -end - -local M = {} - ---- @param filename string ---- @return {} classes ---- @return nvim.cdoc.parser.fun[] funs ---- @return string[] briefs -function M.parse(filename) - local funs = {} --- @type nvim.cdoc.parser.fun[] - local briefs = {} --- @type string[] - local state = {} --- @type nvim.cdoc.parser.State - - local txt = assert(io.open(filename, 'r')):read('*all') - - local parsed = c_grammar.grammar:match(txt) - for _, item in ipairs(parsed) do - if item.comment then - process_doc_line(item.comment, state) - else - add_doc_lines_to_obj(state) - if item[1] == 'proto' then - process_proto(item, state) - table.insert(funs, state.cur_obj) - end - local cur_obj = state.cur_obj - if cur_obj and not item.static then - if cur_obj.kind == 'brief' then - table.insert(briefs, cur_obj.desc) - end - end - state = {} - end - end - - return {}, funs, briefs -end - --- M.parse('src/nvim/api/vim.c') - -return M diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua deleted file mode 100755 index aaf76a0411..0000000000 --- a/scripts/gen_eval_files.lua +++ /dev/null @@ -1,1090 +0,0 @@ -#!/usr/bin/env -S nvim -l - --- Generator for various vimdoc and Lua type files - -local util = require('scripts.util') -local fmt = string.format - -local DEP_API_METADATA = 'build/funcs_metadata.mpack' -local TEXT_WIDTH = 78 - ---- @class vim.api.metadata ---- @field name string ---- @field parameters [string,string][] ---- @field return_type string ---- @field deprecated_since integer ---- @field eval boolean ---- @field fast boolean ---- @field handler_id integer ---- @field impl_name string ---- @field lua boolean ---- @field method boolean ---- @field remote boolean ---- @field since integer - -local LUA_API_RETURN_OVERRIDES = { - nvim_buf_get_command = 'table', - nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item_by_id', - nvim_buf_get_extmarks = 'vim.api.keyset.get_extmark_item[]', - nvim_buf_get_keymap = 'vim.api.keyset.get_keymap[]', - nvim_get_autocmds = 'vim.api.keyset.get_autocmds.ret[]', - nvim_get_color_map = 'table', - nvim_get_command = 'table', - nvim_get_keymap = 'vim.api.keyset.get_keymap[]', - nvim_get_mark = 'vim.api.keyset.get_mark', - - -- Can also return table, however we need to - -- pick one to get some benefit. - -- REVISIT lewrus01 (26/01/24): we can maybe add - -- @overload fun(ns: integer, {}): table - nvim_get_hl = 'vim.api.keyset.get_hl_info', - - nvim_get_mode = 'vim.api.keyset.get_mode', - nvim_get_namespaces = 'table', - nvim_get_option_info = 'vim.api.keyset.get_option_info', - nvim_get_option_info2 = 'vim.api.keyset.get_option_info', - nvim_parse_cmd = 'vim.api.keyset.parse_cmd', - nvim_win_get_config = 'vim.api.keyset.win_config', -} - -local LUA_API_KEYSET_OVERRIDES = { - create_autocmd = { - callback = 'string|(fun(args: vim.api.keyset.create_autocmd.callback_args): boolean?)', - }, -} - -local LUA_API_PARAM_OVERRIDES = { - nvim_create_user_command = { - command = 'string|fun(args: vim.api.keyset.create_user_command.command_args)', - }, -} - -local LUA_META_HEADER = { - '--- @meta _', - '-- THIS FILE IS GENERATED', - '-- DO NOT EDIT', - "error('Cannot require a meta file')", -} - -local LUA_API_META_HEADER = { - '--- @meta _', - '-- THIS FILE IS GENERATED', - '-- DO NOT EDIT', - "error('Cannot require a meta file')", - '', - '--- This file embeds vimdoc as the function descriptions', - '--- so ignore any doc related errors.', - '--- @diagnostic disable: undefined-doc-name,luadoc-miss-symbol', - '', - 'vim.api = {}', -} - -local LUA_OPTION_META_HEADER = { - '--- @meta _', - '-- THIS FILE IS GENERATED', - '-- DO NOT EDIT', - "error('Cannot require a meta file')", - '', - '---@class vim.bo', - '---@field [integer] vim.bo', - 'vim.bo = vim.bo', - '', - '---@class vim.wo', - '---@field [integer] vim.wo', - 'vim.wo = vim.wo', -} - -local LUA_VVAR_META_HEADER = { - '--- @meta _', - '-- THIS FILE IS GENERATED', - '-- DO NOT EDIT', - "error('Cannot require a meta file')", - '', - '--- @class vim.v', - 'vim.v = ...', -} - -local LUA_KEYWORDS = { - ['and'] = true, - ['end'] = true, - ['function'] = true, - ['or'] = true, - ['if'] = true, - ['while'] = true, - ['repeat'] = true, - ['true'] = true, - ['false'] = true, -} - -local OPTION_TYPES = { - boolean = 'boolean', - number = 'integer', - string = 'string', -} - -local API_TYPES = { - Window = 'integer', - Tabpage = 'integer', - Buffer = 'integer', - Boolean = 'boolean', - Object = 'any', - Integer = 'integer', - String = 'string', - Array = 'any[]', - LuaRef = 'function', - Dict = 'table', - Float = 'number', - HLGroupID = 'integer|string', - void = '', -} - ---- @param s string ---- @return string -local function luaescape(s) - if LUA_KEYWORDS[s] then - return s .. '_' - end - return s -end - ---- @param x string ---- @param sep? string ---- @return string[] -local function split(x, sep) - return vim.split(x, sep or '\n', { plain = true }) -end - ---- Convert an API type to Lua ---- @param t string ---- @return string -local function api_type(t) - if vim.startswith(t, '*') then - return api_type(t:sub(2)) .. '?' - end - - local as0 = t:match('^ArrayOf%((.*)%)') - if as0 then - local as = split(as0, ', ') - return api_type(as[1]) .. '[]' - end - - local d = t:match('^Dict%((.*)%)') - if d then - return 'vim.api.keyset.' .. d - end - - local d0 = t:match('^DictOf%((.*)%)') - if d0 then - return 'table' - end - - local u = t:match('^Union%((.*)%)') - if u then - local us = vim.split(u, ',%s*') - return table.concat(vim.tbl_map(api_type, us), '|') - end - - local l = t:match('^LuaRefOf%((.*)%)') - if l then - --- @type string - l = l:gsub('%s+', ' ') - --- @type string?, string? - local as, r = l:match('%((.*)%),%s*(.*)') - if not as then - --- @type string - as = assert(l:match('%((.*)%)')) - end - - local as1 = {} --- @type string[] - for a in vim.gsplit(as, ',%s') do - local a1 = vim.split(a, '%s+', { trimempty = true }) - local nm = a1[2]:gsub('%*(.*)$', '%1?') - as1[#as1 + 1] = nm .. ': ' .. api_type(a1[1]) - end - - return ('fun(%s)%s'):format(table.concat(as1, ', '), r and ': ' .. api_type(r) or '') - end - - return API_TYPES[t] or t -end - ---- @param f string ---- @param params [string,string][]|true ---- @return string -local function render_fun_sig(f, params) - local param_str --- @type string - if params == true then - param_str = '...' - else - param_str = table.concat( - vim.tbl_map( - --- @param v [string,string] - --- @return string - function(v) - return luaescape(v[1]) - end, - params - ), - ', ' - ) - end - - if LUA_KEYWORDS[f] then - return fmt("vim.fn['%s'] = function(%s) end", f, param_str) - else - return fmt('function vim.fn.%s(%s) end', f, param_str) - end -end - ---- Uniquify names ---- @param params [string,string,string][] ---- @return [string,string,string][] -local function process_params(params) - local seen = {} --- @type table - local sfx = 1 - - for _, p in ipairs(params) do - if seen[p[1]] then - p[1] = p[1] .. sfx - sfx = sfx + 1 - else - seen[p[1]] = true - end - end - - return params -end - ---- @return table -local function get_api_meta() - local ret = {} --- @type table - - local cdoc_parser = require('scripts.cdoc_parser') - - local f = 'src/nvim/api' - - local function include(fun) - if not vim.startswith(fun.name, 'nvim_') then - return false - end - if vim.tbl_contains(fun.attrs or {}, 'lua_only') then - return true - end - if vim.tbl_contains(fun.attrs or {}, 'remote_only') then - return false - end - return true - end - - --- @type table - local functions = {} - for path, ty in vim.fs.dir(f) do - if ty == 'file' then - local filename = vim.fs.joinpath(f, path) - local _, funs = cdoc_parser.parse(filename) - for _, fn in ipairs(funs) do - if include(fn) then - functions[fn.name] = fn - end - end - end - end - - for _, fun in pairs(functions) do - local deprecated = fun.deprecated_since ~= nil - - local notes = {} --- @type string[] - for _, note in ipairs(fun.notes or {}) do - notes[#notes + 1] = note.desc - end - - local sees = {} --- @type string[] - for _, see in ipairs(fun.see or {}) do - sees[#sees + 1] = see.desc - end - - local pty_overrides = LUA_API_PARAM_OVERRIDES[fun.name] or {} - - local params = {} --- @type [string,string][] - for _, p in ipairs(fun.params) do - params[#params + 1] = { - p.name, - api_type(pty_overrides[p.name] or p.type), - not deprecated and p.desc or nil, - } - end - - local r = { - signature = 'NA', - name = fun.name, - params = params, - notes = notes, - see = sees, - returns = api_type(fun.returns[1].type), - deprecated = deprecated, - } - - if not deprecated then - r.desc = fun.desc - r.returns_desc = fun.returns[1].desc - end - - ret[fun.name] = r - end - return ret -end - ---- Convert vimdoc references to markdown literals ---- Convert vimdoc codeblocks to markdown codeblocks ---- ---- Ensure code blocks have one empty line before the start fence and after the closing fence. ---- ---- @param x string ---- @param special string? ---- | 'see-api-meta' Normalize `@see` for API meta docstrings. ---- @return string -local function norm_text(x, special) - if special == 'see-api-meta' then - -- Try to guess a symbol that actually works in @see. - -- "nvim_xx()" => "vim.api.nvim_xx" - x = x:gsub([=[%|?(nvim_[^.()| ]+)%(?%)?%|?]=], 'vim.api.%1') - -- TODO: Remove backticks when LuaLS resolves: https://github.com/LuaLS/lua-language-server/issues/2889 - -- "|foo|" => "`:help foo`" - x = x:gsub([=[|([^ ]+)|]=], '`:help %1`') - end - - return ( - x:gsub('|([^ ]+)|', '`%1`') - :gsub('\n*>lua', '\n\n```lua') - :gsub('\n*>vim', '\n\n```vim') - :gsub('\n+<$', '\n```') - :gsub('\n+<\n+', '\n```\n\n') - :gsub('%s+>\n+', '\n```\n') - :gsub('\n+<%s+\n?', '\n```\n') - ) -end - ---- Generates LuaLS docstring for an API function. ---- @param _f string ---- @param fun vim.EvalFn ---- @param write fun(line: string) -local function render_api_meta(_f, fun, write) - write('') - - if vim.startswith(fun.name, 'nvim__') then - write('--- @private') - end - - if fun.deprecated then - write('--- @deprecated') - end - - local desc = fun.desc - if desc then - write(util.prefix_lines('--- ', norm_text(desc))) - end - - -- LuaLS doesn't support @note. Render @note items as a markdown list. - if fun.notes and #fun.notes > 0 then - write('--- Note:') - write(util.prefix_lines('--- ', table.concat(fun.notes, '\n'))) - write('---') - end - - for _, see in ipairs(fun.see or {}) do - write(util.prefix_lines('--- @see ', norm_text(see, 'see-api-meta'))) - end - - local param_names = {} --- @type string[] - local params = process_params(fun.params) - for _, p in ipairs(params) do - local pname, ptype, pdesc = luaescape(p[1]), p[2], p[3] - param_names[#param_names + 1] = pname - if pdesc then - local s = '--- @param ' .. pname .. ' ' .. ptype .. ' ' - local pdesc_a = split(vim.trim(norm_text(pdesc))) - write(s .. pdesc_a[1]) - for i = 2, #pdesc_a do - if not pdesc_a[i] then - break - end - write('--- ' .. pdesc_a[i]) - end - else - write('--- @param ' .. pname .. ' ' .. ptype) - end - end - - if fun.returns ~= '' then - local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or '' - local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns - write(util.prefix_lines('--- ', '@return ' .. ret .. ret_desc)) - end - local param_str = table.concat(param_names, ', ') - - write(fmt('function vim.api.%s(%s) end', fun.name, param_str)) -end - ---- @return table -local function get_api_keysets_meta() - local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) - local metadata = assert(vim.mpack.decode(mpack_f:read('*all'))) - - local ret = {} --- @type table - - --- @type {name: string, keys: string[], types: table}[] - local keysets = metadata.keysets - - for _, k in ipairs(keysets) do - local pty_overrides = LUA_API_KEYSET_OVERRIDES[k.name] or {} - local params = {} - for _, key in ipairs(k.keys) do - local pty = pty_overrides[key] or k.types[key] or 'any' - table.insert(params, { key .. '?', api_type(pty) }) - end - ret[k.name] = { - signature = 'NA', - name = k.name, - params = params, - } - end - - return ret -end - ---- Generates LuaLS docstring for an API keyset. ---- @param _f string ---- @param fun vim.EvalFn ---- @param write fun(line: string) -local function render_api_keyset_meta(_f, fun, write) - if string.sub(fun.name, 1, 1) == '_' then - return -- not exported - end - write('') - write('--- @class vim.api.keyset.' .. fun.name) - for _, p in ipairs(fun.params) do - write('--- @field ' .. p[1] .. ' ' .. p[2]) - end -end - ---- @return table -local function get_eval_meta() - return require('src/nvim/eval').funcs -end - ---- Generates LuaLS docstring for a Vimscript "eval" function. ---- @param f string ---- @param fun vim.EvalFn ---- @param write fun(line: string) -local function render_eval_meta(f, fun, write) - if fun.lua == false then - return - end - - local funname = fun.name or f - local params = process_params(fun.params) - - write('') - if fun.deprecated then - write('--- @deprecated') - end - - local desc = fun.desc - - if desc then - --- @type string - desc = desc:gsub('\n%s*\n%s*$', '\n') - for _, l in ipairs(split(desc)) do - l = l:gsub('^ ', ''):gsub('\t', ' '):gsub('@', '\\@') - write('--- ' .. l) - end - end - - for _, text in ipairs(vim.fn.reverse(fun.generics or {})) do - write(fmt('--- @generic %s', text)) - end - - local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 - - for i, param in ipairs(params) do - local pname, ptype = luaescape(param[1]), param[2] - local optional = (pname ~= '...' and i > req_args) and '?' or '' - write(fmt('--- @param %s%s %s', pname, optional, ptype)) - end - - if fun.returns ~= false then - local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or '' - write('--- @return ' .. (fun.returns or 'any') .. ret_desc) - end - - write(render_fun_sig(funname, params)) -end - ---- Generates vimdoc heading for a Vimscript "eval" function signature. ---- @param name string ---- @param name_tag boolean ---- @param fun vim.EvalFn ---- @param write fun(line: string) -local function render_sig_and_tag(name, name_tag, fun, write) - if not fun.signature then - return - end - - local tags = name_tag and { '*' .. name .. '()*' } or {} - - if fun.tags then - for _, t in ipairs(fun.tags) do - tags[#tags + 1] = '*' .. t .. '*' - end - end - - if #tags == 0 then - write(fun.signature) - return - end - - local tag = table.concat(tags, ' ') - local siglen = #fun.signature - local conceal_offset = 2 * (#tags - 1) - local tag_pad_len = math.max(1, 80 - #tag + conceal_offset) - - if siglen + #tag > 80 then - write(string.rep(' ', tag_pad_len) .. tag) - write(fun.signature) - else - write(fmt('%s%s%s', fun.signature, string.rep(' ', tag_pad_len - siglen), tag)) - end -end - ---- Generates vimdoc for a Vimscript "eval" function. ---- @param f string ---- @param fun vim.EvalFn ---- @param write fun(line: string) -local function render_eval_doc(f, fun, write) - if fun.deprecated or not fun.signature then - return - end - - render_sig_and_tag(fun.name or f, not f:find('__%d+$'), fun, write) - - if not fun.desc then - return - end - - local params = process_params(fun.params) - local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 - - local desc_l = split(vim.trim(fun.desc)) - for _, l in ipairs(desc_l) do - l = l:gsub('^ ', '') - if vim.startswith(l, '<') and not l:match('^<[^ \t]+>') then - write('<\t\t' .. l:sub(2)) - elseif l:match('^>[a-z0-9]*$') then - write(l) - else - write('\t\t' .. l) - end - end - - if #desc_l > 0 and not desc_l[#desc_l]:match('^ 0 then - write(util.md_to_vimdoc('Parameters: ~', 16, 16, TEXT_WIDTH)) - for i, param in ipairs(params) do - local pname, ptype = param[1], param[2] - local optional = (pname ~= '...' and i > req_args) and '?' or '' - local s = fmt('- %-14s (`%s%s`)', fmt('{%s}', pname), ptype, optional) - write(util.md_to_vimdoc(s, 16, 18, TEXT_WIDTH)) - end - write('') - end - - if fun.returns ~= false then - write(util.md_to_vimdoc('Return: ~', 16, 16, TEXT_WIDTH)) - local ret = ('(`%s`)'):format((fun.returns or 'any')) - ret = ret .. (fun.returns_desc and ' ' .. fun.returns_desc or '') - ret = util.md_to_vimdoc(ret, 18, 18, TEXT_WIDTH) - write(ret) - write('') - end -end - ---- @param d vim.option_defaults ---- @param vimdoc? boolean ---- @return string -local function render_option_default(d, vimdoc) - local dt --- @type integer|boolean|string|fun(): string - if d.if_false ~= nil then - dt = d.if_false - else - dt = d.if_true - end - - if vimdoc then - if d.doc then - return d.doc - end - if type(dt) == 'boolean' then - return dt and 'on' or 'off' - end - end - - if dt == '' or dt == nil or type(dt) == 'function' then - dt = d.meta - end - - local v --- @type string - if not vimdoc then - v = vim.inspect(dt) --[[@as string]] - else - v = type(dt) == 'string' and '"' .. dt .. '"' or tostring(dt) - end - - --- @type table - local envvars = { - TMPDIR = false, - VIMRUNTIME = false, - XDG_CONFIG_HOME = vim.env.HOME .. '/.local/config', - XDG_DATA_HOME = vim.env.HOME .. '/.local/share', - XDG_STATE_HOME = vim.env.HOME .. '/.local/state', - } - - for name, default in pairs(envvars) do - local value = vim.env[name] or default - if value then - v = v:gsub(vim.pesc(value), '$' .. name) - end - end - - return v -end - ---- @param _f string ---- @param opt vim.option_meta ---- @param write fun(line: string) -local function render_option_meta(_f, opt, write) - write('') - for _, l in ipairs(split(norm_text(opt.desc))) do - write('--- ' .. l) - end - - if opt.type == 'string' and not opt.list and opt.values then - local values = {} --- @type string[] - for _, e in ipairs(opt.values) do - values[#values + 1] = fmt("'%s'", e) - end - write('--- @type ' .. table.concat(values, '|')) - else - write('--- @type ' .. OPTION_TYPES[opt.type]) - end - - write('vim.o.' .. opt.full_name .. ' = ' .. render_option_default(opt.defaults)) - if opt.abbreviation then - write('vim.o.' .. opt.abbreviation .. ' = vim.o.' .. opt.full_name) - end - - for _, s in pairs { - { 'wo', 'win' }, - { 'bo', 'buf' }, - { 'go', 'global' }, - } do - local id, scope = s[1], s[2] - if vim.list_contains(opt.scope, scope) or (id == 'go' and #opt.scope > 1) then - local pfx = 'vim.' .. id .. '.' - write(pfx .. opt.full_name .. ' = vim.o.' .. opt.full_name) - if opt.abbreviation then - write(pfx .. opt.abbreviation .. ' = ' .. pfx .. opt.full_name) - end - end - end -end - ---- @param _f string ---- @param opt vim.option_meta ---- @param write fun(line: string) -local function render_vvar_meta(_f, opt, write) - write('') - - local desc = split(norm_text(opt.desc)) - while desc[#desc]:match('^%s*$') do - desc[#desc] = nil - end - - for _, l in ipairs(desc) do - write('--- ' .. l) - end - - write('--- @type ' .. (opt.type or 'any')) - - if LUA_KEYWORDS[opt.full_name] then - write("vim.v['" .. opt.full_name .. "'] = ...") - else - write('vim.v.' .. opt.full_name .. ' = ...') - end -end - ---- @param s string[] ---- @return string -local function scope_to_doc(s) - local m = { - global = 'global', - buf = 'local to buffer', - win = 'local to window', - tab = 'local to tab page', - } - - if #s == 1 then - return m[s[1]] - end - assert(s[1] == 'global') - return 'global or ' .. m[s[2]] .. (s[2] ~= 'tab' and ' |global-local|' or '') -end - --- @param o vim.option_meta --- @return string -local function scope_more_doc(o) - if - vim.list_contains({ - 'bufhidden', - 'buftype', - 'filetype', - 'modified', - 'previewwindow', - 'readonly', - 'scroll', - 'syntax', - 'winfixheight', - 'winfixwidth', - }, o.full_name) - then - return ' |local-noglobal|' - end - - return '' -end - ---- @param x string ---- @return string -local function dedent(x) - local xs = split(x) - local leading_ws = xs[1]:match('^%s*') --[[@as string]] - local leading_ws_pat = '^' .. leading_ws - - for i in ipairs(xs) do - local strip_pat = xs[i]:match(leading_ws_pat) and leading_ws_pat or '^%s*' - xs[i] = xs[i]:gsub(strip_pat, '') - end - - return table.concat(xs, '\n') -end - ---- @return table -local function get_option_meta() - local opts = require('src/nvim/options').options - local optinfo = vim.api.nvim_get_all_options_info() - local ret = {} --- @type table - for _, o in ipairs(opts) do - local is_window_option = #o.scope == 1 and o.scope[1] == 'win' - local is_option_hidden = o.immutable and not o.varname and not is_window_option - if not is_option_hidden and o.desc then - if o.full_name == 'cmdheight' then - table.insert(o.scope, 'tab') - end - local r = vim.deepcopy(o) --[[@as vim.option_meta]] - r.desc = o.desc:gsub('^ ', ''):gsub('\n ', '\n') - r.defaults = r.defaults or {} - if r.defaults.meta == nil then - r.defaults.meta = optinfo[o.full_name].default - end - ret[o.full_name] = r - end - end - return ret -end - ---- @return table -local function get_vvar_meta() - local info = require('src/nvim/vvars').vars - local ret = {} --- @type table - for name, o in pairs(info) do - o.desc = dedent(o.desc) - o.full_name = name - ret[name] = o - end - return ret -end - ---- @param opt vim.option_meta ---- @return string[] -local function build_option_tags(opt) - --- @type string[] - local tags = { opt.full_name } - - tags[#tags + 1] = opt.abbreviation - if opt.type == 'boolean' then - for i = 1, #tags do - tags[#tags + 1] = 'no' .. tags[i] - end - end - - for i, t in ipairs(tags) do - tags[i] = "'" .. t .. "'" - end - - for _, t in ipairs(opt.tags or {}) do - tags[#tags + 1] = t - end - - for i, t in ipairs(tags) do - tags[i] = '*' .. t .. '*' - end - - return tags -end - ---- @param _f string ---- @param opt vim.option_meta ---- @param write fun(line: string) -local function render_option_doc(_f, opt, write) - local tags = build_option_tags(opt) - local tag_str = table.concat(tags, ' ') - local conceal_offset = 2 * (#tags - 1) - local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) - -- local pad = string.rep(' ', 80 - #tag_str + conceal_offset) - write(tag_pad .. tag_str) - - local name_str --- @type string - if opt.abbreviation then - name_str = fmt("'%s' '%s'", opt.full_name, opt.abbreviation) - else - name_str = fmt("'%s'", opt.full_name) - end - - local otype = opt.type == 'boolean' and 'boolean' or opt.type - if opt.defaults.doc or opt.defaults.if_true ~= nil or opt.defaults.meta ~= nil then - local v = render_option_default(opt.defaults, true) - local pad = string.rep('\t', math.max(1, math.ceil((24 - #name_str) / 8))) - if opt.defaults.doc then - local deflen = #fmt('%s%s%s (', name_str, pad, otype) - --- @type string - v = v:gsub('\n', '\n' .. string.rep(' ', deflen - 2)) - end - write(fmt('%s%s%s\t(default %s)', name_str, pad, otype, v)) - else - write(fmt('%s\t%s', name_str, otype)) - end - - write('\t\t\t' .. scope_to_doc(opt.scope) .. scope_more_doc(opt)) - for _, l in ipairs(split(opt.desc)) do - if l == '<' or l:match('^<%s') then - write(l) - else - write('\t' .. l:gsub('\\<', '<')) - end - end -end - ---- @param _f string ---- @param vvar vim.option_meta ---- @param write fun(line: string) -local function render_vvar_doc(_f, vvar, write) - local name = vvar.full_name - - local tags = { 'v:' .. name, name .. '-variable' } - if vvar.tags then - vim.list_extend(tags, vvar.tags) - end - - for i, t in ipairs(tags) do - tags[i] = '*' .. t .. '*' - end - - local tag_str = table.concat(tags, ' ') - local conceal_offset = 2 * (#tags - 1) - - local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) - write(tag_pad .. tag_str) - - local desc = split(vvar.desc) - - if (#desc == 1 or #desc == 2 and desc[2]:match('^%s*$')) and #name < 10 then - -- single line - write('v:' .. name .. '\t' .. desc[1]:gsub('^%s*', '')) - write('') - else - write('v:' .. name) - for _, l in ipairs(desc) do - if l == '<' or l:match('^<%s') then - write(l) - else - write('\t\t' .. l:gsub('\\<', '<')) - end - end - end -end - ---- @class nvim.gen_eval_files.elem ---- @field path string ---- @field from? string Skip lines in path until this pattern is reached. ---- @field funcs fun(): table ---- @field render fun(f:string,obj:table,write:fun(line:string)) ---- @field header? string[] ---- @field footer? string[] - ---- @type nvim.gen_eval_files.elem[] -local CONFIG = { - { - path = 'runtime/lua/vim/_meta/vimfn.lua', - header = LUA_META_HEADER, - funcs = get_eval_meta, - render = render_eval_meta, - }, - { - path = 'runtime/lua/vim/_meta/api.lua', - header = LUA_API_META_HEADER, - funcs = get_api_meta, - render = render_api_meta, - }, - { - path = 'runtime/lua/vim/_meta/api_keysets.lua', - header = LUA_META_HEADER, - funcs = get_api_keysets_meta, - render = render_api_keyset_meta, - }, - { - path = 'runtime/doc/builtin.txt', - funcs = get_eval_meta, - render = render_eval_doc, - header = { - '*builtin.txt* Nvim', - '', - '', - '\t\t NVIM REFERENCE MANUAL', - '', - '', - 'Builtin functions\t\t*vimscript-functions* *builtin-functions*', - '', - 'For functions grouped by what they are used for see |function-list|.', - '', - '\t\t\t\t Type |gO| to see the table of contents.', - '==============================================================================', - '1. Details *builtin-function-details*', - '', - }, - footer = { - '==============================================================================', - '2. Matching a pattern in a String *string-match*', - '', - 'This is common between several functions. A regexp pattern as explained at', - '|pattern| is normally used to find a match in the buffer lines. When a', - 'pattern is used to find a match in a String, almost everything works in the', - 'same way. The difference is that a String is handled like it is one line.', - 'When it contains a "\\n" character, this is not seen as a line break for the', - 'pattern. It can be matched with a "\\n" in the pattern, or with ".". Example:', - '>vim', - '\tlet a = "aaaa\\nxxxx"', - '\techo matchstr(a, "..\\n..")', - '\t" aa', - '\t" xx', - '\techo matchstr(a, "a.x")', - '\t" a', - '\t" x', - '', - 'Don\'t forget that "^" will only match at the first character of the String and', - '"$" at the last character of the string. They don\'t match after or before a', - '"\\n".', - '', - ' vim:tw=78:ts=8:noet:ft=help:norl:', - }, - }, - { - path = 'runtime/lua/vim/_meta/options.lua', - header = LUA_OPTION_META_HEADER, - funcs = get_option_meta, - render = render_option_meta, - }, - { - path = 'runtime/doc/options.txt', - header = { '' }, - from = 'A jump table for the options with a short description can be found at |Q_op|.', - footer = { - ' vim:tw=78:ts=8:noet:ft=help:norl:', - }, - funcs = get_option_meta, - render = render_option_doc, - }, - { - path = 'runtime/lua/vim/_meta/vvars.lua', - header = LUA_VVAR_META_HEADER, - funcs = get_vvar_meta, - render = render_vvar_meta, - }, - { - path = 'runtime/doc/vvars.txt', - header = { '' }, - from = 'Type |gO| to see the table of contents.', - footer = { - ' vim:tw=78:ts=8:noet:ft=help:norl:', - }, - funcs = get_vvar_meta, - render = render_vvar_doc, - }, -} - ---- @param elem nvim.gen_eval_files.elem -local function render(elem) - print('Rendering ' .. elem.path) - local from_lines = {} --- @type string[] - local from = elem.from - if from then - for line in io.lines(elem.path) do - from_lines[#from_lines + 1] = line - if line:match(from) then - break - end - end - end - - local o = assert(io.open(elem.path, 'w')) - - --- @param l string - local function write(l) - local l1 = l:gsub('%s+$', '') - o:write(l1) - o:write('\n') - end - - for _, l in ipairs(from_lines) do - write(l) - end - - for _, l in ipairs(elem.header or {}) do - write(l) - end - - local funcs = elem.funcs() - - --- @type string[] - local fnames = vim.tbl_keys(funcs) - table.sort(fnames) - - for _, f in ipairs(fnames) do - elem.render(f, funcs[f], write) - end - - for _, l in ipairs(elem.footer or {}) do - write(l) - end - - o:close() -end - -local function main() - for _, c in ipairs(CONFIG) do - render(c) - end -end - -main() diff --git a/scripts/gen_filetype.lua b/scripts/gen_filetype.lua deleted file mode 100644 index 18b53f1ea4..0000000000 --- a/scripts/gen_filetype.lua +++ /dev/null @@ -1,209 +0,0 @@ -local do_not_run = true -if do_not_run then - print([[ - This script was used to bootstrap the filetype patterns in runtime/lua/vim/filetype.lua. It - should no longer be used except for testing purposes. New filetypes, or changes to existing - filetypes, should be ported manually as part of the vim-patch process. - ]]) - return -end - -local filetype_vim = 'runtime/filetype.vim' -local filetype_lua = 'runtime/lua/vim/filetype.lua' - -local keywords = { - ['for'] = true, - ['or'] = true, - ['and'] = true, - ['end'] = true, - ['do'] = true, - ['if'] = true, - ['while'] = true, - ['repeat'] = true, -} - -local sections = { - extension = { str = {}, func = {} }, - filename = { str = {}, func = {} }, - pattern = { str = {}, func = {} }, -} - -local specialchars = '%*%?\\%$%[%]%{%}' - -local function add_pattern(pat, ft) - local ok = true - - -- Patterns that start or end with { or } confuse splitting on commas and make parsing harder, so just skip those - if not string.find(pat, '^%{') and not string.find(pat, '%}$') then - for part in string.gmatch(pat, '[^,]+') do - if not string.find(part, '[' .. specialchars .. ']') then - if type(ft) == 'string' then - sections.filename.str[part] = ft - else - sections.filename.func[part] = ft - end - elseif string.match(part, '^%*%.[^%./' .. specialchars .. ']+$') then - if type(ft) == 'string' then - sections.extension.str[part:sub(3)] = ft - else - sections.extension.func[part:sub(3)] = ft - end - else - if string.match(part, '^%*/[^' .. specialchars .. ']+$') then - -- For patterns matching */some/pattern we want to easily match files - -- with path /some/pattern, so include those in filename detection - if type(ft) == 'string' then - sections.filename.str[part:sub(2)] = ft - else - sections.filename.func[part:sub(2)] = ft - end - end - - if string.find(part, '^[%w-_.*?%[%]/]+$') then - local p = part:gsub('%.', '%%.'):gsub('%*', '.*'):gsub('%?', '.') - -- Insert into array to maintain order rather than setting - -- key-value directly - if type(ft) == 'string' then - sections.pattern.str[p] = ft - else - sections.pattern.func[p] = ft - end - else - ok = false - end - end - end - end - - return ok -end - -local function parse_line(line) - local pat, ft - pat, ft = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)') - if pat then - return add_pattern(pat, ft) - else - local func - pat, func = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)') - if pat then - return add_pattern(pat, function() - return func - end) - end - end -end - -local unparsed = {} -local full_line -for line in io.lines(filetype_vim) do - local cont = string.match(line, '^%s*\\%s*(.*)$') - if cont then - full_line = full_line .. ' ' .. cont - else - if full_line then - if not parse_line(full_line) and string.find(full_line, '^%s*au%a* Buf') then - table.insert(unparsed, full_line) - end - end - full_line = line - end -end - -if #unparsed > 0 then - print('Failed to parse the following patterns:') - for _, v in ipairs(unparsed) do - print(v) - end -end - -local function add_item(indent, key, ft) - if type(ft) == 'string' then - if string.find(key, '%A') or keywords[key] then - key = string.format('["%s"]', key) - end - return string.format([[%s%s = "%s",]], indent, key, ft) - elseif type(ft) == 'function' then - local func = ft() - if string.find(key, '%A') or keywords[key] then - key = string.format('["%s"]', key) - end - -- Right now only a single argument is supported, which covers - -- everything in filetype.vim as of this writing - local arg = string.match(func, '%((.*)%)$') - func = string.gsub(func, '%(.*$', '') - if arg == '' then - -- Function with no arguments, call the function directly - return string.format([[%s%s = function() vim.fn["%s"]() end,]], indent, key, func) - elseif string.match(arg, [[^(["']).*%1$]]) then - -- String argument - if func == 's:StarSetf' then - return string.format([[%s%s = starsetf(%s),]], indent, key, arg) - else - return string.format([[%s%s = function() vim.fn["%s"](%s) end,]], indent, key, func, arg) - end - elseif string.find(arg, '%(') then - -- Function argument - return string.format( - [[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]], - indent, - key, - func, - arg - ) - else - assert(false, arg) - end - end -end - -do - local lines = {} - local start = false - for line in io.lines(filetype_lua) do - if line:match('^%s+-- END [A-Z]+$') then - start = false - end - - if not start then - table.insert(lines, line) - end - - local indent, section = line:match('^(%s+)-- BEGIN ([A-Z]+)$') - if section then - start = true - local t = sections[string.lower(section)] - - local sorted = {} - for k, v in pairs(t.str) do - table.insert(sorted, { [k] = v }) - end - - table.sort(sorted, function(a, b) - return a[next(a)] < b[next(b)] - end) - - for _, v in ipairs(sorted) do - local k = next(v) - table.insert(lines, add_item(indent, k, v[k])) - end - - sorted = {} - for k, v in pairs(t.func) do - table.insert(sorted, { [k] = v }) - end - - table.sort(sorted, function(a, b) - return next(a) < next(b) - end) - - for _, v in ipairs(sorted) do - local k = next(v) - table.insert(lines, add_item(indent, k, v[k])) - end - end - end - local f = io.open(filetype_lua, 'w') - f:write(table.concat(lines, '\n') .. '\n') - f:close() -end diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua deleted file mode 100644 index 53a65fd65f..0000000000 --- a/scripts/gen_help_html.lua +++ /dev/null @@ -1,1491 +0,0 @@ ---- Converts Nvim :help files to HTML. Validates |tag| links and document syntax (parser errors). --- --- USAGE (For CI/local testing purposes): Simply `make lintdoc` or `scripts/lintdoc.lua`, which --- basically does the following: --- 1. :helptags ALL --- 2. nvim -V1 -es +"lua require('scripts.gen_help_html').run_validate()" +q --- 3. nvim -V1 -es +"lua require('scripts.gen_help_html').test_gen()" +q --- --- USAGE (GENERATE HTML): --- 1. `:helptags ALL` first; this script depends on vim.fn.taglist(). --- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./runtime/doc', 'target/dir/')" +q --- - Read the docstring at gen(). --- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 --- 4. Visit http://localhost:4000/…/help.txt.html --- --- USAGE (VALIDATE): --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate('./runtime/doc')" +q --- - validate() is 10x faster than gen(), so it is used in CI. --- --- SELF-TEST MODE: --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +q --- --- NOTES: --- * This script is used by the automation repo: https://github.com/neovim/doc --- * :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). --- * gen() and validate() are the primary (programmatic) entrypoints. validate() only exists --- because gen() is too slow (~1 min) to run in per-commit CI. --- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. --- * visit_validate() is the core function used by validate(). --- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. - -local tagmap = nil ---@type table -local helpfiles = nil ---@type string[] -local invalid_links = {} ---@type table -local invalid_urls = {} ---@type table -local invalid_spelling = {} ---@type table> -local spell_dict = { - Neovim = 'Nvim', - NeoVim = 'Nvim', - neovim = 'Nvim', - lua = 'Lua', - VimL = 'Vimscript', - vimL = 'Vimscript', - viml = 'Vimscript', - ['tree-sitter'] = 'treesitter', - ['Tree-sitter'] = 'Treesitter', -} ---- specify the list of keywords to ignore (i.e. allow), or true to disable spell check completely. ---- @type table -local spell_ignore_files = { - ['credits.txt'] = { 'Neovim' }, - ['news.txt'] = { 'tree-sitter' }, -- in news, may refer to the upstream "tree-sitter" library - ['news-0.10.txt'] = { 'tree-sitter' }, -} -local language = nil - -local M = {} - --- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). --- All other files are "legacy" files which require fixed-width layout. -local new_layout = { - ['api.txt'] = true, - ['lsp.txt'] = true, - ['channel.txt'] = true, - ['deprecated.txt'] = true, - ['develop.txt'] = true, - ['dev_style.txt'] = true, - ['dev_theme.txt'] = true, - ['dev_tools.txt'] = true, - ['dev_vimpatch.txt'] = true, - ['editorconfig.txt'] = true, - ['faq.txt'] = true, - ['gui.txt'] = true, - ['intro.txt'] = true, - ['lua.txt'] = true, - ['luaref.txt'] = true, - ['news.txt'] = true, - ['news-0.9.txt'] = true, - ['news-0.10.txt'] = true, - ['nvim.txt'] = true, - ['provider.txt'] = true, - ['tui.txt'] = true, - ['ui.txt'] = true, - ['vim_diff.txt'] = true, -} - --- Map of new:old pages, to redirect renamed pages. -local redirects = { - ['credits'] = 'backers', - ['tui'] = 'term', - ['terminal'] = 'nvim_terminal_emulator', -} - --- TODO: These known invalid |links| require an update to the relevant docs. -local exclude_invalid = { - ["'string'"] = 'eval.txt', - Query = 'treesitter.txt', - matchit = 'vim_diff.txt', - ['set!'] = 'treesitter.txt', -} - --- False-positive "invalid URLs". -local exclude_invalid_urls = { - ['http://'] = 'usr_23.txt', - ['http://.'] = 'usr_23.txt', - ['http://aspell.net/man-html/Affix-Compression.html'] = 'spell.txt', - ['http://aspell.net/man-html/Phonetic-Code.html'] = 'spell.txt', - ['http://canna.sourceforge.jp/'] = 'mbyte.txt', - ['http://gnuada.sourceforge.net'] = 'ft_ada.txt', - ['http://lua-users.org/wiki/StringLibraryTutorial'] = 'lua.txt', - ['http://michael.toren.net/code/'] = 'pi_tar.txt', - ['http://papp.plan9.de'] = 'syntax.txt', - ['http://wiki.services.openoffice.org/wiki/Dictionaries'] = 'spell.txt', - ['http://www.adapower.com'] = 'ft_ada.txt', - ['http://www.jclark.com/'] = 'quickfix.txt', - ['http://oldblog.antirez.com/post/redis-and-scripting.html'] = 'faq.txt', -} - --- Deprecated, brain-damaged files that I don't care about. -local ignore_errors = { - ['pi_netrw.txt'] = true, - ['credits.txt'] = true, -} - -local function tofile(fname, text) - local f = io.open(fname, 'w') - if not f then - error(('failed to write: %s'):format(f)) - else - f:write(text) - f:close() - end -end - ----@type fun(s: string): string -local function html_esc(s) - return (s:gsub('&', '&'):gsub('<', '<'):gsub('>', '>')) -end - -local function url_encode(s) - -- Credit: tpope / vim-unimpaired - -- NOTE: these chars intentionally *not* escaped: ' ( ) - return vim.fn.substitute( - vim.fn.iconv(s, 'latin1', 'utf-8'), - [=[[^A-Za-z0-9()'_.~-]]=], - [=[\="%".printf("%02X",char2nr(submatch(0)))]=], - 'g' - ) -end - -local function expandtabs(s) - return s:gsub('\t', (' '):rep(8)) --[[ @as string ]] -end - -local function to_titlecase(s) - local text = '' - for w in vim.gsplit(s, '[ \t]+') do - text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2)) - end - return text -end - -local function to_heading_tag(text) - -- Prepend "_" to avoid conflicts with actual :help tags. - return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown' -end - -local function basename_noext(f) - return vim.fs.basename(f:gsub('%.txt', '')) -end - -local function is_blank(s) - return not not s:find([[^[\t ]*$]]) -end - ----@type fun(s: string, dir?:0|1|2): string -local function trim(s, dir) - return vim.fn.trim(s, '\r\t\n ', dir or 0) -end - ---- Removes common punctuation from URLs. ---- ---- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc ---- ---- @param url string ---- @return string, string (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. -local function fix_url(url) - local removed_chars = '' - local fixed_url = url - -- Remove up to one of each char from end of the URL, in this order. - for _, c in ipairs({ '.', ')' }) do - if fixed_url:sub(-1) == c then - removed_chars = c .. removed_chars - fixed_url = fixed_url:sub(1, -2) - end - end - return fixed_url, removed_chars -end - ---- Checks if a given line is a "noise" line that doesn't look good in HTML form. -local function is_noise(line, noise_lines) - if - -- First line is always noise. - (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0) - or line:find('Type .*gO.* to see the table of contents') - -- Title line of traditional :help pages. - -- Example: "NVIM REFERENCE MANUAL by ..." - or line:find([[^%s*N?VIM[ \t]*REFERENCE[ \t]*MANUAL]]) - -- First line of traditional :help pages. - -- Example: "*api.txt* Nvim" - or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') - -- modeline - -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" - or line:find('^%s*vim?%:.*ft=help') - or line:find('^%s*vim?%:.*filetype=help') - or line:find('[*>]local%-additions[*<]') - then - -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) - table.insert(noise_lines or {}, line) - return true - end - return false -end - ---- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. ---- @return string -local function get_bug_url_vimdoc(fname, to_fname, sample_text) - local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ( - 'https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' - .. vim.fs.basename(fname) - .. '+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' - .. this_url - .. '%0D%0DContext%3A%0D%0D%60%60%60%0D' - .. url_encode(sample_text) - .. '%0D%60%60%60' - ) - return bug_url -end - ---- Creates a github issue URL at neovim/neovim with prefilled content. ---- @return string -local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) - local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ( - 'https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' - .. vim.fs.basename(fname) - .. '+&body=%60gen_help_html.lua%60+problem+at%3A+' - .. this_url - .. '%0D' - .. (token_name and '+unhandled+token%3A+%60' .. token_name .. '%60' or '') - .. '%0DContext%3A%0D%0D%60%60%60%0D' - .. url_encode(sample_text) - .. '%0D%60%60%60' - ) - return bug_url -end - ---- Gets a "foo.html" name from a "foo.txt" helpfile name. -local function get_helppage(f) - if not f then - return nil - end - -- Special case: help.txt is the "main landing page" of :help files, not index.txt. - if f == 'index.txt' then - return 'vimindex.html' - elseif f == 'help.txt' then - return 'index.html' - end - - return (f:gsub('%.txt$', '')) .. '.html' -end - ---- Counts leading spaces (tab=8) to decide the indent size of multiline text. ---- ---- Blank lines (empty or whitespace-only) are ignored. -local function get_indent(s) - local min_indent = nil - for line in vim.gsplit(s, '\n') do - if line and not is_blank(line) then - local ws = expandtabs(line:match('^%s+') or '') - min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent - end - end - return min_indent or 0 -end - ---- Removes the common indent level, after expanding tabs to 8 spaces. -local function trim_indent(s) - local indent_size = get_indent(s) - local trimmed = '' - for line in vim.gsplit(s, '\n') do - line = expandtabs(line) - trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1)) - end - return trimmed:sub(1, -2) -end - ---- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. ----@param node TSNode ----@param bufnr integer ----@param offset integer -local function getbuflinestr(node, bufnr, offset) - local line1, _, line2, _ = node:range() - line1 = line1 - offset - line2 = line2 + offset - local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1) - return table.concat(lines, '\n') -end - ---- Gets the whitespace just before `node` from the raw buffer text. ---- Needed for preformatted `old` lines. ----@param node TSNode ----@param bufnr integer ----@return string -local function getws(node, bufnr) - local line1, c1, line2, _ = node:range() - ---@type string - local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] - local text_before = raw:sub(1, c1) - local leading_ws = text_before:match('%s+$') or '' - return leading_ws -end - -local function get_tagname(node, bufnr) - local text = vim.treesitter.get_node_text(node, bufnr) - local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') - and ("'%s'"):format(text) - or text - local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" - local helppage = get_helppage(helpfile) -- "api.html" - return helppage, tag -end - ---- Returns true if the given invalid tagname is a false positive. -local function ignore_invalid(s) - return not not ( - exclude_invalid[s] - -- Strings like |~/====| appear in various places and the parser thinks they are links, but they - -- are just table borders. - or s:find('===') - or s:find('%-%-%-') - ) -end - -local function ignore_parse_error(fname, s) - if ignore_errors[vim.fs.basename(fname)] then - return true - end - -- Ignore parse errors for unclosed tag. - -- This is common in vimdocs and is treated as plaintext by :help. - return s:find("^[`'|*]") -end - ----@param node TSNode -local function has_ancestor(node, ancestor_name) - local p = node ---@type TSNode? - while p do - p = p:parent() - if not p or p:type() == 'help_file' then - break - elseif p:type() == ancestor_name then - return true - end - end - return false -end - ---- Gets the first matching child node matching `name`. ----@param node TSNode -local function first(node, name) - for c, _ in node:iter_children() do - if c:named() and c:type() == name then - return c - end - end - return nil -end - -local function validate_link(node, bufnr, fname) - local helppage, tagname = get_tagname(node:child(1), bufnr) - local ignored = false - if not tagmap[tagname] then - ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname) - if not ignored then - invalid_links[tagname] = vim.fs.basename(fname) - end - end - return helppage, tagname, ignored -end - ---- TODO: port the logic from scripts/check_urls.vim -local function validate_url(text, fname) - local ignored = false - if ignore_errors[vim.fs.basename(fname)] then - ignored = true - elseif text:find('http%:') and not exclude_invalid_urls[text] then - invalid_urls[text] = vim.fs.basename(fname) - end - return ignored -end - ---- Traverses the tree at `root` and checks that |tag| links point to valid helptags. ----@param root TSNode ----@param level integer ----@param lang_tree TSTree ----@param opt table ----@param stats table -local function visit_validate(root, level, lang_tree, opt, stats) - level = level or 0 - local node_name = (root.named and root:named()) and root:type() or nil - -- Parent kind (string). - local parent = root:parent() and root:parent():type() or nil - local toplevel = level < 1 - local function node_text(node) - return vim.treesitter.get_node_text(node or root, opt.buf) - end - local text = trim(node_text()) - - if root:child_count() > 0 then - for node, _ in root:iter_children() do - if node:named() then - visit_validate(node, level + 1, lang_tree, opt, stats) - end - end - end - - if node_name == 'ERROR' then - if ignore_parse_error(opt.fname, text) then - return - end - -- Store the raw text to give context to the error report. - local sample_text = not toplevel and getbuflinestr(root, opt.buf, 0) or '[top level!]' - -- Flatten the sample text to a single, truncated line. - sample_text = vim.trim(sample_text):gsub('[\t\n]', ' '):sub(1, 80) - table.insert(stats.parse_errors, sample_text) - elseif - (node_name == 'word' or node_name == 'uppercase_name') - and (not vim.tbl_contains({ 'codespan', 'taglink', 'tag' }, parent)) - then - local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. - local fname_basename = assert(vim.fs.basename(opt.fname)) - if spell_dict[text_nopunct] then - local should_ignore = ( - spell_ignore_files[fname_basename] == true - or vim.tbl_contains( - (spell_ignore_files[fname_basename] or {}) --[[ @as string[] ]], - text_nopunct - ) - ) - if not should_ignore then - invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} - invalid_spelling[text_nopunct][fname_basename] = node_text(root:parent()) - end - end - elseif node_name == 'url' then - local fixed_url, _ = fix_url(trim(text)) - validate_url(fixed_url, opt.fname) - elseif node_name == 'taglink' or node_name == 'optionlink' then - local _, _, _ = validate_link(root, opt.buf, opt.fname) - end -end - --- Fix tab alignment issues caused by concealed characters like |, `, * in tags --- and code blocks. ----@param text string ----@param next_node_text string -local function fix_tab_after_conceal(text, next_node_text) - -- Vim tabs take into account the two concealed characters even though they - -- are invisible, so we need to add back in the two spaces if this is - -- followed by a tab to make the tab alignment to match Vim's behavior. - if string.sub(next_node_text, 1, 1) == '\t' then - text = text .. ' ' - end - return text -end - ----@class (exact) nvim.gen_help_html.heading ----@field name string ----@field subheadings nvim.gen_help_html.heading[] ----@field tag string - --- Generates HTML from node `root` recursively. ----@param root TSNode ----@param level integer ----@param lang_tree TSTree ----@param headings nvim.gen_help_html.heading[] ----@param opt table ----@param stats table -local function visit_node(root, level, lang_tree, headings, opt, stats) - level = level or 0 - - local node_name = (root.named and root:named()) and root:type() or nil - -- Previous sibling kind (string). - local prev = root:prev_sibling() - and (root:prev_sibling().named and root:prev_sibling():named()) - and root:prev_sibling():type() - or nil - -- Next sibling kind (string). - local next_ = root:next_sibling() - and (root:next_sibling().named and root:next_sibling():named()) - and root:next_sibling():type() - or nil - -- Parent kind (string). - local parent = root:parent() and root:parent():type() or nil - -- Gets leading whitespace of `node`. - local function ws(node) - node = node or root - local ws_ = getws(node, opt.buf) - -- XXX: first node of a (line) includes whitespace, even after - -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ? - if ws_ == '' then - ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or '' - end - return ws_ - end - local function node_text(node, ws_) - node = node or root - ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or '' - return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) - end - - local text = '' - local trimmed ---@type string - if root:named_child_count() == 0 or node_name == 'ERROR' then - text = node_text() - trimmed = html_esc(trim(text)) - text = html_esc(text) - else - -- Process children and join them with whitespace. - for node, _ in root:iter_children() do - if node:named() then - local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) - text = string.format('%s%s', text, r) - end - end - trimmed = trim(text) - end - - if node_name == 'help_file' then -- root node - return text - elseif node_name == 'url' then - local fixed_url, removed_chars = fix_url(trimmed) - return ('%s%s%s'):format(ws(), fixed_url, fixed_url, removed_chars) - elseif node_name == 'word' or node_name == 'uppercase_name' then - return text - elseif node_name == 'note' then - return ('%s'):format(text) - elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then - if is_noise(text, stats.noise_lines) then - return '' -- Discard common "noise" lines. - end - -- Remove tags from ToC text. - local heading_node = first(root, 'heading') - local hname = trim(node_text(heading_node):gsub('%*.*%*', '')) - if not heading_node or hname == '' then - return '' -- Spurious "===" or "---" in the help doc. - end - - -- Generate an anchor id from the heading text. - local tagname = to_heading_tag(hname) - if node_name == 'h1' or #headings == 0 then - ---@type nvim.gen_help_html.heading - local heading = { name = hname, subheadings = {}, tag = tagname } - headings[#headings + 1] = heading - else - table.insert( - headings[#headings].subheadings, - { name = hname, subheadings = {}, tag = tagname } - ) - end - local el = node_name == 'h1' and 'h2' or 'h3' - return ('<%s id="%s" class="help-heading">%s\n'):format(el, tagname, trimmed, el) - elseif node_name == 'heading' then - return trimmed - elseif node_name == 'column_heading' or node_name == 'column_name' then - if root:has_error() then - return text - end - return ('
%s
'):format(text) - elseif node_name == 'block' then - if is_blank(text) then - return '' - end - if opt.old then - -- XXX: Treat "old" docs as preformatted: they use indentation for layout. - -- Trim trailing newlines to avoid too much whitespace between divs. - return ('
%s
\n'):format(trim(text, 2)) - end - return string.format('
\n%s\n
\n', text) - elseif node_name == 'line' then - if - (parent ~= 'codeblock' or parent ~= 'code') - and (is_blank(text) or is_noise(text, stats.noise_lines)) - then - return '' -- Discard common "noise" lines. - end - -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. - local div = opt.old - and root:child(0) - and vim.list_contains({ 'column_heading', 'h1', 'h2', 'h3' }, root:child(0):type()) - return string.format('%s%s', div and trim(text) or text, div and '' or '\n') - elseif node_name == 'line_li' then - local sib = root:prev_sibling() - local prev_li = sib and sib:type() == 'line_li' - - if not prev_li then - opt.indent = 1 - else - -- The previous listitem _sibling_ is _logically_ the _parent_ if it is indented less. - local parent_indent = get_indent(node_text(sib)) - local this_indent = get_indent(node_text()) - if this_indent > parent_indent then - opt.indent = opt.indent + 1 - elseif this_indent < parent_indent then - opt.indent = math.max(1, opt.indent - 1) - end - end - local margin = opt.indent == 1 and '' or ('margin-left: %drem;'):format((1.5 * opt.indent)) - - return string.format('
%s
', margin, text) - elseif node_name == 'taglink' or node_name == 'optionlink' then - local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname) - if ignored then - return text - end - local s = ('%s%s'):format( - ws(), - helppage, - url_encode(tagname), - html_esc(tagname) - ) - if opt.old and node_name == 'taglink' then - s = fix_tab_after_conceal(s, node_text(root:next_sibling())) - end - return s - elseif vim.list_contains({ 'codespan', 'keycode' }, node_name) then - if root:has_error() then - return text - end - local s = ('%s%s'):format(ws(), trimmed) - if opt.old and node_name == 'codespan' then - s = fix_tab_after_conceal(s, node_text(root:next_sibling())) - end - return s - elseif node_name == 'argument' then - return ('%s{%s}'):format(ws(), text) - elseif node_name == 'codeblock' then - return text - elseif node_name == 'language' then - language = node_text(root) - return '' - elseif node_name == 'code' then -- Highlighted codeblock (child). - if is_blank(text) then - return '' - end - local code ---@type string - if language then - code = ('
%s
'):format( - language, - trim(trim_indent(text), 2) - ) - language = nil - else - code = ('
%s
'):format(trim(trim_indent(text), 2)) - end - return code - elseif node_name == 'tag' then -- anchor, h4 pseudo-heading - if root:has_error() then - return text - end - local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent) - local h4 = not in_heading and not next_ and get_indent(node_text()) > 8 -- h4 pseudo-heading - local cssclass = h4 and 'help-tag-right' or 'help-tag' - local tagname = node_text(root:child(1), false) - if vim.tbl_count(stats.first_tags) < 2 then - -- Force the first 2 tags in the doc to be anchored at the main heading. - table.insert(stats.first_tags, tagname) - return '' - end - local el = 'span' - local encoded_tagname = url_encode(tagname) - local s = ('%s<%s id="%s" class="%s">%s'):format( - ws(), - el, - encoded_tagname, - cssclass, - encoded_tagname, - trimmed, - el - ) - if opt.old then - s = fix_tab_after_conceal(s, node_text(root:next_sibling())) - end - - if in_heading and prev ~= 'tag' then - -- Start the container for tags in a heading. - -- This makes "justify-content:space-between" right-align the tags. - --

foo bartag1 tag2

- return string.format('%s', s) - elseif in_heading and next_ == nil then - -- End the container for tags in a heading. - return string.format('%s', s) - end - return s .. (h4 and '
' or '') -- HACK:
avoids h4 pseudo-heading mushing with text. - elseif node_name == 'delimiter' or node_name == 'modeline' then - return '' - elseif node_name == 'ERROR' then - if ignore_parse_error(opt.fname, trimmed) then - return text - end - - -- Store the raw text to give context to the bug report. - local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' - table.insert(stats.parse_errors, sample_text) - return ('%s'):format( - get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), - trimmed - ) - else -- Unknown token. - local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' - return ('%s'):format( - node_name, - get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), - trimmed - ), - ('unknown-token:"%s"'):format(node_name) - end -end - ---- @param dir string e.g. '$VIMRUNTIME/doc' ---- @param include string[]|nil ---- @return string[] -local function get_helpfiles(dir, include) - local rv = {} - for f, type in vim.fs.dir(dir) do - if - vim.endswith(f, '.txt') - and type == 'file' - and (not include or vim.list_contains(include, f)) - then - local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') - table.insert(rv, fullpath) - end - end - return rv -end - ---- Populates the helptags map. -local function get_helptags(help_dir) - local m = {} - -- Load a random help file to convince taglist() to do its job. - vim.cmd(string.format('split %s/api.txt', help_dir)) - vim.cmd('lcd %:p:h') - for _, item in ipairs(vim.fn.taglist('.*')) do - if vim.endswith(item.filename, '.txt') then - m[item.name] = item.filename - end - end - vim.cmd('q!') - return m -end - ---- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system. -local function ensure_runtimepath() - if not vim.o.runtimepath:find('build/lib/nvim/') then - vim.cmd [[set runtimepath^=./build/lib/nvim/]] - end -end - ---- Opens `fname` (or `text`, if given) in a buffer and gets a treesitter parser for the buffer contents. ---- ---- @param fname string :help file to parse ---- @param text string? :help file contents ---- @param parser_path string? path to non-default vimdoc.so ---- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr) -local function parse_buf(fname, text, parser_path) - local buf ---@type integer - if text then - vim.cmd('split new') -- Text contents. - vim.api.nvim_put(vim.split(text, '\n'), '', false, false) - vim.cmd('setfiletype help') - -- vim.treesitter.language.add('vimdoc') - buf = vim.api.nvim_get_current_buf() - elseif type(fname) == 'string' then - vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename. - buf = vim.api.nvim_get_current_buf() - else - -- Left for debugging - ---@diagnostic disable-next-line: no-unknown - buf = fname - vim.cmd('sbuffer ' .. tostring(fname)) -- Buffer number. - end - if parser_path then - vim.treesitter.language.add('vimdoc', { path = parser_path }) - end - local lang_tree = assert(vim.treesitter.get_parser(buf, nil, { error = false })) - return lang_tree, buf -end - ---- Validates one :help file `fname`: ---- - checks that |tag| links point to valid helptags. ---- - recursively counts parse errors ("ERROR" nodes) ---- ---- @param fname string help file to validate ---- @param parser_path string? path to non-default vimdoc.so ---- @return { invalid_links: number, parse_errors: string[] } -local function validate_one(fname, parser_path) - local stats = { - parse_errors = {}, - } - local lang_tree, buf = parse_buf(fname, nil, parser_path) - for _, tree in ipairs(lang_tree:trees()) do - visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats) - end - lang_tree:destroy() - vim.cmd.close() - return stats -end - ---- Generates HTML from one :help file `fname` and writes the result to `to_fname`. ---- ---- @param fname string Source :help file. ---- @param text string|nil Source :help file contents, or nil to read `fname`. ---- @param to_fname string Destination .html file ---- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) ---- @param parser_path string? path to non-default vimdoc.so ---- ---- @return string html ---- @return table stats -local function gen_one(fname, text, to_fname, old, commit, parser_path) - local stats = { - noise_lines = {}, - parse_errors = {}, - first_tags = {}, -- Track the first few tags in doc. - } - local lang_tree, buf = parse_buf(fname, text, parser_path) - ---@type nvim.gen_help_html.heading[] - local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. - local title = to_titlecase(basename_noext(fname)) - - local html = ([[ - - - - - - - - - - - - - - - - - - - - %s - Neovim docs - - - ]]):format(title) - - local logo_svg = [[ - - Neovim - - - - - - - - - - - - - - - - - - - - - - - - - - ]] - - local main = '' - for _, tree in ipairs(lang_tree:trees()) do - main = main - .. ( - visit_node( - tree:root(), - 0, - tree, - headings, - { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1 }, - stats - ) - ) - end - - main = ([[ -
- -
- -
-
-

%s

 -

- - Nvim :help pages, generated - from source - using the tree-sitter-vimdoc parser. - -

-
- %s -
- ]]):format( - logo_svg, - stats.first_tags[1] or '', - stats.first_tags[2] or '', - stats.first_tags[2] or '', - title, - vim.fs.basename(fname), - main - ) - - ---@type string - local toc = [[ -
- - - -
- ]] - - local n = 0 -- Count of all headings + subheadings. - for _, h1 in ipairs(headings) do - n = n + 1 + #h1.subheadings - end - for _, h1 in ipairs(headings) do - ---@type string - toc = toc .. ('
%s\n'):format(h1.tag, h1.name) - if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. - for _, h2 in ipairs(h1.subheadings) do - toc = toc - .. ('
%s
\n'):format(h2.tag, h2.name) - end - end - toc = toc .. '
' - end - toc = toc .. '
\n' - - local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil) - local bug_link = string.format('(report docs bug...)', bug_url) - - local footer = ([[ -
-
-
- Generated at %s from %s -
-
- parse_errors: %d %s | noise_lines: %d -
-
- - - - - -
- ]]):format( - os.date('%Y-%m-%d %H:%M'), - commit, - commit:sub(1, 7), - #stats.parse_errors, - bug_link, - html_esc(table.concat(stats.noise_lines, '\n')), - #stats.noise_lines - ) - - html = ('%s%s%s
\n%s\n\n'):format(html, main, toc, footer) - vim.cmd('q!') - lang_tree:destroy() - return html, stats -end - -local function gen_css(fname) - local css = [[ - :root { - --code-color: #004b4b; - --tag-color: #095943; - } - @media (prefers-color-scheme: dark) { - :root { - --code-color: #00c243; - --tag-color: #00b7b7; - } - } - @media (min-width: 40em) { - .toc { - position: fixed; - left: 67%; - } - .golden-grid { - display: grid; - grid-template-columns: 65% auto; - grid-gap: 1em; - } - } - @media (max-width: 40em) { - .golden-grid { - /* Disable grid for narrow viewport (mobile phone). */ - display: block; - } - } - .toc { - /* max-width: 12rem; */ - height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ - overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ - } - .toc > div { - text-overflow: ellipsis; - overflow: hidden; - white-space: nowrap; - } - html { - scroll-behavior: auto; - } - body { - font-size: 18px; - line-height: 1.5; - } - h1, h2, h3, h4, h5 { - font-family: sans-serif; - border-bottom: 1px solid var(--tag-color); /*rgba(0, 0, 0, .9);*/ - } - h3, h4, h5 { - border-bottom-style: dashed; - } - .help-column_heading { - color: var(--code-color); - } - .help-body { - padding-bottom: 2em; - } - .help-line { - /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ - } - .help-li { - white-space: normal; - display: list-item; - margin-left: 1.5rem; /* padding-left: 1rem; */ - } - .help-para { - padding-top: 10px; - padding-bottom: 10px; - } - - .old-help-para { - padding-top: 10px; - padding-bottom: 10px; - /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */ - tab-size: 8; - white-space: pre-wrap; - font-size: 16px; - font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; - word-wrap: break-word; - } - .old-help-para pre, .old-help-para pre:hover { - /* Text following 
 is already visually separated by the linebreak. */
-      margin-bottom: 0;
-      /* Long lines that exceed the textwidth should not be wrapped (no "pre-wrap").
-         Since text may overflow horizontally, we make the contents to be scrollable
-         (only if necessary) to prevent overlapping with the navigation bar at the right. */
-      white-space: pre;
-      overflow-x: auto;
-    }
-
-    /* TODO: should this rule be deleted? help tags are rendered as  or , not  */
-    a.help-tag, a.help-tag:focus, a.help-tag:hover {
-      color: inherit;
-      text-decoration: none;
-    }
-    .help-tag {
-      color: var(--tag-color);
-    }
-    /* Tag pseudo-header common in :help docs. */
-    .help-tag-right {
-      color: var(--tag-color);
-      margin-left: auto;
-      margin-right: 0;
-      float: right;
-      display: block;
-    }
-    .help-tag a,
-    .help-tag-right a {
-      color: inherit;
-    }
-    .help-tag a:not(:hover),
-    .help-tag-right a:not(:hover) {
-      text-decoration: none;
-    }
-    h1 .help-tag, h2 .help-tag, h3 .help-tag {
-      font-size: smaller;
-    }
-    .help-heading {
-      white-space: normal;
-      display: flex;
-      flex-flow: row wrap;
-      justify-content: space-between;
-      gap: 0 15px;
-    }
-    /* The (right-aligned) "tags" part of a section heading. */
-    .help-heading-tags {
-      margin-right: 10px;
-    }
-    .help-toc-h1 {
-    }
-    .help-toc-h2 {
-      margin-left: 1em;
-    }
-    .parse-error {
-      background-color: red;
-    }
-    .unknown-token {
-      color: black;
-      background-color: yellow;
-    }
-    code {
-      color: var(--code-color);
-      font-size: 16px;
-    }
-    pre {
-      /* Tabs are used in codeblocks only for indentation, not alignment, so we can aggressively shrink them. */
-      tab-size: 2;
-      white-space: pre-wrap;
-      line-height: 1.3;  /* Important for ascii art. */
-      overflow: visible;
-      /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */
-      font-size: 16px;
-      margin-top: 10px;
-    }
-    pre:last-child {
-      margin-bottom: 0;
-    }
-    pre:hover {
-      overflow: visible;
-    }
-    .generator-stats {
-      color: gray;
-      font-size: smaller;
-    }
-  ]]
-  tofile(fname, css)
-end
-
--- Testing
-
-local function ok(cond, expected, actual, message)
-  assert(
-    (not expected and not actual) or (expected and actual),
-    'if "expected" is given, "actual" is also required'
-  )
-  if expected then
-    assert(
-      cond,
-      ('%sexpected %s, got: %s'):format(
-        message and (message .. '\n') or '',
-        vim.inspect(expected),
-        vim.inspect(actual)
-      )
-    )
-    return cond
-  else
-    return assert(cond)
-  end
-end
-local function eq(expected, actual, message)
-  return ok(vim.deep_equal(expected, actual), expected, actual, message)
-end
-
-function M._test()
-  tagmap = get_helptags('$VIMRUNTIME/doc')
-  helpfiles = get_helpfiles(vim.fs.normalize('$VIMRUNTIME/doc'))
-
-  ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap))
-  ok(
-    vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'),
-    tagmap['vim.diagnostic.set()'],
-    'diagnostic.txt'
-  )
-  ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt')
-  ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]]))
-  ok(is_noise([[          NVIM  REFERENCE  MANUAL     by  Thiago  de  Arruda      ]]))
-  ok(not is_noise([[vim:tw=78]]))
-
-  eq(0, get_indent('a'))
-  eq(1, get_indent(' a'))
-  eq(2, get_indent('  a\n  b\n  c\n'))
-  eq(5, get_indent('     a\n      \n        b\n      c\n      d\n      e\n'))
-  eq(
-    'a\n        \n   b\n c\n d\n e\n',
-    trim_indent('     a\n             \n        b\n      c\n      d\n      e\n')
-  )
-
-  local fixed_url, removed_chars = fix_url('https://example.com).')
-  eq('https://example.com', fixed_url)
-  eq(').', removed_chars)
-  fixed_url, removed_chars = fix_url('https://example.com.)')
-  eq('https://example.com.', fixed_url)
-  eq(')', removed_chars)
-  fixed_url, removed_chars = fix_url('https://example.com.')
-  eq('https://example.com', fixed_url)
-  eq('.', removed_chars)
-  fixed_url, removed_chars = fix_url('https://example.com)')
-  eq('https://example.com', fixed_url)
-  eq(')', removed_chars)
-  fixed_url, removed_chars = fix_url('https://example.com')
-  eq('https://example.com', fixed_url)
-  eq('', removed_chars)
-
-  print('all tests passed.\n')
-end
-
---- @class nvim.gen_help_html.gen_result
---- @field helpfiles string[] list of generated HTML files, from the source docs {include}
---- @field err_count integer number of parse errors in :help docs
---- @field invalid_links table
-
---- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`.
----
---- Example:
----
----   gen('$VIMRUNTIME/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil)
----
---- @param help_dir string Source directory containing the :help files. Must run `make helptags` first.
---- @param to_dir string Target directory where the .html files will be written.
---- @param include string[]|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'}
----
---- @return nvim.gen_help_html.gen_result result
-function M.gen(help_dir, to_dir, include, commit, parser_path)
-  vim.validate('help_dir', help_dir, function(d)
-    return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
-  end, 'valid directory')
-  vim.validate('to_dir', to_dir, 'string')
-  vim.validate('include', include, 'table', true)
-  vim.validate('commit', commit, 'string', true)
-  vim.validate('parser_path', parser_path, function(f)
-    return vim.fn.filereadable(vim.fs.normalize(f)) == 1
-  end, true, 'valid vimdoc.{so,dll} filepath')
-
-  local err_count = 0
-  local redirects_count = 0
-  ensure_runtimepath()
-  tagmap = get_helptags(vim.fs.normalize(help_dir))
-  helpfiles = get_helpfiles(help_dir, include)
-  to_dir = vim.fs.normalize(to_dir)
-  parser_path = parser_path and vim.fs.normalize(parser_path) or nil
-
-  print(('output dir: %s\n\n'):format(to_dir))
-  vim.fn.mkdir(to_dir, 'p')
-  gen_css(('%s/help.css'):format(to_dir))
-
-  for _, f in ipairs(helpfiles) do
-    -- "foo.txt"
-    local helpfile = vim.fs.basename(f)
-    -- "to/dir/foo.html"
-    local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile))
-    local html, stats =
-      gen_one(f, nil, to_fname, not new_layout[helpfile], commit or '?', parser_path)
-    tofile(to_fname, html)
-    print(
-      ('generated (%-2s errors): %-15s => %s'):format(
-        #stats.parse_errors,
-        helpfile,
-        vim.fs.basename(to_fname)
-      )
-    )
-
-    -- Generate redirect pages for renamed help files.
-    local helpfile_tag = (helpfile:gsub('%.txt$', ''))
-    local redirect_from = redirects[helpfile_tag]
-    if redirect_from then
-      local redirect_text = ([[
-*%s*      Nvim
-
-This document moved to: |%s|
-
-==============================================================================
-This document moved to: |%s|
-
-This document moved to: |%s|
-
-==============================================================================
- vim:tw=78:ts=8:ft=help:norl:
-      ]]):format(
-        redirect_from,
-        helpfile_tag,
-        helpfile_tag,
-        helpfile_tag,
-        helpfile_tag,
-        helpfile_tag
-      )
-      local redirect_to = ('%s/%s'):format(to_dir, get_helppage(redirect_from))
-      local redirect_html, _ =
-        gen_one(redirect_from, redirect_text, redirect_to, false, commit or '?', parser_path)
-      assert(redirect_html:find(helpfile_tag))
-      tofile(redirect_to, redirect_html)
-
-      print(
-        ('generated (redirect) : %-15s => %s'):format(
-          redirect_from .. '.txt',
-          vim.fs.basename(to_fname)
-        )
-      )
-      redirects_count = redirects_count + 1
-    end
-
-    err_count = err_count + #stats.parse_errors
-  end
-
-  print(('\ngenerated %d html pages'):format(#helpfiles + redirects_count))
-  print(('total errors: %d'):format(err_count))
-  print(('invalid tags: %s'):format(vim.inspect(invalid_links)))
-  assert(#(include or {}) > 0 or redirects_count == vim.tbl_count(redirects)) -- sanity check
-  print(('redirects: %d'):format(redirects_count))
-  print('\n')
-
-  --- @type nvim.gen_help_html.gen_result
-  return {
-    helpfiles = helpfiles,
-    err_count = err_count,
-    invalid_links = invalid_links,
-  }
-end
-
---- @class nvim.gen_help_html.validate_result
---- @field helpfiles integer number of generated helpfiles
---- @field err_count integer number of parse errors
---- @field parse_errors table
---- @field invalid_links table invalid tags in :help docs
---- @field invalid_urls table invalid URLs in :help docs
---- @field invalid_spelling table> invalid spelling in :help docs
-
---- Validates all :help files found in `help_dir`:
----  - checks that |tag| links point to valid helptags.
----  - recursively counts parse errors ("ERROR" nodes)
----
---- This is 10x faster than gen(), for use in CI.
----
---- @return nvim.gen_help_html.validate_result result
-function M.validate(help_dir, include, parser_path)
-  vim.validate('help_dir', help_dir, function(d)
-    return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
-  end, 'valid directory')
-  vim.validate('include', include, 'table', true)
-  vim.validate('parser_path', parser_path, function(f)
-    return vim.fn.filereadable(vim.fs.normalize(f)) == 1
-  end, true, 'valid vimdoc.{so,dll} filepath')
-  local err_count = 0 ---@type integer
-  local files_to_errors = {} ---@type table
-  ensure_runtimepath()
-  tagmap = get_helptags(vim.fs.normalize(help_dir))
-  helpfiles = get_helpfiles(help_dir, include)
-  parser_path = parser_path and vim.fs.normalize(parser_path) or nil
-
-  for _, f in ipairs(helpfiles) do
-    local helpfile = vim.fs.basename(f)
-    local rv = validate_one(f, parser_path)
-    print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile))
-    if #rv.parse_errors > 0 then
-      files_to_errors[helpfile] = rv.parse_errors
-      vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v)
-        return s .. '\n    ' .. v
-      end)))
-    end
-    err_count = err_count + #rv.parse_errors
-  end
-
-  ---@type nvim.gen_help_html.validate_result
-  return {
-    helpfiles = #helpfiles,
-    err_count = err_count,
-    parse_errors = files_to_errors,
-    invalid_links = invalid_links,
-    invalid_urls = invalid_urls,
-    invalid_spelling = invalid_spelling,
-  }
-end
-
---- Validates vimdoc files on $VIMRUNTIME. and print human-readable error messages if fails.
----
---- If this fails, try these steps (in order):
---- 1. Fix/cleanup the :help docs.
---- 2. Fix the parser: https://github.com/neovim/tree-sitter-vimdoc
---- 3. File a parser bug, and adjust the tolerance of this test in the meantime.
----
---- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
-function M.run_validate(help_dir)
-  help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
-  print('doc path = ' .. vim.uv.fs_realpath(help_dir))
-
-  local rv = M.validate(help_dir)
-
-  -- Check that we actually found helpfiles.
-  ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles)
-
-  eq({}, rv.parse_errors, 'no parse errors')
-  eq(0, rv.err_count, 'no parse errors')
-  eq({}, rv.invalid_links, 'invalid tags in :help docs')
-  eq({}, rv.invalid_urls, 'invalid URLs in :help docs')
-  eq(
-    {},
-    rv.invalid_spelling,
-    'invalid spelling in :help docs (see spell_dict in scripts/gen_help_html.lua)'
-  )
-end
-
---- Test-generates HTML from docs.
----
---- 1. Test that gen_help_html.lua actually works.
---- 2. Test that parse errors did not increase wildly. Because we explicitly test only a few
----    :help files, we can be precise about the tolerances here.
---- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
-function M.test_gen(help_dir)
-  local tmpdir = vim.fs.dirname(vim.fn.tempname())
-  help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
-  print('doc path = ' .. vim.uv.fs_realpath(help_dir))
-
-  -- Because gen() is slow (~30s), this test is limited to a few files.
-  local input = { 'help.txt', 'index.txt', 'nvim.txt' }
-  local rv = M.gen(help_dir, tmpdir, input)
-  eq(#input, #rv.helpfiles)
-  eq(0, rv.err_count, 'parse errors in :help docs')
-  eq({}, rv.invalid_links, 'invalid tags in :help docs')
-end
-
-return M
diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua
deleted file mode 100644
index 3e419c7d59..0000000000
--- a/scripts/gen_lsp.lua
+++ /dev/null
@@ -1,514 +0,0 @@
--- Generates lua-ls annotations for lsp.
-
-local USAGE = [[
-Generates lua-ls annotations for lsp.
-
-USAGE:
-nvim -l scripts/gen_lsp.lua gen  # by default, this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua
-nvim -l scripts/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua
-nvim -l scripts/gen_lsp.lua gen --version 3.18 --methods --capabilities
-]]
-
-local DEFAULT_LSP_VERSION = '3.18'
-
-local M = {}
-
-local function tofile(fname, text)
-  local f = io.open(fname, 'w')
-  if not f then
-    error(('failed to write: %s'):format(f))
-  else
-    print(('Written to: %s'):format(fname))
-    f:write(text)
-    f:close()
-  end
-end
-
---- The LSP protocol JSON data (it's partial, non-exhaustive).
---- https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/3.18/metaModel/metaModel.schema.json
---- @class vim._gen_lsp.Protocol
---- @field requests vim._gen_lsp.Request[]
---- @field notifications vim._gen_lsp.Notification[]
---- @field structures vim._gen_lsp.Structure[]
---- @field enumerations vim._gen_lsp.Enumeration[]
---- @field typeAliases vim._gen_lsp.TypeAlias[]
-
----@param opt vim._gen_lsp.opt
----@return vim._gen_lsp.Protocol
-local function read_json(opt)
-  local uri = 'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/'
-    .. opt.version
-    .. '/metaModel/metaModel.json'
-  print('Reading ' .. uri)
-
-  local res = vim.system({ 'curl', '--no-progress-meter', uri, '-o', '-' }):wait()
-  if res.code ~= 0 or (res.stdout or ''):len() < 999 then
-    print(('URL failed: %s'):format(uri))
-    vim.print(res)
-    error(res.stdout)
-  end
-  return vim.json.decode(res.stdout)
-end
-
--- Gets the Lua symbol for a given fully-qualified LSP method name.
-local function to_luaname(s)
-  -- "$/" prefix is special: https://microsoft.github.io/language-server-protocol/specification/#dollarRequests
-  return s:gsub('^%$', 'dollar'):gsub('/', '_')
-end
-
----@param protocol vim._gen_lsp.Protocol
----@param gen_methods boolean
----@param gen_capabilities boolean
-local function write_to_protocol(protocol, gen_methods, gen_capabilities)
-  if not gen_methods and not gen_capabilities then
-    return
-  end
-
-  local indent = (' '):rep(2)
-
-  --- @class vim._gen_lsp.Request
-  --- @field deprecated? string
-  --- @field documentation? string
-  --- @field messageDirection string
-  --- @field clientCapability? string
-  --- @field serverCapability? string
-  --- @field method string
-  --- @field params? any
-  --- @field proposed? boolean
-  --- @field registrationMethod? string
-  --- @field registrationOptions? any
-  --- @field since? string
-
-  --- @class vim._gen_lsp.Notification
-  --- @field deprecated? string
-  --- @field documentation? string
-  --- @field errorData? any
-  --- @field messageDirection string
-  --- @field clientCapability? string
-  --- @field serverCapability? string
-  --- @field method string
-  --- @field params? any[]
-  --- @field partialResult? any
-  --- @field proposed? boolean
-  --- @field registrationMethod? string
-  --- @field registrationOptions? any
-  --- @field result any
-  --- @field since? string
-
-  ---@type (vim._gen_lsp.Request|vim._gen_lsp.Notification)[]
-  local all = vim.list_extend(protocol.requests, protocol.notifications)
-  table.sort(all, function(a, b)
-    return to_luaname(a.method) < to_luaname(b.method)
-  end)
-
-  local output = { '-- Generated by gen_lsp.lua, keep at end of file.' }
-
-  if gen_methods then
-    output[#output + 1] = '--- @alias vim.lsp.protocol.Method.ClientToServer'
-
-    for _, item in ipairs(all) do
-      if item.method and item.messageDirection == 'clientToServer' then
-        output[#output + 1] = ("--- | '%s',"):format(item.method)
-      end
-    end
-
-    vim.list_extend(output, {
-      '',
-      '--- @alias vim.lsp.protocol.Method.ServerToClient',
-    })
-    for _, item in ipairs(all) do
-      if item.method and item.messageDirection == 'serverToClient' then
-        output[#output + 1] = ("--- | '%s',"):format(item.method)
-      end
-    end
-
-    vim.list_extend(output, {
-      '',
-      '--- @alias vim.lsp.protocol.Method',
-      '--- | vim.lsp.protocol.Method.ClientToServer',
-      '--- | vim.lsp.protocol.Method.ServerToClient',
-      '',
-      '-- Generated by gen_lsp.lua, keep at end of file.',
-      '--- @enum vim.lsp.protocol.Methods',
-      '--- @see https://microsoft.github.io/language-server-protocol/specification/#metaModel',
-      '--- LSP method names.',
-      'protocol.Methods = {',
-    })
-
-    for _, item in ipairs(all) do
-      if item.method then
-        if item.documentation then
-          local document = vim.split(item.documentation, '\n?\n', { trimempty = true })
-          for _, docstring in ipairs(document) do
-            output[#output + 1] = indent .. '--- ' .. docstring
-          end
-        end
-        output[#output + 1] = ("%s%s = '%s',"):format(indent, to_luaname(item.method), item.method)
-      end
-    end
-    output[#output + 1] = '}'
-  end
-
-  if gen_capabilities then
-    vim.list_extend(output, {
-      '',
-      '-- stylua: ignore start',
-      '-- Generated by gen_lsp.lua, keep at end of file.',
-      '--- Maps method names to the required server capability',
-      'protocol._request_name_to_capability = {',
-    })
-
-    for _, item in ipairs(all) do
-      if item.serverCapability then
-        output[#output + 1] = ("%s['%s'] = { %s },"):format(
-          indent,
-          item.method,
-          table.concat(
-            vim
-              .iter(vim.split(item.serverCapability, '.', { plain = true }))
-              :map(function(segment)
-                return "'" .. segment .. "'"
-              end)
-              :totable(),
-            ', '
-          )
-        )
-      end
-    end
-
-    output[#output + 1] = '}'
-    output[#output + 1] = '-- stylua: ignore end'
-  end
-
-  output[#output + 1] = ''
-  output[#output + 1] = 'return protocol'
-
-  local fname = './runtime/lua/vim/lsp/protocol.lua'
-  local bufnr = vim.fn.bufadd(fname)
-  vim.fn.bufload(bufnr)
-  vim.api.nvim_set_current_buf(bufnr)
-  local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false)
-  local index = vim.iter(ipairs(lines)):find(function(key, item)
-    return vim.startswith(item, '-- Generated by') and key or nil
-  end)
-  index = index and index - 1 or vim.api.nvim_buf_line_count(bufnr) - 1
-  vim.api.nvim_buf_set_lines(bufnr, index, -1, true, output)
-  vim.cmd.write()
-end
-
----@class vim._gen_lsp.opt
----@field output_file string
----@field version string
----@field methods boolean
----@field capabilities boolean
-
----@param opt vim._gen_lsp.opt
-function M.gen(opt)
-  --- @type vim._gen_lsp.Protocol
-  local protocol = read_json(opt)
-
-  write_to_protocol(protocol, opt.methods, opt.capabilities)
-
-  local output = {
-    '--' .. '[[',
-    'THIS FILE IS GENERATED by scripts/gen_lsp.lua',
-    'DO NOT EDIT MANUALLY',
-    '',
-    'Based on LSP protocol ' .. opt.version,
-    '',
-    'Regenerate:',
-    ([=[nvim -l scripts/gen_lsp.lua gen --version %s]=]):format(DEFAULT_LSP_VERSION),
-    '--' .. ']]',
-    '',
-    '---@meta',
-    "error('Cannot require a meta file')",
-    '',
-    '---@alias lsp.null nil',
-    '---@alias uinteger integer',
-    '---@alias decimal number',
-    '---@alias lsp.DocumentUri string',
-    '---@alias lsp.URI string',
-    '',
-  }
-
-  local anonymous_num = 0
-
-  ---@type string[]
-  local anonym_classes = {}
-
-  local simple_types = {
-    'string',
-    'boolean',
-    'integer',
-    'uinteger',
-    'decimal',
-  }
-
-  ---@param documentation string
-  local _process_documentation = function(documentation)
-    documentation = documentation:gsub('\n', '\n---')
-    -- Remove <200b> (zero-width space) unicode characters: e.g., `**/<200b>*`
-    documentation = documentation:gsub('\226\128\139', '')
-    -- Escape annotations that are not recognized by lua-ls
-    documentation = documentation:gsub('%^---@sample', '---\\@sample')
-    return '---' .. documentation
-  end
-
-  --- @class vim._gen_lsp.Type
-  --- @field kind string a common field for all Types.
-  --- @field name? string for ReferenceType, BaseType
-  --- @field element? any for ArrayType
-  --- @field items? vim._gen_lsp.Type[] for OrType, AndType
-  --- @field key? vim._gen_lsp.Type for MapType
-  --- @field value? string|vim._gen_lsp.Type for StringLiteralType, MapType, StructureLiteralType
-
-  ---@param type vim._gen_lsp.Type
-  ---@param prefix? string Optional prefix associated with the this type, made of (nested) field name.
-  ---             Used to generate class name for structure literal types.
-  ---@return string
-  local function parse_type(type, prefix)
-    -- ReferenceType | BaseType
-    if type.kind == 'reference' or type.kind == 'base' then
-      if vim.tbl_contains(simple_types, type.name) then
-        return type.name
-      end
-      return 'lsp.' .. type.name
-
-    -- ArrayType
-    elseif type.kind == 'array' then
-      local parsed_items = parse_type(type.element, prefix)
-      if type.element.items and #type.element.items > 1 then
-        parsed_items = '(' .. parsed_items .. ')'
-      end
-      return parsed_items .. '[]'
-
-    -- OrType
-    elseif type.kind == 'or' then
-      local val = ''
-      for _, item in ipairs(type.items) do
-        val = val .. parse_type(item, prefix) .. '|' --[[ @as string ]]
-      end
-      val = val:sub(0, -2)
-      return val
-
-    -- StringLiteralType
-    elseif type.kind == 'stringLiteral' then
-      return '"' .. type.value .. '"'
-
-    -- MapType
-    elseif type.kind == 'map' then
-      local key = assert(type.key)
-      local value = type.value --[[ @as vim._gen_lsp.Type ]]
-      return 'table<' .. parse_type(key, prefix) .. ', ' .. parse_type(value, prefix) .. '>'
-
-    -- StructureLiteralType
-    elseif type.kind == 'literal' then
-      -- can I use ---@param disabled? {reason: string}
-      -- use | to continue the inline class to be able to add docs
-      -- https://github.com/LuaLS/lua-language-server/issues/2128
-      anonymous_num = anonymous_num + 1
-      local anonymous_classname = 'lsp._anonym' .. anonymous_num
-      if prefix then
-        anonymous_classname = anonymous_classname .. '.' .. prefix
-      end
-      local anonym = vim
-        .iter({
-          (anonymous_num > 1 and { '' } or {}),
-          { '---@class ' .. anonymous_classname },
-        })
-        :flatten()
-        :totable()
-
-      --- @class vim._gen_lsp.StructureLiteral translated to anonymous @class.
-      --- @field deprecated? string
-      --- @field description? string
-      --- @field properties vim._gen_lsp.Property[]
-      --- @field proposed? boolean
-      --- @field since? string
-
-      ---@type vim._gen_lsp.StructureLiteral
-      local structural_literal = assert(type.value) --[[ @as vim._gen_lsp.StructureLiteral ]]
-      for _, field in ipairs(structural_literal.properties) do
-        anonym[#anonym + 1] = '---'
-        if field.documentation then
-          anonym[#anonym + 1] = _process_documentation(field.documentation)
-        end
-        anonym[#anonym + 1] = '---@field '
-          .. field.name
-          .. (field.optional and '?' or '')
-          .. ' '
-          .. parse_type(field.type, prefix .. '.' .. field.name)
-      end
-      -- anonym[#anonym + 1] = ''
-      for _, line in ipairs(anonym) do
-        if line then
-          anonym_classes[#anonym_classes + 1] = line
-        end
-      end
-      return anonymous_classname
-
-    -- TupleType
-    elseif type.kind == 'tuple' then
-      local tuple = '['
-      for _, value in ipairs(type.items) do
-        tuple = tuple .. parse_type(value, prefix) .. ', '
-      end
-      -- remove , at the end
-      tuple = tuple:sub(0, -3)
-      return tuple .. ']'
-    end
-
-    vim.print('WARNING: Unknown type ', type)
-    return ''
-  end
-
-  --- @class vim._gen_lsp.Structure translated to @class
-  --- @field deprecated? string
-  --- @field documentation? string
-  --- @field extends? { kind: string, name: string }[]
-  --- @field mixins? { kind: string, name: string }[]
-  --- @field name string
-  --- @field properties? vim._gen_lsp.Property[]  members, translated to @field
-  --- @field proposed? boolean
-  --- @field since? string
-  for _, structure in ipairs(protocol.structures) do
-    -- output[#output + 1] = ''
-    if structure.documentation then
-      output[#output + 1] = _process_documentation(structure.documentation)
-    end
-    local class_string = ('---@class lsp.%s'):format(structure.name)
-    if structure.extends or structure.mixins then
-      local inherits_from = table.concat(
-        vim.list_extend(
-          vim.tbl_map(parse_type, structure.extends or {}),
-          vim.tbl_map(parse_type, structure.mixins or {})
-        ),
-        ', '
-      )
-      class_string = class_string .. ': ' .. inherits_from
-    end
-    output[#output + 1] = class_string
-
-    --- @class vim._gen_lsp.Property translated to @field
-    --- @field deprecated? string
-    --- @field documentation? string
-    --- @field name string
-    --- @field optional? boolean
-    --- @field proposed? boolean
-    --- @field since? string
-    --- @field type { kind: string, name: string }
-    for _, field in ipairs(structure.properties or {}) do
-      output[#output + 1] = '---' -- Insert a single newline between @fields (and after @class)
-      if field.documentation then
-        output[#output + 1] = _process_documentation(field.documentation)
-      end
-      output[#output + 1] = '---@field '
-        .. field.name
-        .. (field.optional and '?' or '')
-        .. ' '
-        .. parse_type(field.type, field.name)
-    end
-    output[#output + 1] = ''
-  end
-
-  --- @class vim._gen_lsp.Enumeration translated to @enum
-  --- @field deprecated string?
-  --- @field documentation string?
-  --- @field name string?
-  --- @field proposed boolean?
-  --- @field since string?
-  --- @field suportsCustomValues boolean?
-  --- @field values { name: string, value: string, documentation?: string, since?: string }[]
-  for _, enum in ipairs(protocol.enumerations) do
-    if enum.documentation then
-      output[#output + 1] = _process_documentation(enum.documentation)
-    end
-    local enum_type = '---@alias lsp.' .. enum.name
-    for _, value in ipairs(enum.values) do
-      enum_type = enum_type
-        .. '\n---| '
-        .. (type(value.value) == 'string' and '"' .. value.value .. '"' or value.value)
-        .. ' # '
-        .. value.name
-    end
-    output[#output + 1] = enum_type
-    output[#output + 1] = ''
-  end
-
-  --- @class vim._gen_lsp.TypeAlias translated to @alias
-  --- @field deprecated? string?
-  --- @field documentation? string
-  --- @field name string
-  --- @field proposed? boolean
-  --- @field since? string
-  --- @field type vim._gen_lsp.Type
-  for _, alias in ipairs(protocol.typeAliases) do
-    if alias.documentation then
-      output[#output + 1] = _process_documentation(alias.documentation)
-    end
-    if alias.type.kind == 'or' then
-      local alias_type = '---@alias lsp.' .. alias.name .. ' '
-      for _, item in ipairs(alias.type.items) do
-        alias_type = alias_type .. parse_type(item, alias.name) .. '|'
-      end
-      alias_type = alias_type:sub(0, -2)
-      output[#output + 1] = alias_type
-    else
-      output[#output + 1] = '---@alias lsp.'
-        .. alias.name
-        .. ' '
-        .. parse_type(alias.type, alias.name)
-    end
-    output[#output + 1] = ''
-  end
-
-  -- anonymous classes
-  for _, line in ipairs(anonym_classes) do
-    output[#output + 1] = line
-  end
-
-  tofile(opt.output_file, table.concat(output, '\n') .. '\n')
-end
-
----@type vim._gen_lsp.opt
-local opt = {
-  output_file = 'runtime/lua/vim/lsp/_meta/protocol.lua',
-  version = DEFAULT_LSP_VERSION,
-  methods = false,
-  capabilities = false,
-}
-
-local command = nil
-local i = 1
-while i <= #_G.arg do
-  if _G.arg[i] == '--out' then
-    opt.output_file = assert(_G.arg[i + 1], '--out  needed')
-    i = i + 1
-  elseif _G.arg[i] == '--version' then
-    opt.version = assert(_G.arg[i + 1], '--version  needed')
-    i = i + 1
-  elseif _G.arg[i] == '--methods' then
-    opt.methods = true
-  elseif _G.arg[i] == '--capabilities' then
-    opt.capabilities = true
-  elseif vim.startswith(_G.arg[i], '-') then
-    error('Unrecognized args: ' .. _G.arg[i])
-  else
-    if command then
-      error('More than one command was given: ' .. _G.arg[i])
-    else
-      command = _G.arg[i]
-    end
-  end
-  i = i + 1
-end
-
-if not command then
-  print(USAGE)
-elseif M[command] then
-  M[command](opt) -- see M.gen()
-else
-  error('Unknown command: ' .. command)
-end
-
-return M
diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua
deleted file mode 100755
index d200050fe1..0000000000
--- a/scripts/gen_vimdoc.lua
+++ /dev/null
@@ -1,1041 +0,0 @@
-#!/usr/bin/env -S nvim -l
---- Generates Nvim :help docs from Lua/C docstrings
----
---- The generated :help text for each function is formatted as follows:
---- - Max width of 78 columns (`TEXT_WIDTH`).
---- - Indent with spaces (not tabs).
---- - Indent of 4 columns for body text (`INDENTATION`).
---- - Function signature and helptag (right-aligned) on the same line.
----   - Signature and helptag must have a minimum of 8 spaces between them.
----   - If the signature is too long, it is placed on the line after the helptag.
----     Signature wraps with subsequent lines indented to the open parenthesis.
----   - Subsection bodies are indented an additional 4 spaces.
---- - Body consists of function description, parameters, return description, and
----   C declaration (`INCLUDE_C_DECL`).
---- - Parameters are omitted for the `void` and `Error *` types, or if the
----   parameter is marked as [out].
---- - Each function documentation is separated by a single line.
-
-local luacats_parser = require('scripts.luacats_parser')
-local cdoc_parser = require('scripts.cdoc_parser')
-local util = require('scripts.util')
-
-local fmt = string.format
-
-local wrap = util.wrap
-local md_to_vimdoc = util.md_to_vimdoc
-
-local TEXT_WIDTH = 78
-local INDENTATION = 4
-
---- @class (exact) nvim.gen_vimdoc.Config
----
---- Generated documentation target, e.g. api.txt
---- @field filename string
----
---- @field section_order string[]
----
---- List of files/directories for doxygen to read, relative to `base_dir`.
---- @field files string[]
----
---- @field exclude_types? true
----
---- Section name overrides. Key: filename (e.g., vim.c)
---- @field section_name? table
----
---- @field fn_name_pat? string
----
---- @field fn_xform? fun(fun: nvim.luacats.parser.fun)
----
---- For generated section names.
---- @field section_fmt fun(name: string): string
----
---- @field helptag_fmt fun(name: string): string|string[]
----
---- Per-function helptag.
---- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string
----
---- @field append_only? string[]
-
-local function contains(t, xs)
-  return vim.tbl_contains(xs, t)
-end
-
---- @type {level:integer, prerelease:boolean}?
-local nvim_api_info_
-
---- @return {level: integer, prerelease:boolean}
-local function nvim_api_info()
-  if not nvim_api_info_ then
-    --- @type integer?, boolean?
-    local level, prerelease
-    for l in io.lines('CMakeLists.txt') do
-      --- @cast l string
-      if level and prerelease then
-        break
-      end
-      local m1 = l:match('^set%(NVIM_API_LEVEL%s+(%d+)%)')
-      if m1 then
-        level = tonumber(m1) --[[@as integer]]
-      end
-      local m2 = l:match('^set%(NVIM_API_PRERELEASE%s+(%w+)%)')
-      if m2 then
-        prerelease = m2 == 'true'
-      end
-    end
-    nvim_api_info_ = { level = level, prerelease = prerelease }
-  end
-
-  return nvim_api_info_
-end
-
---- @param fun nvim.luacats.parser.fun
---- @return string
-local function fn_helptag_fmt_common(fun)
-  local fn_sfx = fun.table and '' or '()'
-  if fun.classvar then
-    return fmt('%s:%s%s', fun.classvar, fun.name, fn_sfx)
-  end
-  if fun.module then
-    return fmt('%s.%s%s', fun.module, fun.name, fn_sfx)
-  end
-  return fun.name .. fn_sfx
-end
-
---- @type table
-local config = {
-  api = {
-    filename = 'api.txt',
-    section_order = {
-      'vim.c',
-      'vimscript.c',
-      'command.c',
-      'options.c',
-      'buffer.c',
-      'extmark.c',
-      'window.c',
-      'win_config.c',
-      'tabpage.c',
-      'autocmd.c',
-      'ui.c',
-    },
-    exclude_types = true,
-    fn_name_pat = 'nvim_.*',
-    files = { 'src/nvim/api' },
-    section_name = {
-      ['vim.c'] = 'Global',
-    },
-    section_fmt = function(name)
-      return name .. ' Functions'
-    end,
-    helptag_fmt = function(name)
-      return fmt('api-%s', name:lower())
-    end,
-  },
-  lua = {
-    filename = 'lua.txt',
-    section_order = {
-      'hl.lua',
-      'diff.lua',
-      'mpack.lua',
-      'json.lua',
-      'base64.lua',
-      'spell.lua',
-      'builtin.lua',
-      '_options.lua',
-      '_editor.lua',
-      '_inspector.lua',
-      'shared.lua',
-      'loader.lua',
-      'uri.lua',
-      'ui.lua',
-      'filetype.lua',
-      'keymap.lua',
-      'fs.lua',
-      'glob.lua',
-      'lpeg.lua',
-      're.lua',
-      'regex.lua',
-      'secure.lua',
-      'version.lua',
-      'iter.lua',
-      'snippet.lua',
-      'text.lua',
-      'tohtml.lua',
-    },
-    files = {
-      'runtime/lua/vim/iter.lua',
-      'runtime/lua/vim/_editor.lua',
-      'runtime/lua/vim/_options.lua',
-      'runtime/lua/vim/shared.lua',
-      'runtime/lua/vim/loader.lua',
-      'runtime/lua/vim/uri.lua',
-      'runtime/lua/vim/ui.lua',
-      'runtime/lua/vim/filetype.lua',
-      'runtime/lua/vim/keymap.lua',
-      'runtime/lua/vim/fs.lua',
-      'runtime/lua/vim/hl.lua',
-      'runtime/lua/vim/secure.lua',
-      'runtime/lua/vim/version.lua',
-      'runtime/lua/vim/_inspector.lua',
-      'runtime/lua/vim/snippet.lua',
-      'runtime/lua/vim/text.lua',
-      'runtime/lua/vim/glob.lua',
-      'runtime/lua/vim/_meta/builtin.lua',
-      'runtime/lua/vim/_meta/diff.lua',
-      'runtime/lua/vim/_meta/mpack.lua',
-      'runtime/lua/vim/_meta/json.lua',
-      'runtime/lua/vim/_meta/base64.lua',
-      'runtime/lua/vim/_meta/regex.lua',
-      'runtime/lua/vim/_meta/lpeg.lua',
-      'runtime/lua/vim/_meta/re.lua',
-      'runtime/lua/vim/_meta/spell.lua',
-      'runtime/lua/tohtml.lua',
-    },
-    fn_xform = function(fun)
-      if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then
-        fun.module = 'vim'
-      end
-
-      if fun.module == 'vim' and contains(fun.name, { 'cmd', 'inspect' }) then
-        fun.table = nil
-      end
-
-      if fun.classvar or vim.startswith(fun.name, 'vim.') or fun.module == 'vim.iter' then
-        return
-      end
-
-      fun.name = fmt('%s.%s', fun.module, fun.name)
-    end,
-    section_name = {
-      ['_inspector.lua'] = 'inspector',
-    },
-    section_fmt = function(name)
-      name = name:lower()
-      if name == '_editor' then
-        return 'Lua module: vim'
-      elseif name == '_options' then
-        return 'LUA-VIMSCRIPT BRIDGE'
-      elseif name == 'builtin' then
-        return 'VIM'
-      end
-      if
-        contains(name, {
-          'hl',
-          'mpack',
-          'json',
-          'base64',
-          'diff',
-          'spell',
-          'regex',
-          'lpeg',
-          're',
-        })
-      then
-        return 'VIM.' .. name:upper()
-      end
-      if name == 'tohtml' then
-        return 'Lua module: tohtml'
-      end
-      return 'Lua module: vim.' .. name
-    end,
-    helptag_fmt = function(name)
-      if name == '_editor' then
-        return 'lua-vim'
-      elseif name == '_options' then
-        return 'lua-vimscript'
-      elseif name == 'tohtml' then
-        return 'tohtml'
-      end
-      return 'vim.' .. name:lower()
-    end,
-    fn_helptag_fmt = function(fun)
-      local name = fun.name
-
-      if vim.startswith(name, 'vim.') then
-        local fn_sfx = fun.table and '' or '()'
-        return name .. fn_sfx
-      elseif fun.classvar == 'Option' then
-        return fmt('vim.opt:%s()', name)
-      end
-
-      return fn_helptag_fmt_common(fun)
-    end,
-    append_only = {
-      'shared.lua',
-    },
-  },
-  lsp = {
-    filename = 'lsp.txt',
-    section_order = {
-      'lsp.lua',
-      'client.lua',
-      'buf.lua',
-      'diagnostic.lua',
-      'codelens.lua',
-      'completion.lua',
-      'folding_range.lua',
-      'inlay_hint.lua',
-      'tagfunc.lua',
-      'semantic_tokens.lua',
-      'handlers.lua',
-      'util.lua',
-      'log.lua',
-      'rpc.lua',
-      'protocol.lua',
-    },
-    files = {
-      'runtime/lua/vim/lsp',
-      'runtime/lua/vim/lsp.lua',
-    },
-    fn_xform = function(fun)
-      fun.name = fun.name:gsub('result%.', '')
-      if fun.module == 'vim.lsp.protocol' then
-        fun.classvar = nil
-      end
-    end,
-    section_fmt = function(name)
-      if name:lower() == 'lsp' then
-        return 'Lua module: vim.lsp'
-      end
-      return 'Lua module: vim.lsp.' .. name:lower()
-    end,
-    helptag_fmt = function(name)
-      if name:lower() == 'lsp' then
-        return 'lsp-core'
-      end
-      return fmt('lsp-%s', name:lower())
-    end,
-  },
-  diagnostic = {
-    filename = 'diagnostic.txt',
-    section_order = {
-      'diagnostic.lua',
-    },
-    files = { 'runtime/lua/vim/diagnostic.lua' },
-    section_fmt = function()
-      return 'Lua module: vim.diagnostic'
-    end,
-    helptag_fmt = function()
-      return 'diagnostic-api'
-    end,
-  },
-  treesitter = {
-    filename = 'treesitter.txt',
-    section_order = {
-      'tstree.lua',
-      'tsnode.lua',
-      'treesitter.lua',
-      'language.lua',
-      'query.lua',
-      'highlighter.lua',
-      'languagetree.lua',
-      'dev.lua',
-    },
-    files = {
-      'runtime/lua/vim/treesitter/_meta/',
-      'runtime/lua/vim/treesitter.lua',
-      'runtime/lua/vim/treesitter/',
-    },
-    section_fmt = function(name)
-      if name:lower() == 'treesitter' then
-        return 'Lua module: vim.treesitter'
-      elseif name:lower() == 'tstree' then
-        return 'TREESITTER TREES'
-      elseif name:lower() == 'tsnode' then
-        return 'TREESITTER NODES'
-      end
-      return 'Lua module: vim.treesitter.' .. name:lower()
-    end,
-    helptag_fmt = function(name)
-      if name:lower() == 'treesitter' then
-        return 'lua-treesitter-core'
-      elseif name:lower() == 'query' then
-        return 'lua-treesitter-query'
-      elseif name:lower() == 'tstree' then
-        return { 'treesitter-tree', 'TSTree' }
-      elseif name:lower() == 'tsnode' then
-        return { 'treesitter-node', 'TSNode' }
-      end
-      return 'treesitter-' .. name:lower()
-    end,
-  },
-  editorconfig = {
-    filename = 'editorconfig.txt',
-    files = {
-      'runtime/lua/editorconfig.lua',
-    },
-    section_order = {
-      'editorconfig.lua',
-    },
-    section_fmt = function(_name)
-      return 'EditorConfig integration'
-    end,
-    helptag_fmt = function(name)
-      return name:lower()
-    end,
-    fn_xform = function(fun)
-      fun.table = true
-      fun.name = vim.split(fun.name, '.', { plain = true })[2]
-    end,
-  },
-  health = {
-    filename = 'health.txt',
-    files = {
-      'runtime/lua/vim/health.lua',
-    },
-    section_order = {
-      'health.lua',
-    },
-    section_fmt = function(_name)
-      return 'Checkhealth'
-    end,
-    helptag_fmt = function()
-      return { 'vim.health', 'health' }
-    end,
-  },
-}
-
---- @param ty string
---- @param generics table
---- @return string
-local function replace_generics(ty, generics)
-  if ty:sub(-2) == '[]' then
-    local ty0 = ty:sub(1, -3)
-    if generics[ty0] then
-      return generics[ty0] .. '[]'
-    end
-  elseif ty:sub(-1) == '?' then
-    local ty0 = ty:sub(1, -2)
-    if generics[ty0] then
-      return generics[ty0] .. '?'
-    end
-  end
-
-  return generics[ty] or ty
-end
-
---- @param name string
-local function fmt_field_name(name)
-  local name0, opt = name:match('^([^?]*)(%??)$')
-  return fmt('{%s}%s', name0, opt)
-end
-
---- @param ty string
---- @param generics? table
---- @param default? string
-local function render_type(ty, generics, default)
-  if generics then
-    ty = replace_generics(ty, generics)
-  end
-  ty = ty:gsub('%s*|%s*nil', '?')
-  ty = ty:gsub('nil%s*|%s*(.*)', '%1?')
-  ty = ty:gsub('%s*|%s*', '|')
-  if default then
-    return fmt('(`%s`, default: %s)', ty, default)
-  end
-  return fmt('(`%s`)', ty)
-end
-
---- @param p nvim.luacats.parser.param|nvim.luacats.parser.field
-local function should_render_field_or_param(p)
-  return not p.nodoc
-    and not p.access
-    and not contains(p.name, { '_', 'self' })
-    and not vim.startswith(p.name, '_')
-end
-
---- @param desc? string
---- @return string?, string?
-local function get_default(desc)
-  if not desc then
-    return
-  end
-
-  local default = desc:match('\n%s*%([dD]efault: ([^)]+)%)')
-  if default then
-    desc = desc:gsub('\n%s*%([dD]efault: [^)]+%)', '')
-  end
-
-  return desc, default
-end
-
---- @param ty string
---- @param classes? table
---- @return nvim.luacats.parser.class?
-local function get_class(ty, classes)
-  if not classes then
-    return
-  end
-
-  local cty = ty:gsub('%s*|%s*nil', '?'):gsub('?$', ''):gsub('%[%]$', '')
-
-  return classes[cty]
-end
-
---- @param obj nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.field
---- @param classes? table
-local function inline_type(obj, classes)
-  local ty = obj.type
-  if not ty then
-    return
-  end
-
-  local cls = get_class(ty, classes)
-
-  if not cls or cls.nodoc then
-    return
-  end
-
-  if not cls.inlinedoc then
-    -- Not inlining so just add a: "See |tag|."
-    local tag = fmt('|%s|', cls.name)
-    if obj.desc and obj.desc:find(tag) then
-      -- Tag already there
-      return
-    end
-
-    -- TODO(lewis6991): Aim to remove this. Need this to prevent dead
-    -- references to types defined in runtime/lua/vim/lsp/_meta/protocol.lua
-    if not vim.startswith(cls.name, 'vim.') then
-      return
-    end
-
-    obj.desc = obj.desc or ''
-    local period = (obj.desc == '' or vim.endswith(obj.desc, '.')) and '' or '.'
-    obj.desc = obj.desc .. fmt('%s See %s.', period, tag)
-    return
-  end
-
-  local ty_isopt = (ty:match('%?$') or ty:match('%s*|%s*nil')) ~= nil
-  local ty_islist = (ty:match('%[%]$')) ~= nil
-  ty = ty_isopt and 'table?' or ty_islist and 'table[]' or 'table'
-
-  local desc = obj.desc or ''
-  if cls.desc then
-    desc = desc .. cls.desc
-  elseif desc == '' then
-    if ty_islist then
-      desc = desc .. 'A list of objects with the following fields:'
-    elseif cls.parent then
-      desc = desc .. fmt('Extends |%s| with the additional fields:', cls.parent)
-    else
-      desc = desc .. 'A table with the following fields:'
-    end
-  end
-
-  local desc_append = {}
-  for _, f in ipairs(cls.fields) do
-    if not f.access then
-      local fdesc, default = get_default(f.desc)
-      local fty = render_type(f.type, nil, default)
-      local fnm = fmt_field_name(f.name)
-      table.insert(desc_append, table.concat({ '-', fnm, fty, fdesc }, ' '))
-    end
-  end
-
-  desc = desc .. '\n' .. table.concat(desc_append, '\n')
-  obj.type = ty
-  obj.desc = desc
-end
-
---- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[]
---- @param generics? table
---- @param classes? table
---- @param exclude_types? true
---- @param cfg nvim.gen_vimdoc.Config
-local function render_fields_or_params(xs, generics, classes, exclude_types, cfg)
-  local ret = {} --- @type string[]
-
-  xs = vim.tbl_filter(should_render_field_or_param, xs)
-
-  local indent = 0
-  for _, p in ipairs(xs) do
-    if p.type or p.desc then
-      indent = math.max(indent, #p.name + 3)
-    end
-    if exclude_types then
-      p.type = nil
-    end
-  end
-
-  for _, p in ipairs(xs) do
-    local pdesc, default = get_default(p.desc)
-    p.desc = pdesc
-
-    inline_type(p, classes)
-    local nm, ty = p.name, p.type
-
-    local desc = p.classvar and string.format('See |%s|.', cfg.fn_helptag_fmt(p)) or p.desc
-
-    local fnm = p.kind == 'operator' and fmt('op(%s)', nm) or fmt_field_name(nm)
-    local pnm = fmt('      • %-' .. indent .. 's', fnm)
-
-    if ty then
-      local pty = render_type(ty, generics, default)
-
-      if desc then
-        table.insert(ret, pnm)
-        if #pty > TEXT_WIDTH - indent then
-          vim.list_extend(ret, { ' ', pty, '\n' })
-          table.insert(ret, md_to_vimdoc(desc, 9 + indent, 9 + indent, TEXT_WIDTH, true))
-        else
-          desc = fmt('%s %s', pty, desc)
-          table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true))
-        end
-      else
-        table.insert(ret, fmt('%s %s\n', pnm, pty))
-      end
-    else
-      if desc then
-        table.insert(ret, pnm)
-        table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true))
-      end
-    end
-  end
-
-  return table.concat(ret)
-end
-
---- @param class nvim.luacats.parser.class
---- @param classes table
---- @param cfg nvim.gen_vimdoc.Config
-local function render_class(class, classes, cfg)
-  if class.access or class.nodoc or class.inlinedoc then
-    return
-  end
-
-  local ret = {} --- @type string[]
-
-  table.insert(ret, fmt('*%s*\n', class.name))
-
-  if class.parent then
-    local txt = fmt('Extends: |%s|', class.parent)
-    table.insert(ret, md_to_vimdoc(txt, INDENTATION, INDENTATION, TEXT_WIDTH))
-    table.insert(ret, '\n')
-  end
-
-  if class.desc then
-    table.insert(ret, md_to_vimdoc(class.desc, INDENTATION, INDENTATION, TEXT_WIDTH))
-  end
-
-  local fields_txt = render_fields_or_params(class.fields, nil, classes, nil, cfg)
-  if not fields_txt:match('^%s*$') then
-    table.insert(ret, '\n    Fields: ~\n')
-    table.insert(ret, fields_txt)
-  end
-  table.insert(ret, '\n')
-
-  return table.concat(ret)
-end
-
---- @param classes table
---- @param cfg nvim.gen_vimdoc.Config
-local function render_classes(classes, cfg)
-  local ret = {} --- @type string[]
-
-  for _, class in vim.spairs(classes) do
-    ret[#ret + 1] = render_class(class, classes, cfg)
-  end
-
-  return table.concat(ret)
-end
-
---- @param fun nvim.luacats.parser.fun
---- @param cfg nvim.gen_vimdoc.Config
-local function render_fun_header(fun, cfg)
-  local ret = {} --- @type string[]
-
-  local args = {} --- @type string[]
-  for _, p in ipairs(fun.params or {}) do
-    if p.name ~= 'self' then
-      args[#args + 1] = fmt_field_name(p.name)
-    end
-  end
-
-  local nm = fun.name
-  if fun.classvar then
-    nm = fmt('%s:%s', fun.classvar, nm)
-  end
-  if nm == 'vim.bo' then
-    nm = 'vim.bo[{bufnr}]'
-  end
-  if nm == 'vim.wo' then
-    nm = 'vim.wo[{winid}][{bufnr}]'
-  end
-
-  local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')'
-
-  local tag = '*' .. cfg.fn_helptag_fmt(fun) .. '*'
-
-  if #proto + #tag > TEXT_WIDTH - 8 then
-    table.insert(ret, fmt('%78s\n', tag))
-    local name, pargs = proto:match('([^(]+%()(.*)')
-    table.insert(ret, name)
-    table.insert(ret, wrap(pargs, 0, #name, TEXT_WIDTH))
-  else
-    local pad = TEXT_WIDTH - #proto - #tag
-    table.insert(ret, proto .. string.rep(' ', pad) .. tag)
-  end
-
-  return table.concat(ret)
-end
-
---- @param returns nvim.luacats.parser.return[]
---- @param generics? table
---- @param classes? table
---- @param exclude_types boolean
-local function render_returns(returns, generics, classes, exclude_types)
-  local ret = {} --- @type string[]
-
-  returns = vim.deepcopy(returns)
-  if exclude_types then
-    for _, r in ipairs(returns) do
-      r.type = nil
-    end
-  end
-
-  if #returns > 1 then
-    table.insert(ret, '    Return (multiple): ~\n')
-  elseif #returns == 1 and next(returns[1]) then
-    table.insert(ret, '    Return: ~\n')
-  end
-
-  for _, p in ipairs(returns) do
-    inline_type(p, classes)
-    local rnm, ty, desc = p.name, p.type, p.desc
-
-    local blk = {} --- @type string[]
-    if ty then
-      blk[#blk + 1] = render_type(ty, generics)
-    end
-    blk[#blk + 1] = rnm
-    blk[#blk + 1] = desc
-
-    table.insert(ret, md_to_vimdoc(table.concat(blk, ' '), 8, 8, TEXT_WIDTH, true))
-  end
-
-  return table.concat(ret)
-end
-
---- @param fun nvim.luacats.parser.fun
---- @param classes table
---- @param cfg nvim.gen_vimdoc.Config
-local function render_fun(fun, classes, cfg)
-  if fun.access or fun.deprecated or fun.nodoc then
-    return
-  end
-
-  if cfg.fn_name_pat and not fun.name:match(cfg.fn_name_pat) then
-    return
-  end
-
-  if vim.startswith(fun.name, '_') or fun.name:find('[:.]_') then
-    return
-  end
-
-  local ret = {} --- @type string[]
-
-  table.insert(ret, render_fun_header(fun, cfg))
-  table.insert(ret, '\n')
-
-  if fun.since then
-    local since = assert(tonumber(fun.since), 'invalid @since on ' .. fun.name)
-    local info = nvim_api_info()
-    if since == 0 or (info.prerelease and since == info.level) then
-      -- Experimental = (since==0 or current prerelease)
-      local s = 'WARNING: This feature is experimental/unstable.'
-      table.insert(ret, md_to_vimdoc(s, INDENTATION, INDENTATION, TEXT_WIDTH))
-      table.insert(ret, '\n')
-    else
-      local v = assert(util.version_level[since], 'invalid @since on ' .. fun.name)
-      fun.attrs = fun.attrs or {}
-      table.insert(fun.attrs, ('Since: %s'):format(v))
-    end
-  end
-
-  if fun.desc then
-    table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH))
-  end
-
-  if fun.notes then
-    table.insert(ret, '\n    Note: ~\n')
-    for _, p in ipairs(fun.notes) do
-      table.insert(ret, '      • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true))
-    end
-  end
-
-  if fun.attrs then
-    table.insert(ret, '\n    Attributes: ~\n')
-    for _, attr in ipairs(fun.attrs) do
-      local attr_str = ({
-        textlock = 'not allowed when |textlock| is active or in the |cmdwin|',
-        textlock_allow_cmdwin = 'not allowed when |textlock| is active',
-        fast = '|api-fast|',
-        remote_only = '|RPC| only',
-        lua_only = 'Lua |vim.api| only',
-      })[attr] or attr
-      table.insert(ret, fmt('        %s\n', attr_str))
-    end
-  end
-
-  if fun.params and #fun.params > 0 then
-    local param_txt =
-      render_fields_or_params(fun.params, fun.generics, classes, cfg.exclude_types, cfg)
-    if not param_txt:match('^%s*$') then
-      table.insert(ret, '\n    Parameters: ~\n')
-      ret[#ret + 1] = param_txt
-    end
-  end
-
-  if fun.returns then
-    local txt = render_returns(fun.returns, fun.generics, classes, cfg.exclude_types)
-    if not txt:match('^%s*$') then
-      table.insert(ret, '\n')
-      ret[#ret + 1] = txt
-    end
-  end
-
-  if fun.see then
-    table.insert(ret, '\n    See also: ~\n')
-    for _, p in ipairs(fun.see) do
-      table.insert(ret, '      • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true))
-    end
-  end
-
-  table.insert(ret, '\n')
-  return table.concat(ret)
-end
-
---- @param funs nvim.luacats.parser.fun[]
---- @param classes table
---- @param cfg nvim.gen_vimdoc.Config
-local function render_funs(funs, classes, cfg)
-  local ret = {} --- @type string[]
-
-  for _, f in ipairs(funs) do
-    if cfg.fn_xform then
-      cfg.fn_xform(f)
-    end
-    ret[#ret + 1] = render_fun(f, classes, cfg)
-  end
-
-  -- Sort via prototype. Experimental API functions ("nvim__") sort last.
-  table.sort(ret, function(a, b)
-    local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n')
-    local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n')
-
-    local a1__ = a1:find('^%s*nvim__') and 1 or 0
-    local b1__ = b1:find('^%s*nvim__') and 1 or 0
-    if a1__ ~= b1__ then
-      return a1__ < b1__
-    end
-
-    return a1:lower() < b1:lower()
-  end)
-
-  return table.concat(ret)
-end
-
---- @return string
-local function get_script_path()
-  local str = debug.getinfo(2, 'S').source:sub(2)
-  return str:match('(.*[/\\])') or './'
-end
-
-local script_path = get_script_path()
-local base_dir = vim.fs.dirname(vim.fs.dirname(script_path))
-
-local function delete_lines_below(doc_file, tokenstr)
-  local lines = {} --- @type string[]
-  local found = false
-  for line in io.lines(doc_file) do
-    if line:find(vim.pesc(tokenstr)) then
-      found = true
-      break
-    end
-    lines[#lines + 1] = line
-  end
-  if not found then
-    error(fmt('not found: %s in %s', tokenstr, doc_file))
-  end
-  lines[#lines] = nil
-  local fp = assert(io.open(doc_file, 'w'))
-  fp:write(table.concat(lines, '\n'))
-  fp:write('\n')
-  fp:close()
-end
-
---- @param x string
-local function mktitle(x)
-  if x == 'ui' then
-    return 'UI'
-  end
-  return x:sub(1, 1):upper() .. x:sub(2)
-end
-
---- @class nvim.gen_vimdoc.Section
---- @field name string
---- @field title string
---- @field help_tag string
---- @field funs_txt string
---- @field doc? string[]
-
---- @param filename string
---- @param cfg nvim.gen_vimdoc.Config
---- @param section_docs table
---- @param funs_txt string
---- @return nvim.gen_vimdoc.Section?
-local function make_section(filename, cfg, section_docs, funs_txt)
-  -- filename: e.g., 'autocmd.c'
-  -- name: e.g. 'autocmd'
-  local name = filename:match('(.*)%.[a-z]+')
-
-  -- Formatted (this is what's going to be written in the vimdoc)
-  -- e.g., "Autocmd Functions"
-  local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name)
-
-  -- section tag: e.g., "*api-autocmd*"
-  local help_labels = cfg.helptag_fmt(sectname)
-  if type(help_labels) == 'table' then
-    help_labels = table.concat(help_labels, '* *')
-  end
-  local help_tags = '*' .. help_labels .. '*'
-
-  if funs_txt == '' and #section_docs == 0 then
-    return
-  end
-
-  return {
-    name = sectname,
-    title = cfg.section_fmt(sectname),
-    help_tag = help_tags,
-    funs_txt = funs_txt,
-    doc = section_docs,
-  }
-end
-
---- @param section nvim.gen_vimdoc.Section
---- @param add_header? boolean
-local function render_section(section, add_header)
-  local doc = {} --- @type string[]
-
-  if add_header ~= false then
-    vim.list_extend(doc, {
-      string.rep('=', TEXT_WIDTH),
-      '\n',
-      section.title,
-      fmt('%' .. (TEXT_WIDTH - section.title:len()) .. 's', section.help_tag),
-    })
-  end
-
-  local sdoc = '\n\n' .. table.concat(section.doc or {}, '\n')
-  if sdoc:find('[^%s]') then
-    doc[#doc + 1] = sdoc
-  end
-
-  if section.funs_txt then
-    table.insert(doc, '\n\n')
-    table.insert(doc, section.funs_txt)
-  end
-
-  return table.concat(doc)
-end
-
-local parsers = {
-  lua = luacats_parser.parse,
-  c = cdoc_parser.parse,
-  h = cdoc_parser.parse,
-}
-
---- @param files string[]
-local function expand_files(files)
-  for k, f in pairs(files) do
-    if vim.fn.isdirectory(f) == 1 then
-      table.remove(files, k)
-      for path, ty in vim.fs.dir(f) do
-        if ty == 'file' then
-          table.insert(files, vim.fs.joinpath(f, path))
-        end
-      end
-    end
-  end
-end
-
---- @param cfg nvim.gen_vimdoc.Config
-local function gen_target(cfg)
-  cfg.fn_helptag_fmt = cfg.fn_helptag_fmt or fn_helptag_fmt_common
-  print('Target:', cfg.filename)
-  local sections = {} --- @type table
-
-  expand_files(cfg.files)
-
-  --- @type table, nvim.luacats.parser.fun[], string[]]>
-  local file_results = {}
-
-  --- @type table
-  local all_classes = {}
-
-  --- First pass so we can collect all classes
-  for _, f in vim.spairs(cfg.files) do
-    local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']]
-    local parser = assert(parsers[ext])
-    local classes, funs, briefs = parser(f)
-    file_results[f] = { classes, funs, briefs }
-    all_classes = vim.tbl_extend('error', all_classes, classes)
-  end
-
-  for f, r in vim.spairs(file_results) do
-    local classes, funs, briefs = r[1], r[2], r[3]
-
-    local briefs_txt = {} --- @type string[]
-    for _, b in ipairs(briefs) do
-      briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH)
-    end
-    print('    Processing file:', f)
-    local funs_txt = render_funs(funs, all_classes, cfg)
-    if next(classes) then
-      local classes_txt = render_classes(classes, cfg)
-      if vim.trim(classes_txt) ~= '' then
-        funs_txt = classes_txt .. '\n' .. funs_txt
-      end
-    end
-    -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua`
-    local f_base = vim.fs.basename(f)
-    sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt)
-  end
-
-  local first_section_tag = sections[cfg.section_order[1]].help_tag
-  local docs = {} --- @type string[]
-  for _, f in ipairs(cfg.section_order) do
-    local section = sections[f]
-    if section then
-      print(string.format("    Rendering section: '%s'", section.title))
-      local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f)
-      docs[#docs + 1] = render_section(section, add_sep_and_header)
-    end
-  end
-
-  table.insert(
-    docs,
-    fmt(' vim:tw=78:ts=8:sw=%d:sts=%d:et:ft=help:norl:\n', INDENTATION, INDENTATION)
-  )
-
-  local doc_file = vim.fs.joinpath(base_dir, 'runtime', 'doc', cfg.filename)
-
-  if vim.uv.fs_stat(doc_file) then
-    delete_lines_below(doc_file, first_section_tag)
-  end
-
-  local fp = assert(io.open(doc_file, 'a'))
-  fp:write(table.concat(docs, '\n'))
-  fp:close()
-end
-
-local function run()
-  for _, cfg in vim.spairs(config) do
-    gen_target(cfg)
-  end
-end
-
-run()
diff --git a/scripts/lintdoc.lua b/scripts/lintdoc.lua
index 5e78b4cdcb..78cf9fed72 100755
--- a/scripts/lintdoc.lua
+++ b/scripts/lintdoc.lua
@@ -14,7 +14,7 @@ print('Running lintdoc ...')
 -- :helptags checks for duplicate tags.
 vim.cmd [[ helptags ALL ]]
 
-require('scripts.gen_help_html').run_validate()
-require('scripts.gen_help_html').test_gen()
+require('src.gen.gen_help_html').run_validate()
+require('src.gen.gen_help_html').test_gen()
 
 print('lintdoc PASSED.')
diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua
deleted file mode 100644
index b700bcf58f..0000000000
--- a/scripts/luacats_grammar.lua
+++ /dev/null
@@ -1,207 +0,0 @@
---[[!
-LPEG grammar for LuaCATS
-]]
-
-local lpeg = vim.lpeg
-local P, R, S = lpeg.P, lpeg.R, lpeg.S
-local C, Ct, Cg = lpeg.C, lpeg.Ct, lpeg.Cg
-
---- @param x vim.lpeg.Pattern
-local function rep(x)
-  return x ^ 0
-end
-
---- @param x vim.lpeg.Pattern
-local function rep1(x)
-  return x ^ 1
-end
-
---- @param x vim.lpeg.Pattern
-local function opt(x)
-  return x ^ -1
-end
-
-local ws = rep1(S(' \t'))
-local fill = opt(ws)
-local any = P(1) -- (consume one character)
-local letter = R('az', 'AZ')
-local num = R('09')
-
---- @param x string | vim.lpeg.Pattern
-local function Pf(x)
-  return fill * P(x) * fill
-end
-
---- @param x string | vim.lpeg.Pattern
-local function Plf(x)
-  return fill * P(x)
-end
-
---- @param x string
-local function Sf(x)
-  return fill * S(x) * fill
-end
-
---- @param x vim.lpeg.Pattern
-local function paren(x)
-  return Pf('(') * x * fill * P(')')
-end
-
---- @param x vim.lpeg.Pattern
-local function parenOpt(x)
-  return paren(x) + x
-end
-
---- @param x vim.lpeg.Pattern
-local function comma1(x)
-  return parenOpt(x * rep(Pf(',') * x))
-end
-
---- @param x vim.lpeg.Pattern
-local function comma(x)
-  return opt(comma1(x))
-end
-
---- @type table
-local v = setmetatable({}, {
-  __index = function(_, k)
-    return lpeg.V(k)
-  end,
-})
-
---- @class nvim.luacats.Param
---- @field kind 'param'
---- @field name string
---- @field type string
---- @field desc? string
-
---- @class nvim.luacats.Return
---- @field kind 'return'
---- @field [integer] { type: string, name?: string}
---- @field desc? string
-
---- @class nvim.luacats.Generic
---- @field kind 'generic'
---- @field name string
---- @field type? string
-
---- @class nvim.luacats.Class
---- @field kind 'class'
---- @field name string
---- @field parent? string
---- @field access? 'private'|'protected'|'package'
-
---- @class nvim.luacats.Field
---- @field kind 'field'
---- @field name string
---- @field type string
---- @field desc? string
---- @field access? 'private'|'protected'|'package'
-
---- @class nvim.luacats.Note
---- @field desc? string
-
---- @alias nvim.luacats.grammar.result
---- | nvim.luacats.Param
---- | nvim.luacats.Return
---- | nvim.luacats.Generic
---- | nvim.luacats.Class
---- | nvim.luacats.Field
---- | nvim.luacats.Note
-
---- @class nvim.luacats.grammar
---- @field match fun(self, input: string): nvim.luacats.grammar.result?
-
-local function annot(nm, pat)
-  if type(nm) == 'string' then
-    nm = P(nm)
-  end
-  if pat then
-    return Ct(Cg(P(nm), 'kind') * fill * pat)
-  end
-  return Ct(Cg(P(nm), 'kind'))
-end
-
-local colon = Pf(':')
-local ellipsis = P('...')
-local ident_first = P('_') + letter
-local ident = ident_first * rep(ident_first + num)
-local opt_ident = ident * opt(P('?'))
-local ty_ident_sep = S('-._')
-local ty_ident = ident * rep(ty_ident_sep * ident)
-local string_single = P "'" * rep(any - P "'") * P "'"
-local string_double = P('"') * rep(any - P('"')) * P('"')
-local generic = P('`') * ty_ident * P('`')
-local literal = string_single + string_double + (opt(P('-')) * rep1(num)) + P('false') + P('true')
-local ty_prims = ty_ident + literal + generic
-
-local array_postfix = rep1(Plf('[]'))
-local opt_postfix = rep1(Plf('?'))
-local rep_array_opt_postfix = rep(array_postfix + opt_postfix)
-
-local typedef = P({
-  'typedef',
-  typedef = C(v.type),
-
-  type = v.ty * rep_array_opt_postfix * rep(Pf('|') * v.ty * rep_array_opt_postfix),
-  ty = v.composite + paren(v.typedef),
-  composite = (v.types * array_postfix) + (v.types * opt_postfix) + v.types,
-  types = v.generics + v.kv_table + v.tuple + v.dict + v.table_literal + v.fun + ty_prims,
-
-  tuple = Pf('[') * comma1(v.type) * Plf(']'),
-  dict = Pf('{') * comma1(Pf('[') * v.type * Pf(']') * colon * v.type) * Plf('}'),
-  kv_table = Pf('table') * Pf('<') * v.type * Pf(',') * v.type * Plf('>'),
-  table_literal = Pf('{') * comma1(opt_ident * Pf(':') * v.type) * Plf('}'),
-  fun_param = (opt_ident + ellipsis) * opt(colon * v.type),
-  fun_ret = v.type + (ellipsis * opt(colon * v.type)),
-  fun = Pf('fun') * paren(comma(v.fun_param)) * opt(Pf(':') * comma1(v.fun_ret)),
-  generics = P(ty_ident) * Pf('<') * comma1(v.type) * Plf('>'),
-}) / function(match)
-  return vim.trim(match):gsub('^%((.*)%)$', '%1'):gsub('%?+', '?')
-end
-
-local access = P('private') + P('protected') + P('package')
-local caccess = Cg(access, 'access')
-local cattr = Cg(comma(access + P('exact')), 'access')
-local desc_delim = Sf '#:' + ws
-local desc = Cg(rep(any), 'desc')
-local opt_desc = opt(desc_delim * desc)
-local ty_name = Cg(ty_ident, 'name')
-local opt_parent = opt(colon * Cg(ty_ident, 'parent'))
-local lname = (ident + ellipsis) * opt(P('?'))
-
-local grammar = P {
-  rep1(P('@') * (v.ats + v.ext_ats)),
-
-  ats = annot('param', Cg(lname, 'name') * ws * v.ctype * opt_desc)
-    + annot('return', comma1(Ct(v.ctype * opt(ws * (ty_name + Cg(ellipsis, 'name'))))) * opt_desc)
-    + annot('type', comma1(Ct(v.ctype)) * opt_desc)
-    + annot('cast', ty_name * ws * opt(Sf('+-')) * v.ctype)
-    + annot('generic', ty_name * opt(colon * v.ctype))
-    + annot('class', opt(paren(cattr)) * fill * ty_name * opt_parent)
-    + annot('field', opt(caccess * ws) * v.field_name * ws * v.ctype * opt_desc)
-    + annot('operator', ty_name * opt(paren(Cg(v.ctype, 'argtype'))) * colon * v.ctype)
-    + annot(access)
-    + annot('deprecated')
-    + annot('alias', ty_name * opt(ws * v.ctype))
-    + annot('enum', ty_name)
-    + annot('overload', v.ctype)
-    + annot('see', opt(desc_delim) * desc)
-    + annot('diagnostic', opt(desc_delim) * desc)
-    + annot('meta'),
-
-  --- Custom extensions
-  ext_ats = (
-    annot('note', desc)
-    + annot('since', desc)
-    + annot('nodoc')
-    + annot('inlinedoc')
-    + annot('brief', desc)
-  ),
-
-  field_name = Cg(lname + (v.ty_index * opt(P('?'))), 'name'),
-  ty_index = C(Pf('[') * typedef * fill * P(']')),
-  ctype = Cg(typedef, 'type'),
-}
-
-return grammar --[[@as nvim.luacats.grammar]]
diff --git a/scripts/luacats_parser.lua b/scripts/luacats_parser.lua
deleted file mode 100644
index 8a50077aa8..0000000000
--- a/scripts/luacats_parser.lua
+++ /dev/null
@@ -1,535 +0,0 @@
-local luacats_grammar = require('scripts.luacats_grammar')
-
---- @class nvim.luacats.parser.param : nvim.luacats.Param
-
---- @class nvim.luacats.parser.return
---- @field name string
---- @field type string
---- @field desc string
-
---- @class nvim.luacats.parser.note
---- @field desc string
-
---- @class nvim.luacats.parser.brief
---- @field kind 'brief'
---- @field desc string
-
---- @class nvim.luacats.parser.alias
---- @field kind 'alias'
---- @field type string[]
---- @field desc string
-
---- @class nvim.luacats.parser.fun
---- @field name string
---- @field params nvim.luacats.parser.param[]
---- @field returns nvim.luacats.parser.return[]
---- @field desc string
---- @field access? 'private'|'package'|'protected'
---- @field class? string
---- @field module? string
---- @field modvar? string
---- @field classvar? string
---- @field deprecated? true
---- @field since? string
---- @field attrs? string[]
---- @field nodoc? true
---- @field generics? table
---- @field table? true
---- @field notes? nvim.luacats.parser.note[]
---- @field see? nvim.luacats.parser.note[]
-
---- @class nvim.luacats.parser.field : nvim.luacats.Field
---- @field classvar? string
---- @field nodoc? true
-
---- @class nvim.luacats.parser.class : nvim.luacats.Class
---- @field desc? string
---- @field nodoc? true
---- @field inlinedoc? true
---- @field fields nvim.luacats.parser.field[]
---- @field notes? string[]
-
---- @class nvim.luacats.parser.State
---- @field doc_lines? string[]
---- @field cur_obj? nvim.luacats.parser.obj
---- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note
---- @field last_doc_item_indent? integer
-
---- @alias nvim.luacats.parser.obj
---- | nvim.luacats.parser.class
---- | nvim.luacats.parser.fun
---- | nvim.luacats.parser.brief
---- | nvim.luacats.parser.alias
-
--- Remove this when we document classes properly
---- Some doc lines have the form:
----   param name some.complex.type (table) description
---- if so then transform the line to remove the complex type:
----   param name (table) description
---- @param line string
-local function use_type_alt(line)
-  for _, type in ipairs({ 'table', 'function' }) do
-    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2')
-    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2')
-    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2')
-
-    line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1')
-    line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1')
-    line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1')
-  end
-  return line
-end
-
---- If we collected any `---` lines. Add them to the existing (or new) object
---- Used for function/class descriptions and multiline param descriptions.
---- @param state nvim.luacats.parser.State
-local function add_doc_lines_to_obj(state)
-  if state.doc_lines then
-    state.cur_obj = state.cur_obj or {}
-    local cur_obj = assert(state.cur_obj)
-    local txt = table.concat(state.doc_lines, '\n')
-    if cur_obj.desc then
-      cur_obj.desc = cur_obj.desc .. '\n' .. txt
-    else
-      cur_obj.desc = txt
-    end
-    state.doc_lines = nil
-  end
-end
-
---- @param line string
---- @param state nvim.luacats.parser.State
-local function process_doc_line(line, state)
-  line = line:sub(4):gsub('^%s+@', '@')
-  line = use_type_alt(line)
-
-  local parsed = luacats_grammar:match(line)
-
-  if not parsed then
-    if line:match('^ ') then
-      line = line:sub(2)
-    end
-
-    if state.last_doc_item then
-      if not state.last_doc_item_indent then
-        state.last_doc_item_indent = #line:match('^%s*') + 1
-      end
-      state.last_doc_item.desc = (state.last_doc_item.desc or '')
-        .. '\n'
-        .. line:sub(state.last_doc_item_indent or 1)
-    else
-      state.doc_lines = state.doc_lines or {}
-      table.insert(state.doc_lines, line)
-    end
-    return
-  end
-
-  state.last_doc_item_indent = nil
-  state.last_doc_item = nil
-  state.cur_obj = state.cur_obj or {}
-  local cur_obj = assert(state.cur_obj)
-
-  local kind = parsed.kind
-
-  if kind == 'brief' then
-    state.cur_obj = {
-      kind = 'brief',
-      desc = parsed.desc,
-    }
-  elseif kind == 'class' then
-    --- @cast parsed nvim.luacats.Class
-    cur_obj.kind = 'class'
-    cur_obj.name = parsed.name
-    cur_obj.parent = parsed.parent
-    cur_obj.access = parsed.access
-    cur_obj.desc = state.doc_lines and table.concat(state.doc_lines, '\n') or nil
-    state.doc_lines = nil
-    cur_obj.fields = {}
-  elseif kind == 'field' then
-    --- @cast parsed nvim.luacats.Field
-    parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil
-    if parsed.desc then
-      parsed.desc = vim.trim(parsed.desc)
-    end
-    table.insert(cur_obj.fields, parsed)
-    state.doc_lines = nil
-  elseif kind == 'operator' then
-    parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil
-    if parsed.desc then
-      parsed.desc = vim.trim(parsed.desc)
-    end
-    table.insert(cur_obj.fields, parsed)
-    state.doc_lines = nil
-  elseif kind == 'param' then
-    state.last_doc_item_indent = nil
-    cur_obj.params = cur_obj.params or {}
-    if vim.endswith(parsed.name, '?') then
-      parsed.name = parsed.name:sub(1, -2)
-      parsed.type = parsed.type .. '?'
-    end
-    state.last_doc_item = {
-      name = parsed.name,
-      type = parsed.type,
-      desc = parsed.desc,
-    }
-    table.insert(cur_obj.params, state.last_doc_item)
-  elseif kind == 'return' then
-    cur_obj.returns = cur_obj.returns or {}
-    for _, t in ipairs(parsed) do
-      table.insert(cur_obj.returns, {
-        name = t.name,
-        type = t.type,
-        desc = parsed.desc,
-      })
-    end
-    state.last_doc_item_indent = nil
-    state.last_doc_item = cur_obj.returns[#cur_obj.returns]
-  elseif kind == 'private' then
-    cur_obj.access = 'private'
-  elseif kind == 'package' then
-    cur_obj.access = 'package'
-  elseif kind == 'protected' then
-    cur_obj.access = 'protected'
-  elseif kind == 'deprecated' then
-    cur_obj.deprecated = true
-  elseif kind == 'inlinedoc' then
-    cur_obj.inlinedoc = true
-  elseif kind == 'nodoc' then
-    cur_obj.nodoc = true
-  elseif kind == 'since' then
-    cur_obj.since = parsed.desc
-  elseif kind == 'see' then
-    cur_obj.see = cur_obj.see or {}
-    table.insert(cur_obj.see, { desc = parsed.desc })
-  elseif kind == 'note' then
-    state.last_doc_item_indent = nil
-    state.last_doc_item = {
-      desc = parsed.desc,
-    }
-    cur_obj.notes = cur_obj.notes or {}
-    table.insert(cur_obj.notes, state.last_doc_item)
-  elseif kind == 'type' then
-    cur_obj.desc = parsed.desc
-    parsed.desc = nil
-    parsed.kind = nil
-    cur_obj.type = parsed
-  elseif kind == 'alias' then
-    state.cur_obj = {
-      kind = 'alias',
-      desc = parsed.desc,
-    }
-  elseif kind == 'enum' then
-    -- TODO
-    state.doc_lines = nil
-  elseif
-    vim.tbl_contains({
-      'diagnostic',
-      'cast',
-      'overload',
-      'meta',
-    }, kind)
-  then
-    -- Ignore
-    return
-  elseif kind == 'generic' then
-    cur_obj.generics = cur_obj.generics or {}
-    cur_obj.generics[parsed.name] = parsed.type or 'any'
-  else
-    error('Unhandled' .. vim.inspect(parsed))
-  end
-end
-
---- @param fun nvim.luacats.parser.fun
---- @return nvim.luacats.parser.field
-local function fun2field(fun)
-  local parts = { 'fun(' }
-
-  local params = {} ---@type string[]
-  for _, p in ipairs(fun.params or {}) do
-    params[#params + 1] = string.format('%s: %s', p.name, p.type)
-  end
-  parts[#parts + 1] = table.concat(params, ', ')
-  parts[#parts + 1] = ')'
-  if fun.returns then
-    parts[#parts + 1] = ': '
-    local tys = {} --- @type string[]
-    for _, p in ipairs(fun.returns) do
-      tys[#tys + 1] = p.type
-    end
-    parts[#parts + 1] = table.concat(tys, ', ')
-  end
-
-  return {
-    name = fun.name,
-    type = table.concat(parts, ''),
-    access = fun.access,
-    desc = fun.desc,
-    nodoc = fun.nodoc,
-  }
-end
-
---- Function to normalize known form for declaring functions and normalize into a more standard
---- form.
---- @param line string
---- @return string
-local function filter_decl(line)
-  -- M.fun = vim._memoize(function(...)
-  --   ->
-  -- function M.fun(...)
-  line = line:gsub('^local (.+) = memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)')
-  line = line:gsub('^(.+) = memoize%([^,]+, function%((.*)%)$', 'function %1(%2)')
-  return line
-end
-
---- @param line string
---- @param state nvim.luacats.parser.State
---- @param classes table
---- @param classvars table
---- @param has_indent boolean
-local function process_lua_line(line, state, classes, classvars, has_indent)
-  line = filter_decl(line)
-
-  if state.cur_obj and state.cur_obj.kind == 'class' then
-    local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=')
-    if nm then
-      classvars[nm] = state.cur_obj.name
-    end
-    return
-  end
-
-  do
-    local parent_tbl, sep, fun_or_meth_nm =
-      line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(')
-    if parent_tbl then
-      -- Have a decl. Ensure cur_obj
-      state.cur_obj = state.cur_obj or {}
-      local cur_obj = assert(state.cur_obj)
-
-      -- Match `Class:foo` methods for defined classes
-      local class = classvars[parent_tbl]
-      if class then
-        --- @cast cur_obj nvim.luacats.parser.fun
-        cur_obj.name = fun_or_meth_nm
-        cur_obj.class = class
-        cur_obj.classvar = parent_tbl
-        -- Add self param to methods
-        if sep == ':' then
-          cur_obj.params = cur_obj.params or {}
-          table.insert(cur_obj.params, 1, {
-            name = 'self',
-            type = class,
-          })
-        end
-
-        -- Add method as the field to the class
-        local cls = classes[class]
-        local field = fun2field(cur_obj)
-        field.classvar = cur_obj.classvar
-        table.insert(cls.fields, field)
-        return
-      end
-
-      -- Match `M.foo`
-      if cur_obj and parent_tbl == cur_obj.modvar then
-        cur_obj.name = fun_or_meth_nm
-        return
-      end
-    end
-  end
-
-  do
-    -- Handle: `function A.B.C.foo(...)`
-    local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(')
-    if fn_nm then
-      state.cur_obj = state.cur_obj or {}
-      state.cur_obj.name = fn_nm
-      return
-    end
-  end
-
-  do
-    -- Handle: `M.foo = {...}` where `M` is the modvar
-    local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=')
-    if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then
-      state.cur_obj.name = tbl_nm
-      state.cur_obj.table = true
-      return
-    end
-  end
-
-  do
-    -- Handle: `foo = {...}`
-    local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=')
-    if tbl_nm and not has_indent then
-      state.cur_obj = state.cur_obj or {}
-      state.cur_obj.name = tbl_nm
-      state.cur_obj.table = true
-      return
-    end
-  end
-
-  do
-    -- Handle: `vim.foo = {...}`
-    local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=')
-    if state.cur_obj and tbl_nm and not has_indent then
-      state.cur_obj.name = tbl_nm
-      state.cur_obj.table = true
-      return
-    end
-  end
-
-  if state.cur_obj then
-    if line:find('^%s*%-%- luacheck:') then
-      state.cur_obj = nil
-    elseif line:find('^%s*local%s+') then
-      state.cur_obj = nil
-    elseif line:find('^%s*return%s+') then
-      state.cur_obj = nil
-    elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then
-      state.cur_obj = nil
-    end
-  end
-end
-
---- Determine the table name used to export functions of a module
---- Usually this is `M`.
---- @param str string
---- @return string?
-local function determine_modvar(str)
-  local modvar --- @type string?
-  for line in vim.gsplit(str, '\n') do
-    do
-      --- @type string?
-      local m = line:match('^return%s+([a-zA-Z_]+)')
-      if m then
-        modvar = m
-      end
-    end
-    do
-      --- @type string?
-      local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),')
-      if m then
-        modvar = m
-      end
-    end
-  end
-  return modvar
-end
-
---- @param obj nvim.luacats.parser.obj
---- @param funs nvim.luacats.parser.fun[]
---- @param classes table
---- @param briefs string[]
---- @param uncommitted nvim.luacats.parser.obj[]
-local function commit_obj(obj, classes, funs, briefs, uncommitted)
-  local commit = false
-  if obj.kind == 'class' then
-    --- @cast obj nvim.luacats.parser.class
-    if not classes[obj.name] then
-      classes[obj.name] = obj
-      commit = true
-    end
-  elseif obj.kind == 'alias' then
-    -- Just pretend
-    commit = true
-  elseif obj.kind == 'brief' then
-    --- @cast obj nvim.luacats.parser.brief`
-    briefs[#briefs + 1] = obj.desc
-    commit = true
-  else
-    --- @cast obj nvim.luacats.parser.fun`
-    if obj.name then
-      funs[#funs + 1] = obj
-      commit = true
-    end
-  end
-  if not commit then
-    table.insert(uncommitted, obj)
-  end
-  return commit
-end
-
---- @param filename string
---- @param uncommitted nvim.luacats.parser.obj[]
--- luacheck: no unused
-local function dump_uncommitted(filename, uncommitted)
-  local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt'
-  if #uncommitted > 0 then
-    print(string.format('Could not commit %d objects in %s', #uncommitted, filename))
-    vim.fn.mkdir(vim.fs.dirname(out_path), 'p')
-    local f = assert(io.open(out_path, 'w'))
-    for i, x in ipairs(uncommitted) do
-      f:write(i)
-      f:write(': ')
-      f:write(vim.inspect(x))
-      f:write('\n')
-    end
-    f:close()
-  else
-    vim.fn.delete(out_path)
-  end
-end
-
-local M = {}
-
-function M.parse_str(str, filename)
-  local funs = {} --- @type nvim.luacats.parser.fun[]
-  local classes = {} --- @type table
-  local briefs = {} --- @type string[]
-
-  local mod_return = determine_modvar(str)
-
-  --- @type string
-  local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename
-  module = module:gsub('/', '.')
-
-  local classvars = {} --- @type table
-
-  local state = {} --- @type nvim.luacats.parser.State
-
-  -- Keep track of any partial objects we don't commit
-  local uncommitted = {} --- @type nvim.luacats.parser.obj[]
-
-  for line in vim.gsplit(str, '\n') do
-    local has_indent = line:match('^%s+') ~= nil
-    line = vim.trim(line)
-    if vim.startswith(line, '---') then
-      process_doc_line(line, state)
-    else
-      add_doc_lines_to_obj(state)
-
-      if state.cur_obj then
-        state.cur_obj.modvar = mod_return
-        state.cur_obj.module = module
-      end
-
-      process_lua_line(line, state, classes, classvars, has_indent)
-
-      -- Commit the object
-      local cur_obj = state.cur_obj
-      if cur_obj then
-        if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then
-          --- @diagnostic disable-next-line:inject-field
-          cur_obj.line = line
-        end
-      end
-
-      state = {}
-    end
-  end
-
-  -- dump_uncommitted(filename, uncommitted)
-
-  return classes, funs, briefs, uncommitted
-end
-
---- @param filename string
-function M.parse(filename)
-  local f = assert(io.open(filename, 'r'))
-  local txt = f:read('*all')
-  f:close()
-
-  return M.parse_str(txt, filename)
-end
-
-return M
diff --git a/scripts/release.sh b/scripts/release.sh
index 257fa127c4..58acbf85a0 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -59,7 +59,7 @@ _do_release_commit() {
     $__sed -i.bk 's/(NVIM_API_PRERELEASE) true/\1 false/' CMakeLists.txt
     build/bin/nvim --api-info > "test/functional/fixtures/api_level_$__API_LEVEL.mpack"
     git add "test/functional/fixtures/api_level_${__API_LEVEL}.mpack"
-    VIMRUNTIME=./runtime build/bin/nvim -u NONE -l scripts/gen_vimdoc.lua
+    VIMRUNTIME=./runtime build/bin/nvim -u NONE -l src/gen/preload.lua src/gen/gen_vimdoc.lua
     git add -u -- runtime/doc/
   fi
 
diff --git a/scripts/util.lua b/scripts/util.lua
deleted file mode 100644
index 5940221abe..0000000000
--- a/scripts/util.lua
+++ /dev/null
@@ -1,399 +0,0 @@
--- TODO(justinmk): move most of this to `vim.text`.
-
-local fmt = string.format
-
---- @class nvim.util.MDNode
---- @field [integer] nvim.util.MDNode
---- @field type string
---- @field text? string
-
-local INDENTATION = 4
-
-local NBSP = string.char(160)
-
-local M = {}
-
-local function contains(t, xs)
-  return vim.tbl_contains(xs, t)
-end
-
--- Map of api_level:version, by inspection of:
---    :lua= vim.mpack.decode(vim.fn.readfile('test/functional/fixtures/api_level_9.mpack','B')).version
-M.version_level = {
-  [13] = '0.11.0',
-  [12] = '0.10.0',
-  [11] = '0.9.0',
-  [10] = '0.8.0',
-  [9] = '0.7.0',
-  [8] = '0.6.0',
-  [7] = '0.5.0',
-  [6] = '0.4.0',
-  [5] = '0.3.2',
-  [4] = '0.3.0',
-  [3] = '0.2.1',
-  [2] = '0.2.0',
-  [1] = '0.1.0',
-}
-
---- @param txt string
---- @param srow integer
---- @param scol integer
---- @param erow? integer
---- @param ecol? integer
---- @return string
-local function slice_text(txt, srow, scol, erow, ecol)
-  local lines = vim.split(txt, '\n')
-
-  if srow == erow then
-    return lines[srow + 1]:sub(scol + 1, ecol)
-  end
-
-  if erow then
-    -- Trim the end
-    for _ = erow + 2, #lines do
-      table.remove(lines, #lines)
-    end
-  end
-
-  -- Trim the start
-  for _ = 1, srow do
-    table.remove(lines, 1)
-  end
-
-  lines[1] = lines[1]:sub(scol + 1)
-  lines[#lines] = lines[#lines]:sub(1, ecol)
-
-  return table.concat(lines, '\n')
-end
-
---- @param text string
---- @return nvim.util.MDNode
-local function parse_md_inline(text)
-  local parser = vim.treesitter.languagetree.new(text, 'markdown_inline')
-  local root = parser:parse(true)[1]:root()
-
-  --- @param node TSNode
-  --- @return nvim.util.MDNode?
-  local function extract(node)
-    local ntype = node:type()
-
-    if ntype:match('^%p$') then
-      return
-    end
-
-    --- @type table
-    local ret = { type = ntype }
-    ret.text = vim.treesitter.get_node_text(node, text)
-
-    local row, col = 0, 0
-
-    for child, child_field in node:iter_children() do
-      local e = extract(child)
-      if e and ntype == 'inline' then
-        local srow, scol = child:start()
-        if (srow == row and scol > col) or srow > row then
-          local t = slice_text(ret.text, row, col, srow, scol)
-          if t and t ~= '' then
-            table.insert(ret, { type = 'text', j = true, text = t })
-          end
-        end
-        row, col = child:end_()
-      end
-
-      if child_field then
-        ret[child_field] = e
-      else
-        table.insert(ret, e)
-      end
-    end
-
-    if ntype == 'inline' and (row > 0 or col > 0) then
-      local t = slice_text(ret.text, row, col)
-      if t and t ~= '' then
-        table.insert(ret, { type = 'text', text = t })
-      end
-    end
-
-    return ret
-  end
-
-  return extract(root) or {}
-end
-
---- @param text string
---- @return nvim.util.MDNode
-local function parse_md(text)
-  local parser = vim.treesitter.languagetree.new(text, 'markdown', {
-    injections = { markdown = '' },
-  })
-
-  local root = parser:parse(true)[1]:root()
-
-  local EXCLUDE_TEXT_TYPE = {
-    list = true,
-    list_item = true,
-    section = true,
-    document = true,
-    fenced_code_block = true,
-    fenced_code_block_delimiter = true,
-  }
-
-  --- @param node TSNode
-  --- @return nvim.util.MDNode?
-  local function extract(node)
-    local ntype = node:type()
-
-    if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then
-      return
-    end
-
-    --- @type table
-    local ret = { type = ntype }
-
-    if not EXCLUDE_TEXT_TYPE[ntype] then
-      ret.text = vim.treesitter.get_node_text(node, text)
-    end
-
-    if ntype == 'inline' then
-      ret = parse_md_inline(ret.text)
-    end
-
-    for child, child_field in node:iter_children() do
-      local e = extract(child)
-      if child_field then
-        ret[child_field] = e
-      else
-        table.insert(ret, e)
-      end
-    end
-
-    return ret
-  end
-
-  return extract(root) or {}
-end
-
---- Prefixes each line in `text`.
----
---- Does not wrap, not important for "meta" files? (You probably want md_to_vimdoc instead.)
----
---- @param text string
---- @param prefix_ string
-function M.prefix_lines(prefix_, text)
-  local r = ''
-  for _, l in ipairs(vim.split(text, '\n', { plain = true })) do
-    r = r .. vim.trim(prefix_ .. l) .. '\n'
-  end
-  return r
-end
-
---- @param x string
---- @param start_indent integer
---- @param indent integer
---- @param text_width integer
---- @return string
-function M.wrap(x, start_indent, indent, text_width)
-  local words = vim.split(vim.trim(x), '%s+')
-  local parts = { string.rep(' ', start_indent) } --- @type string[]
-  local count = indent
-
-  for i, w in ipairs(words) do
-    if count > indent and count + #w > text_width - 1 then
-      parts[#parts + 1] = '\n'
-      parts[#parts + 1] = string.rep(' ', indent)
-      count = indent
-    elseif i ~= 1 then
-      parts[#parts + 1] = ' '
-      count = count + 1
-    end
-    count = count + #w
-    parts[#parts + 1] = w
-  end
-
-  return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', ''))
-end
-
---- @param node nvim.util.MDNode
---- @param start_indent integer
---- @param indent integer
---- @param text_width integer
---- @param level integer
---- @return string[]
-local function render_md(node, start_indent, indent, text_width, level, is_list)
-  local parts = {} --- @type string[]
-
-  -- For debugging
-  local add_tag = false
-  -- local add_tag = true
-
-  local ntype = node.type
-
-  if add_tag then
-    parts[#parts + 1] = '<' .. ntype .. '>'
-  end
-
-  if ntype == 'text' then
-    parts[#parts + 1] = node.text
-  elseif ntype == 'html_tag' then
-    error('html_tag: ' .. node.text)
-  elseif ntype == 'inline_link' then
-    vim.list_extend(parts, { '*', node[1].text, '*' })
-  elseif ntype == 'shortcut_link' then
-    if node[1].text:find('^<.*>$') then
-      parts[#parts + 1] = node[1].text
-    elseif node[1].text:find('^%d+$') then
-      vim.list_extend(parts, { '[', node[1].text, ']' })
-    else
-      vim.list_extend(parts, { '|', node[1].text, '|' })
-    end
-  elseif ntype == 'backslash_escape' then
-    parts[#parts + 1] = node.text
-  elseif ntype == 'emphasis' then
-    parts[#parts + 1] = node.text:sub(2, -2)
-  elseif ntype == 'code_span' then
-    vim.list_extend(parts, { '`', node.text:sub(2, -2):gsub(' ', NBSP), '`' })
-  elseif ntype == 'inline' then
-    if #node == 0 then
-      local text = assert(node.text)
-      parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width)
-    else
-      for _, child in ipairs(node) do
-        vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1))
-      end
-    end
-  elseif ntype == 'paragraph' then
-    local pparts = {}
-    for _, child in ipairs(node) do
-      vim.list_extend(pparts, render_md(child, start_indent, indent, text_width, level + 1))
-    end
-    parts[#parts + 1] = M.wrap(table.concat(pparts), start_indent, indent, text_width)
-    parts[#parts + 1] = '\n'
-  elseif ntype == 'code_fence_content' then
-    local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n')
-
-    local cindent = indent + INDENTATION
-    if level > 3 then
-      -- The tree-sitter markdown parser doesn't parse the code blocks indents
-      -- correctly in lists. Fudge it!
-      lines[1] = '    ' .. lines[1] -- ¯\_(ツ)_/¯
-      cindent = indent - level
-      local _, initial_indent = lines[1]:find('^%s*')
-      initial_indent = initial_indent + cindent
-      if initial_indent < indent then
-        cindent = indent - INDENTATION
-      end
-    end
-
-    for _, l in ipairs(lines) do
-      if #l > 0 then
-        parts[#parts + 1] = string.rep(' ', cindent)
-        parts[#parts + 1] = l
-      end
-      parts[#parts + 1] = '\n'
-    end
-  elseif ntype == 'fenced_code_block' then
-    parts[#parts + 1] = '>'
-    for _, child in ipairs(node) do
-      if child.type == 'info_string' then
-        parts[#parts + 1] = child.text
-        break
-      end
-    end
-    parts[#parts + 1] = '\n'
-    for _, child in ipairs(node) do
-      if child.type ~= 'info_string' then
-        vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1))
-      end
-    end
-    parts[#parts + 1] = '<\n'
-  elseif ntype == 'html_block' then
-    local text = node.text:gsub('^
help', '')
-    text = text:gsub('
%s*$', '')
-    parts[#parts + 1] = text
-  elseif ntype == 'list_marker_dot' then
-    parts[#parts + 1] = node.text
-  elseif contains(ntype, { 'list_marker_minus', 'list_marker_star' }) then
-    parts[#parts + 1] = '• '
-  elseif ntype == 'list_item' then
-    parts[#parts + 1] = string.rep(' ', indent)
-    local offset = node[1].type == 'list_marker_dot' and 3 or 2
-    for i, child in ipairs(node) do
-      local sindent = i <= 2 and 0 or (indent + offset)
-      vim.list_extend(
-        parts,
-        render_md(child, sindent, indent + offset, text_width, level + 1, true)
-      )
-    end
-  else
-    if node.text then
-      error(fmt('cannot render:\n%s', vim.inspect(node)))
-    end
-    for i, child in ipairs(node) do
-      local start_indent0 = i == 1 and start_indent or indent
-      vim.list_extend(
-        parts,
-        render_md(child, start_indent0, indent, text_width, level + 1, is_list)
-      )
-      if ntype ~= 'list' and i ~= #node then
-        if (node[i + 1] or {}).type ~= 'list' then
-          parts[#parts + 1] = '\n'
-        end
-      end
-    end
-  end
-
-  if add_tag then
-    parts[#parts + 1] = ''
-  end
-
-  return parts
-end
-
---- @param text_width integer
-local function align_tags(text_width)
-  --- @param line string
-  --- @return string
-  return function(line)
-    local tag_pat = '%s*(%*.+%*)%s*$'
-    local tags = {}
-    for m in line:gmatch(tag_pat) do
-      table.insert(tags, m)
-    end
-
-    if #tags > 0 then
-      line = line:gsub(tag_pat, '')
-      local tags_str = ' ' .. table.concat(tags, ' ')
-      --- @type integer
-      local conceal_offset = select(2, tags_str:gsub('%*', '')) - 2
-      local pad = string.rep(' ', text_width - #line - #tags_str + conceal_offset)
-      return line .. pad .. tags_str
-    end
-
-    return line
-  end
-end
-
---- @param text string
---- @param start_indent integer
---- @param indent integer
---- @param is_list? boolean
---- @return string
-function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list)
-  -- Add an extra newline so the parser can properly capture ending ```
-  local parsed = parse_md(text .. '\n')
-  local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list)
-
-  local lines = vim.split(table.concat(ret):gsub(NBSP, ' '), '\n')
-
-  lines = vim.tbl_map(align_tags(text_width), lines)
-
-  local s = table.concat(lines, '\n')
-
-  -- Reduce whitespace in code-blocks
-  s = s:gsub('\n+%s*>([a-z]+)\n', ' >%1\n')
-  s = s:gsub('\n+%s*>\n?\n', ' >\n')
-
-  return s
-end
-
-return M
-- 
cgit