From 0f24b0826a27b7868a3aacc25199787e7453d4cc Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Wed, 26 Feb 2025 11:38:07 +0000 Subject: build: move all generator scripts to `src/gen/` - Move all generator Lua scripts to the `src/gen/` - Add a `.luarc.json` to `src/gen/` - Add a `preload.lua` to `src/gen/` - Add `src` to `package.path` so it aligns with `.luarc.json' - Fix all `require` statements in `src/gen/` so they are consistent: - `require('scripts.foo')` -> `require('gen.foo')` - `require('src.nvim.options')` -> `require('nvim.options')` - `require('api.dispatch_deprecated')` -> `require('nvim.api.dispatch_deprecated')` --- src/.luarc.json | 24 + src/gen/c_grammar.lua | 300 ++++++ src/gen/cdoc_grammar.lua | 87 ++ src/gen/cdoc_parser.lua | 223 +++++ src/gen/dump_bin_array.lua | 17 + src/gen/gen_api_dispatch.lua | 990 +++++++++++++++++++ src/gen/gen_api_ui_events.lua | 219 +++++ src/gen/gen_char_blob.lua | 96 ++ src/gen/gen_declarations.lua | 186 ++++ src/gen/gen_eval.lua | 112 +++ src/gen/gen_eval_files.lua | 1090 +++++++++++++++++++++ src/gen/gen_events.lua | 42 + src/gen/gen_ex_cmds.lua | 194 ++++ src/gen/gen_filetype.lua | 209 ++++ src/gen/gen_help_html.lua | 1491 +++++++++++++++++++++++++++++ src/gen/gen_lsp.lua | 514 ++++++++++ src/gen/gen_options.lua | 535 +++++++++++ src/gen/gen_vimdoc.lua | 1041 ++++++++++++++++++++ src/gen/gen_vimvim.lua | 156 +++ src/gen/hashy.lua | 145 +++ src/gen/luacats_grammar.lua | 207 ++++ src/gen/luacats_parser.lua | 535 +++++++++++ src/gen/nvim_version.lua.in | 9 + src/gen/preload.lua | 6 + src/gen/preload_nlua.lua | 17 + src/gen/util.lua | 399 ++++++++ src/nvim/CMakeLists.txt | 18 +- src/nvim/func_attr.h | 2 +- src/nvim/generators/c_grammar.lua | 300 ------ src/nvim/generators/dump_bin_array.lua | 17 - src/nvim/generators/gen_api_dispatch.lua | 990 ------------------- src/nvim/generators/gen_api_ui_events.lua | 219 ----- src/nvim/generators/gen_char_blob.lua | 96 -- src/nvim/generators/gen_declarations.lua | 186 ---- src/nvim/generators/gen_eval.lua | 112 --- src/nvim/generators/gen_events.lua | 42 - src/nvim/generators/gen_ex_cmds.lua | 194 ---- src/nvim/generators/gen_options.lua | 535 ----------- src/nvim/generators/gen_vimvim.lua | 156 --- src/nvim/generators/hashy.lua | 145 --- src/nvim/generators/nvim_version.lua.in | 9 - src/nvim/generators/preload.lua | 13 - 42 files changed, 8856 insertions(+), 3022 deletions(-) create mode 100644 src/.luarc.json create mode 100644 src/gen/c_grammar.lua create mode 100644 src/gen/cdoc_grammar.lua create mode 100644 src/gen/cdoc_parser.lua create mode 100644 src/gen/dump_bin_array.lua create mode 100644 src/gen/gen_api_dispatch.lua create mode 100644 src/gen/gen_api_ui_events.lua create mode 100644 src/gen/gen_char_blob.lua create mode 100644 src/gen/gen_declarations.lua create mode 100644 src/gen/gen_eval.lua create mode 100755 src/gen/gen_eval_files.lua create mode 100644 src/gen/gen_events.lua create mode 100644 src/gen/gen_ex_cmds.lua create mode 100644 src/gen/gen_filetype.lua create mode 100644 src/gen/gen_help_html.lua create mode 100644 src/gen/gen_lsp.lua create mode 100644 src/gen/gen_options.lua create mode 100755 src/gen/gen_vimdoc.lua create mode 100644 src/gen/gen_vimvim.lua create mode 100644 src/gen/hashy.lua create mode 100644 src/gen/luacats_grammar.lua create mode 100644 src/gen/luacats_parser.lua create mode 100644 src/gen/nvim_version.lua.in create mode 100644 src/gen/preload.lua create mode 100644 src/gen/preload_nlua.lua create mode 100644 src/gen/util.lua delete mode 100644 src/nvim/generators/c_grammar.lua delete mode 100644 src/nvim/generators/dump_bin_array.lua delete mode 100644 src/nvim/generators/gen_api_dispatch.lua delete mode 100644 src/nvim/generators/gen_api_ui_events.lua delete mode 100644 src/nvim/generators/gen_char_blob.lua delete mode 100644 src/nvim/generators/gen_declarations.lua delete mode 100644 src/nvim/generators/gen_eval.lua delete mode 100644 src/nvim/generators/gen_events.lua delete mode 100644 src/nvim/generators/gen_ex_cmds.lua delete mode 100644 src/nvim/generators/gen_options.lua delete mode 100644 src/nvim/generators/gen_vimvim.lua delete mode 100644 src/nvim/generators/hashy.lua delete mode 100644 src/nvim/generators/nvim_version.lua.in delete mode 100644 src/nvim/generators/preload.lua (limited to 'src') diff --git a/src/.luarc.json b/src/.luarc.json new file mode 100644 index 0000000000..06f49f65d0 --- /dev/null +++ b/src/.luarc.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://raw.githubusercontent.com/LuaLS/vscode-lua/master/setting/schema.json", + "runtime": { + "version": "LuaJIT" + }, + "workspace": { + "library": [ + "../runtime/lua", + "${3rd}/luv/library" + ], + "checkThirdParty": "Disable" + }, + "diagnostics": { + "groupFileStatus": { + "strict": "Opened", + "strong": "Opened" + }, + "groupSeverity": { + "strong": "Warning", + "strict": "Warning" + }, + "unusedLocalExclude": [ "_*" ] + } +} diff --git a/src/gen/c_grammar.lua b/src/gen/c_grammar.lua new file mode 100644 index 0000000000..890c260843 --- /dev/null +++ b/src/gen/c_grammar.lua @@ -0,0 +1,300 @@ +-- lpeg grammar for building api metadata from a set of header files. It +-- ignores comments and preprocessor commands and parses a very small subset +-- of C prototypes with a limited set of types + +--- @class nvim.c_grammar.Proto +--- @field [1] 'proto' +--- @field pos integer +--- @field endpos integer +--- @field fast boolean +--- @field name string +--- @field return_type string +--- @field parameters [string, string][] +--- @field static true? +--- @field inline true? + +--- @class nvim.c_grammar.Preproc +--- @field [1] 'preproc' +--- @field content string + +--- @class nvim.c_grammar.Empty +--- @field [1] 'empty' + +--- @alias nvim.c_grammar.result +--- | nvim.c_grammar.Proto +--- | nvim.c_grammar.Preproc +--- | nvim.c_grammar.Empty + +--- @class nvim.c_grammar +--- @field match fun(self, input: string): nvim.c_grammar.result[] + +local lpeg = vim.lpeg + +local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V +local C, Ct, Cc, Cg, Cp = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cp + +--- @param pat vim.lpeg.Pattern +local function rep(pat) + return pat ^ 0 +end + +--- @param pat vim.lpeg.Pattern +local function rep1(pat) + return pat ^ 1 +end + +--- @param pat vim.lpeg.Pattern +local function opt(pat) + return pat ^ -1 +end + +local any = P(1) +local letter = R('az', 'AZ') + S('_$') +local num = R('09') +local alpha = letter + num +local nl = P('\r\n') + P('\n') +local space = S(' \t') +local str = P('"') * rep((P('\\') * any) + (1 - P('"'))) * P('"') +local char = P("'") * (any - P("'")) * P("'") +local ws = space + nl +local wb = #-alpha -- word boundary +local id = letter * rep(alpha) + +local comment_inline = P('/*') * rep(1 - P('*/')) * P('*/') +local comment = P('//') * rep(1 - nl) * nl +local preproc = Ct(Cc('preproc') * P('#') * Cg(rep(1 - nl) * nl, 'content')) + +local fill = rep(ws + comment_inline + comment + preproc) + +--- @param s string +--- @return vim.lpeg.Pattern +local function word(s) + return fill * P(s) * wb * fill +end + +--- @param x vim.lpeg.Pattern +local function comma1(x) + return x * rep(fill * P(',') * fill * x) +end + +--- @param v string +local function Pf(v) + return fill * P(v) * fill +end + +--- @param x vim.lpeg.Pattern +local function paren(x) + return P('(') * fill * x * fill * P(')') +end + +local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(1 - nl), 'comment'))) + +local braces = P({ + 'S', + A = comment_inline + comment + preproc + str + char + (any - S('{}')), + S = P('{') * rep(V('A')) * rep(V('S') + V('A')) * P('}'), +}) + +-- stylua: ignore start +local typed_container = P({ + 'S', + S = ( + (P('Union') * paren(comma1(V('ID')))) + + (P('ArrayOf') * paren(id * opt(P(',') * fill * rep1(num)))) + + (P('DictOf') * paren(id)) + + (P('LuaRefOf') * paren( + paren(comma1((V('ID') + str) * rep1(ws) * opt(P('*')) * id)) + * opt(P(',') * fill * opt(P('*')) * V('ID')) + )) + + (P('Dict') * paren(id))), + ID = V('S') + id, +}) +-- stylua: ignore end + +local ptr_mod = word('restrict') + word('__restrict') + word('const') +local opt_ptr = rep(Pf('*') * opt(ptr_mod)) + +--- @param name string +--- @param var string +--- @return vim.lpeg.Pattern +local function attr(name, var) + return Cg((P(name) * Cc(true)), var) +end + +--- @param name string +--- @param var string +--- @return vim.lpeg.Pattern +local function attr_num(name, var) + return Cg((P(name) * paren(C(rep1(num)))), var) +end + +local fattr = ( + attr_num('FUNC_API_SINCE', 'since') + + attr_num('FUNC_API_DEPRECATED_SINCE', 'deprecated_since') + + attr('FUNC_API_FAST', 'fast') + + attr('FUNC_API_RET_ALLOC', 'ret_alloc') + + attr('FUNC_API_NOEXPORT', 'noexport') + + attr('FUNC_API_REMOTE_ONLY', 'remote_only') + + attr('FUNC_API_LUA_ONLY', 'lua_only') + + attr('FUNC_API_TEXTLOCK_ALLOW_CMDWIN', 'textlock_allow_cmdwin') + + attr('FUNC_API_TEXTLOCK', 'textlock') + + attr('FUNC_API_REMOTE_IMPL', 'remote_impl') + + attr('FUNC_API_COMPOSITOR_IMPL', 'compositor_impl') + + attr('FUNC_API_CLIENT_IMPL', 'client_impl') + + attr('FUNC_API_CLIENT_IGNORE', 'client_ignore') + + (P('FUNC_') * rep(alpha) * opt(fill * paren(rep(1 - P(')') * any)))) +) + +local void = P('void') * wb + +local api_param_type = ( + (word('Error') * opt_ptr * Cc('error')) + + (word('Arena') * opt_ptr * Cc('arena')) + + (word('lua_State') * opt_ptr * Cc('lstate')) +) + +local ctype = C( + opt(word('const')) + * ( + typed_container + -- 'unsigned' is a type modifier, and a type itself + + (word('unsigned char') + word('unsigned')) + + (word('struct') * fill * id) + + id + ) + * opt(word('const')) + * opt_ptr +) + +local return_type = (C(void) * fill) + ctype + +-- stylua: ignore start +local params = Ct( + (void * #P(')')) + + comma1(Ct( + (api_param_type + ctype) + * fill + * C(id) + * rep(Pf('[') * rep(alpha) * Pf(']')) + * rep(fill * fattr) + )) + * opt(Pf(',') * P('...')) +) +-- stylua: ignore end + +local ignore_line = rep1(1 - nl) * nl +local empty_line = Ct(Cc('empty') * nl * nl) + +local proto_name = opt_ptr * fill * id + +-- __inline is used in MSVC +local decl_mod = ( + Cg(word('static') * Cc(true), 'static') + + Cg((word('inline') + word('__inline')) * Cc(true), 'inline') +) + +local proto = Ct( + Cg(Cp(), 'pos') + * Cc('proto') + * -#P('typedef') + * #alpha + * opt(P('DLLEXPORT') * rep1(ws)) + * rep(decl_mod) + * Cg(return_type, 'return_type') + * fill + * Cg(proto_name, 'name') + * fill + * paren(Cg(params, 'parameters')) + * Cg(Cc(false), 'fast') + * rep(fill * fattr) + * Cg(Cp(), 'endpos') + * (fill * (S(';') + braces)) +) + +local keyset_field = Ct( + Cg(ctype, 'type') + * fill + * Cg(id, 'name') + * fill + * opt(P('DictKey') * paren(Cg(rep1(1 - P(')')), 'dict_key'))) + * Pf(';') +) + +local keyset = Ct( + P('typedef') + * word('struct') + * Pf('{') + * Cg(Ct(rep1(keyset_field)), 'fields') + * Pf('}') + * P('Dict') + * paren(Cg(id, 'keyset_name')) + * Pf(';') +) + +local grammar = + Ct(rep1(empty_line + proto + cdoc_comment + comment + preproc + ws + keyset + ignore_line)) + +if arg[1] == '--test' then + for i, t in ipairs({ + 'void multiqueue_put_event(MultiQueue *self, Event event) {} ', + 'void *xmalloc(size_t size) {} ', + { + 'struct tm *os_localtime_r(const time_t *restrict clock,', + ' struct tm *restrict result) FUNC_ATTR_NONNULL_ALL {}', + }, + { + '_Bool', + '# 163 "src/nvim/event/multiqueue.c"', + ' multiqueue_empty(MultiQueue *self)', + '{}', + }, + 'const char *find_option_end(const char *arg, OptIndex *opt_idxp) {}', + 'bool semsg(const char *const fmt, ...) {}', + 'int32_t utf_ptr2CharInfo_impl(uint8_t const *p, uintptr_t const len) {}', + 'void ex_argdedupe(exarg_T *eap FUNC_ATTR_UNUSED) {}', + 'static TermKeySym register_c0(TermKey *tk, TermKeySym sym, unsigned char ctrl, const char *name) {}', + 'unsigned get_bkc_flags(buf_T *buf) {}', + 'char *xstpcpy(char *restrict dst, const char *restrict src) {}', + 'bool try_leave(const TryState *const tstate, Error *const err) {}', + 'void api_set_error(ErrorType errType) {}', + { + 'void nvim_subscribe(uint64_t channel_id, String event)', + 'FUNC_API_SINCE(1) FUNC_API_DEPRECATED_SINCE(13) FUNC_API_REMOTE_ONLY', + '{}', + }, + + -- Do not consume leading preproc statements + { + '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"', + 'static __inline int mark_global_index(const char name)', + ' FUNC_ATTR_CONST', + '{}', + }, + { + '', + '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"', + 'static __inline int mark_global_index(const char name)', + '{}', + }, + { + 'size_t xstrlcpy(char *__restrict dst, const char *__restrict src, size_t dsize)', + ' FUNC_ATTR_NONNULL_ALL', + ' {}', + }, + }) do + if type(t) == 'table' then + t = table.concat(t, '\n') .. '\n' + end + t = t:gsub(' +', ' ') + local r = grammar:match(t) + if not r then + print('Test ' .. i .. ' failed') + print(' |' .. table.concat(vim.split(t, '\n'), '\n |')) + end + end +end + +return { + grammar = grammar --[[@as nvim.c_grammar]], + typed_container = typed_container, +} diff --git a/src/gen/cdoc_grammar.lua b/src/gen/cdoc_grammar.lua new file mode 100644 index 0000000000..6a7610883b --- /dev/null +++ b/src/gen/cdoc_grammar.lua @@ -0,0 +1,87 @@ +--[[! +LPEG grammar for C doc comments +]] + +--- @class nvim.cdoc.Param +--- @field kind 'param' +--- @field name string +--- @field desc? string + +--- @class nvim.cdoc.Return +--- @field kind 'return' +--- @field desc string + +--- @class nvim.cdoc.Note +--- @field desc? string + +--- @alias nvim.cdoc.grammar.result +--- | nvim.cdoc.Param +--- | nvim.cdoc.Return +--- | nvim.cdoc.Note + +--- @class nvim.cdoc.grammar +--- @field match fun(self, input: string): nvim.cdoc.grammar.result? + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local Ct, Cg = lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local nl = P('\r\n') + P('\n') +local ws = rep1(S(' \t') + nl) + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local ident = letter * rep(letter + R('09')) + +local io = P('[') * (P('in') + P('out') + P('inout')) * P(']') + +--- @param x string +local function Pf(x) + return opt(ws) * P(x) * opt(ws) +end + +--- @type table +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +local grammar = P { + rep1(P('@') * v.ats), + + ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc, + + at_param = Ct( + Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc')) + ), + + at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))), + + at_deprecated = Ct(Cg(P('deprecated'), 'kind')), + + at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + + at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_nodoc = Ct(Cg(P('nodoc'), 'kind')), +} + +return grammar --[[@as nvim.cdoc.grammar]] diff --git a/src/gen/cdoc_parser.lua b/src/gen/cdoc_parser.lua new file mode 100644 index 0000000000..38314c0efd --- /dev/null +++ b/src/gen/cdoc_parser.lua @@ -0,0 +1,223 @@ +local cdoc_grammar = require('gen.cdoc_grammar') +local c_grammar = require('gen.c_grammar') + +--- @class nvim.cdoc.parser.param +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.note +--- @field desc string + +--- @class nvim.cdoc.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.cdoc.parser.fun +--- @field name string +--- @field params nvim.cdoc.parser.param[] +--- @field returns nvim.cdoc.parser.return[] +--- @field desc string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field notes? nvim.cdoc.parser.note[] +--- @field see? nvim.cdoc.parser.note[] + +--- @class nvim.cdoc.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.cdoc.parser.obj +--- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.cdoc.parser.obj +--- | nvim.cdoc.parser.fun +--- | nvim.cdoc.parser.brief + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.cdoc.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.cdoc.parser.State +local function process_doc_line(line, state) + line = line:gsub('^%s+@', '@') + + local parsed = cdoc_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + + local kind = parsed.kind + + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + state.last_doc_item = { + name = parsed.name, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = { { + desc = parsed.desc, + } } + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[1] + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param item table +--- @param state nvim.cdoc.parser.State +local function process_proto(item, state) + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + cur_obj.name = item.name + cur_obj.params = cur_obj.params or {} + + for _, p in ipairs(item.parameters) do + local param = { name = p[2], type = p[1] } + local added = false + for _, cp in ipairs(cur_obj.params) do + if cp.name == param.name then + cp.type = param.type + added = true + break + end + end + + if not added then + table.insert(cur_obj.params, param) + end + end + + cur_obj.returns = cur_obj.returns or { {} } + cur_obj.returns[1].type = item.return_type + + for _, a in ipairs({ + 'fast', + 'remote_only', + 'lua_only', + 'textlock', + 'textlock_allow_cmdwin', + }) do + if item[a] then + cur_obj.attrs = cur_obj.attrs or {} + table.insert(cur_obj.attrs, a) + end + end + + cur_obj.deprecated_since = item.deprecated_since + + -- Remove some arguments + for i = #cur_obj.params, 1, -1 do + local p = cur_obj.params[i] + if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then + table.remove(cur_obj.params, i) + end + end +end + +local M = {} + +--- @param filename string +--- @return {} classes +--- @return nvim.cdoc.parser.fun[] funs +--- @return string[] briefs +function M.parse(filename) + local funs = {} --- @type nvim.cdoc.parser.fun[] + local briefs = {} --- @type string[] + local state = {} --- @type nvim.cdoc.parser.State + + local txt = assert(io.open(filename, 'r')):read('*all') + + local parsed = c_grammar.grammar:match(txt) + for _, item in ipairs(parsed) do + if item.comment then + process_doc_line(item.comment, state) + else + add_doc_lines_to_obj(state) + if item[1] == 'proto' then + process_proto(item, state) + table.insert(funs, state.cur_obj) + end + local cur_obj = state.cur_obj + if cur_obj and not item.static then + if cur_obj.kind == 'brief' then + table.insert(briefs, cur_obj.desc) + end + end + state = {} + end + end + + return {}, funs, briefs +end + +-- M.parse('src/nvim/api/vim.c') + +return M diff --git a/src/gen/dump_bin_array.lua b/src/gen/dump_bin_array.lua new file mode 100644 index 0000000000..c6cda25e73 --- /dev/null +++ b/src/gen/dump_bin_array.lua @@ -0,0 +1,17 @@ +local function dump_bin_array(output, name, data) + output:write([[ + static const uint8_t ]] .. name .. [[[] = { +]]) + + for i = 1, #data do + output:write(string.byte(data, i) .. ', ') + if i % 10 == 0 then + output:write('\n ') + end + end + output:write([[ +}; +]]) +end + +return dump_bin_array diff --git a/src/gen/gen_api_dispatch.lua b/src/gen/gen_api_dispatch.lua new file mode 100644 index 0000000000..a5d0890c2f --- /dev/null +++ b/src/gen/gen_api_dispatch.lua @@ -0,0 +1,990 @@ +-- Example (manual) invocation: +-- +-- make +-- cp build/nvim_version.lua src/nvim/ +-- cd src/nvim +-- nvim -l generators/gen_api_dispatch.lua "../../build/src/nvim/auto/api/private/dispatch_wrappers.generated.h" "../../build/src/nvim/auto/api/private/api_metadata.generated.h" "../../build/funcs_metadata.mpack" "../../build/src/nvim/auto/lua_api_c_bindings.generated.h" "../../build/src/nvim/auto/keysets_defs.generated.h" "../../build/ui_metadata.mpack" "../../build/cmake.config/auto/versiondef_git.h" "./api/autocmd.h" "./api/buffer.h" "./api/command.h" "./api/deprecated.h" "./api/extmark.h" "./api/keysets_defs.h" "./api/options.h" "./api/tabpage.h" "./api/ui.h" "./api/vim.h" "./api/vimscript.h" "./api/win_config.h" "./api/window.h" "../../build/include/api/autocmd.h.generated.h" "../../build/include/api/buffer.h.generated.h" "../../build/include/api/command.h.generated.h" "../../build/include/api/deprecated.h.generated.h" "../../build/include/api/extmark.h.generated.h" "../../build/include/api/options.h.generated.h" "../../build/include/api/tabpage.h.generated.h" "../../build/include/api/ui.h.generated.h" "../../build/include/api/vim.h.generated.h" "../../build/include/api/vimscript.h.generated.h" "../../build/include/api/win_config.h.generated.h" "../../build/include/api/window.h.generated.h" + +local mpack = vim.mpack + +local hashy = require 'gen.hashy' + +local pre_args = 7 +assert(#arg >= pre_args) +-- output h file with generated dispatch functions (dispatch_wrappers.generated.h) +local dispatch_outputf = arg[1] +-- output h file with packed metadata (api_metadata.generated.h) +local api_metadata_outputf = arg[2] +-- output metadata mpack file, for use by other build scripts (funcs_metadata.mpack) +local mpack_outputf = arg[3] +local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c +local keysets_outputf = arg[5] -- keysets_defs.generated.h +local ui_metadata_inputf = arg[6] -- ui events metadata +local git_version_inputf = arg[7] -- git version header + +local functions = {} + +-- names of all headers relative to the source root (for inclusion in the +-- generated file) +local headers = {} + +-- set of function names, used to detect duplicates +local function_names = {} + +local c_grammar = require('gen.c_grammar') + +local startswith = vim.startswith + +local function add_function(fn) + local public = startswith(fn.name, 'nvim_') or fn.deprecated_since + if public and not fn.noexport then + functions[#functions + 1] = fn + function_names[fn.name] = true + if + #fn.parameters >= 2 + and fn.parameters[2][1] == 'Array' + and fn.parameters[2][2] == 'uidata' + then + -- function receives the "args" as a parameter + fn.receives_array_args = true + -- remove the args parameter + table.remove(fn.parameters, 2) + end + if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then + -- this function should receive the channel id + fn.receives_channel_id = true + -- remove the parameter since it won't be passed by the api client + table.remove(fn.parameters, 1) + end + if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then + -- function can fail if the last parameter type is 'Error' + fn.can_fail = true + -- remove the error parameter, msgpack has it's own special field + -- for specifying errors + fn.parameters[#fn.parameters] = nil + end + if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'lstate' then + fn.has_lua_imp = true + fn.parameters[#fn.parameters] = nil + end + if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'arena' then + fn.receives_arena = true + fn.parameters[#fn.parameters] = nil + end + end +end + +local keysets = {} + +local function add_keyset(val) + local keys = {} + local types = {} + local c_names = {} + local is_set_name = 'is_set__' .. val.keyset_name .. '_' + local has_optional = false + for i, field in ipairs(val.fields) do + local dict_key = field.dict_key or field.name + if field.type ~= 'Object' then + types[dict_key] = field.type + end + if field.name ~= is_set_name and field.type ~= 'OptionalKeys' then + table.insert(keys, dict_key) + if dict_key ~= field.name then + c_names[dict_key] = field.name + end + else + if i > 1 then + error("'is_set__{type}_' must be first if present") + elseif field.name ~= is_set_name then + error(val.keyset_name .. ': name of first key should be ' .. is_set_name) + elseif field.type ~= 'OptionalKeys' then + error("'" .. is_set_name .. "' must have type 'OptionalKeys'") + end + has_optional = true + end + end + table.insert(keysets, { + name = val.keyset_name, + keys = keys, + c_names = c_names, + types = types, + has_optional = has_optional, + }) +end + +local ui_options_text = nil + +-- read each input file, parse and append to the api metadata +for i = pre_args + 1, #arg do + local full_path = arg[i] + local parts = {} + for part in string.gmatch(full_path, '[^/]+') do + parts[#parts + 1] = part + end + headers[#headers + 1] = parts[#parts - 1] .. '/' .. parts[#parts] + + local input = assert(io.open(full_path, 'rb')) + + local text = input:read('*all') + local tmp = c_grammar.grammar:match(text) + for j = 1, #tmp do + local val = tmp[j] + if val.keyset_name then + add_keyset(val) + elseif val.name then + add_function(val) + end + end + + ui_options_text = ui_options_text or string.match(text, 'ui_ext_names%[][^{]+{([^}]+)}') + input:close() +end + +local function shallowcopy(orig) + local copy = {} + for orig_key, orig_value in pairs(orig) do + copy[orig_key] = orig_value + end + return copy +end + +-- Export functions under older deprecated names. +-- These will be removed eventually. +local deprecated_aliases = require('nvim.api.dispatch_deprecated') +for _, f in ipairs(shallowcopy(functions)) do + local ismethod = false + if startswith(f.name, 'nvim_') then + if startswith(f.name, 'nvim__') or f.name == 'nvim_error_event' then + f.since = -1 + elseif f.since == nil then + print('Function ' .. f.name .. ' lacks since field.\n') + os.exit(1) + end + f.since = tonumber(f.since) + if f.deprecated_since ~= nil then + f.deprecated_since = tonumber(f.deprecated_since) + end + + if startswith(f.name, 'nvim_buf_') then + ismethod = true + elseif startswith(f.name, 'nvim_win_') then + ismethod = true + elseif startswith(f.name, 'nvim_tabpage_') then + ismethod = true + end + f.remote = f.remote_only or not f.lua_only + f.lua = f.lua_only or not f.remote_only + f.eval = (not f.lua_only) and not f.remote_only + else + f.deprecated_since = tonumber(f.deprecated_since) + assert(f.deprecated_since == 1) + f.remote = true + f.since = 0 + end + f.method = ismethod + local newname = deprecated_aliases[f.name] + if newname ~= nil then + if function_names[newname] then + -- duplicate + print( + 'Function ' + .. f.name + .. ' has deprecated alias\n' + .. newname + .. ' which has a separate implementation.\n' + .. 'Please remove it from src/nvim/api/dispatch_deprecated.lua' + ) + os.exit(1) + end + local newf = shallowcopy(f) + newf.name = newname + if newname == 'ui_try_resize' then + -- The return type was incorrectly set to Object in 0.1.5. + -- Keep it that way for clients that rely on this. + newf.return_type = 'Object' + end + newf.impl_name = f.name + newf.lua = false + newf.eval = false + newf.since = 0 + newf.deprecated_since = 1 + functions[#functions + 1] = newf + end +end + +-- don't expose internal attributes like "impl_name" in public metadata +local exported_attributes = { 'name', 'return_type', 'method', 'since', 'deprecated_since' } +local exported_functions = {} +for _, f in ipairs(functions) do + if not (startswith(f.name, 'nvim__') or f.name == 'nvim_error_event' or f.name == 'redraw') then + local f_exported = {} + for _, attr in ipairs(exported_attributes) do + f_exported[attr] = f[attr] + end + f_exported.parameters = {} + for i, param in ipairs(f.parameters) do + if param[1] == 'DictOf(LuaRef)' then + param = { 'Dict', param[2] } + elseif startswith(param[1], 'Dict(') then + param = { 'Dict', param[2] } + end + f_exported.parameters[i] = param + end + if startswith(f.return_type, 'Dict(') then + f_exported.return_type = 'Dict' + end + exported_functions[#exported_functions + 1] = f_exported + end +end + +local ui_options = { 'rgb' } +for x in string.gmatch(ui_options_text, '"([a-z][a-z_]+)"') do + table.insert(ui_options, x) +end + +local version = require 'nvim_version' -- `build/nvim_version.lua` file. +local git_version = io.open(git_version_inputf):read '*a' +local version_build = string.match(git_version, '#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL + +-- serialize the API metadata using msgpack and embed into the resulting +-- binary for easy querying by clients +local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb')) +local pieces = {} + +-- Naively using mpack.encode({foo=x, bar=y}) will make the build +-- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead +local function fixdict(num) + if num > 15 then + error 'implement more dict codes' + end + table.insert(pieces, string.char(128 + num)) +end +local function put(item, item2) + table.insert(pieces, mpack.encode(item)) + if item2 ~= nil then + table.insert(pieces, mpack.encode(item2)) + end +end + +fixdict(6) + +put('version') +fixdict(1 + #version) +for _, item in ipairs(version) do + -- NB: all items are mandatory. But any error will be less confusing + -- with placeholder vim.NIL (than invalid mpack data) + local val = item[2] == nil and vim.NIL or item[2] + put(item[1], val) +end +put('build', version_build) + +put('functions', exported_functions) +put('ui_events') +table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all')) +put('ui_options', ui_options) + +put('error_types') +fixdict(2) +put('Exception', { id = 0 }) +put('Validation', { id = 1 }) + +put('types') +local types = + { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } } +fixdict(#types) +for i, item in ipairs(types) do + put(item[1]) + fixdict(2) + put('id', i - 1) + put('prefix', item[2]) +end + +local packed = table.concat(pieces) +local dump_bin_array = require('gen.dump_bin_array') +dump_bin_array(api_metadata_output, 'packed_api_metadata', packed) +api_metadata_output:close() + +-- start building the dispatch wrapper output +local output = assert(io.open(dispatch_outputf, 'wb')) + +local keysets_defs = assert(io.open(keysets_outputf, 'wb')) + +-- =========================================================================== +-- NEW API FILES MUST GO HERE. +-- +-- When creating a new API file, you must include it here, +-- so that the dispatcher can find the C functions that you are creating! +-- =========================================================================== +output:write([[ +#include "nvim/errors.h" +#include "nvim/ex_docmd.h" +#include "nvim/ex_getln.h" +#include "nvim/globals.h" +#include "nvim/log.h" +#include "nvim/map_defs.h" + +#include "nvim/api/autocmd.h" +#include "nvim/api/buffer.h" +#include "nvim/api/command.h" +#include "nvim/api/deprecated.h" +#include "nvim/api/extmark.h" +#include "nvim/api/options.h" +#include "nvim/api/tabpage.h" +#include "nvim/api/ui.h" +#include "nvim/api/vim.h" +#include "nvim/api/vimscript.h" +#include "nvim/api/win_config.h" +#include "nvim/api/window.h" +#include "nvim/ui_client.h" + +]]) + +keysets_defs:write('// IWYU pragma: private, include "nvim/api/private/dispatch.h"\n\n') + +for _, k in ipairs(keysets) do + local neworder, hashfun = hashy.hashy_hash(k.name, k.keys, function(idx) + return k.name .. '_table[' .. idx .. '].str' + end) + + keysets_defs:write('extern KeySetLink ' .. k.name .. '_table[' .. (1 + #neworder) .. '];\n') + + local function typename(type) + if type == 'HLGroupID' then + return 'kObjectTypeInteger' + elseif not type or vim.startswith(type, 'Union') then + return 'kObjectTypeNil' + elseif vim.startswith(type, 'LuaRefOf') then + return 'kObjectTypeLuaRef' + elseif type == 'StringArray' then + return 'kUnpackTypeStringArray' + elseif vim.startswith(type, 'ArrayOf') then + return 'kObjectTypeArray' + else + return 'kObjectType' .. type + end + end + + output:write('KeySetLink ' .. k.name .. '_table[] = {\n') + for i, key in ipairs(neworder) do + local ind = -1 + if k.has_optional then + ind = i + keysets_defs:write('#define KEYSET_OPTIDX_' .. k.name .. '__' .. key .. ' ' .. ind .. '\n') + end + output:write( + ' {"' + .. key + .. '", offsetof(KeyDict_' + .. k.name + .. ', ' + .. (k.c_names[key] or key) + .. '), ' + .. typename(k.types[key]) + .. ', ' + .. ind + .. ', ' + .. (k.types[key] == 'HLGroupID' and 'true' or 'false') + .. '},\n' + ) + end + output:write(' {NULL, 0, kObjectTypeNil, -1, false},\n') + output:write('};\n\n') + + output:write(hashfun) + + output:write([[ +KeySetLink *KeyDict_]] .. k.name .. [[_get_field(const char *str, size_t len) +{ + int hash = ]] .. k.name .. [[_hash(str, len); + if (hash == -1) { + return NULL; + } + return &]] .. k.name .. [[_table[hash]; +} + +]]) +end + +local function real_type(type) + local rv = type + local rmatch = string.match(type, 'Dict%(([_%w]+)%)') + if rmatch then + return 'KeyDict_' .. rmatch + elseif c_grammar.typed_container:match(rv) then + if rv:match('Array') then + rv = 'Array' + else + rv = 'Dict' + end + end + return rv +end + +local function attr_name(rt) + if rt == 'Float' then + return 'floating' + else + return rt:lower() + end +end + +-- start the handler functions. Visit each function metadata to build the +-- handler function with code generated for validating arguments and calling to +-- the real API. +for i = 1, #functions do + local fn = functions[i] + if fn.impl_name == nil and fn.remote then + local args = {} + + output:write( + 'Object handle_' .. fn.name .. '(uint64_t channel_id, Array args, Arena* arena, Error *error)' + ) + output:write('\n{') + output:write('\n#ifdef NVIM_LOG_DEBUG') + output:write('\n DLOG("RPC: ch %" PRIu64 ": invoke ' .. fn.name .. '", channel_id);') + output:write('\n#endif') + output:write('\n Object ret = NIL;') + -- Declare/initialize variables that will hold converted arguments + for j = 1, #fn.parameters do + local param = fn.parameters[j] + local rt = real_type(param[1]) + local converted = 'arg_' .. j + output:write('\n ' .. rt .. ' ' .. converted .. ';') + end + output:write('\n') + if not fn.receives_array_args then + output:write('\n if (args.size != ' .. #fn.parameters .. ') {') + output:write( + '\n api_set_error(error, kErrorTypeException, \ + "Wrong number of arguments: expecting ' + .. #fn.parameters + .. ' but got %zu", args.size);' + ) + output:write('\n goto cleanup;') + output:write('\n }\n') + end + + -- Validation/conversion for each argument + for j = 1, #fn.parameters do + local converted, param + param = fn.parameters[j] + converted = 'arg_' .. j + local rt = real_type(param[1]) + if rt == 'Object' then + output:write('\n ' .. converted .. ' = args.items[' .. (j - 1) .. '];\n') + elseif rt:match('^KeyDict_') then + converted = '&' .. converted + output:write('\n if (args.items[' .. (j - 1) .. '].type == kObjectTypeDict) {') --luacheck: ignore 631 + output:write('\n memset(' .. converted .. ', 0, sizeof(*' .. converted .. '));') -- TODO: neeeee + output:write( + '\n if (!api_dict_to_keydict(' + .. converted + .. ', ' + .. rt + .. '_get_field, args.items[' + .. (j - 1) + .. '].data.dict, error)) {' + ) + output:write('\n goto cleanup;') + output:write('\n }') + output:write( + '\n } else if (args.items[' + .. (j - 1) + .. '].type == kObjectTypeArray && args.items[' + .. (j - 1) + .. '].data.array.size == 0) {' + ) --luacheck: ignore 631 + output:write('\n memset(' .. converted .. ', 0, sizeof(*' .. converted .. '));') + + output:write('\n } else {') + output:write( + '\n api_set_error(error, kErrorTypeException, \ + "Wrong type for argument ' + .. j + .. ' when calling ' + .. fn.name + .. ', expecting ' + .. param[1] + .. '");' + ) + output:write('\n goto cleanup;') + output:write('\n }\n') + else + if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then + -- Buffer, Window, and Tabpage have a specific type, but are stored in integer + output:write( + '\n if (args.items[' + .. (j - 1) + .. '].type == kObjectType' + .. rt + .. ' && args.items[' + .. (j - 1) + .. '].data.integer >= 0) {' + ) + output:write( + '\n ' .. converted .. ' = (handle_T)args.items[' .. (j - 1) .. '].data.integer;' + ) + else + output:write('\n if (args.items[' .. (j - 1) .. '].type == kObjectType' .. rt .. ') {') + output:write( + '\n ' + .. converted + .. ' = args.items[' + .. (j - 1) + .. '].data.' + .. attr_name(rt) + .. ';' + ) + end + if + rt:match('^Buffer$') + or rt:match('^Window$') + or rt:match('^Tabpage$') + or rt:match('^Boolean$') + then + -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages + output:write( + '\n } else if (args.items[' + .. (j - 1) + .. '].type == kObjectTypeInteger && args.items[' + .. (j - 1) + .. '].data.integer >= 0) {' + ) + output:write( + '\n ' .. converted .. ' = (handle_T)args.items[' .. (j - 1) .. '].data.integer;' + ) + end + if rt:match('^Float$') then + -- accept integers for Floats + output:write('\n } else if (args.items[' .. (j - 1) .. '].type == kObjectTypeInteger) {') + output:write( + '\n ' .. converted .. ' = (Float)args.items[' .. (j - 1) .. '].data.integer;' + ) + end + -- accept empty lua tables as empty dictionaries + if rt:match('^Dict') then + output:write( + '\n } else if (args.items[' + .. (j - 1) + .. '].type == kObjectTypeArray && args.items[' + .. (j - 1) + .. '].data.array.size == 0) {' + ) --luacheck: ignore 631 + output:write('\n ' .. converted .. ' = (Dict)ARRAY_DICT_INIT;') + end + output:write('\n } else {') + output:write( + '\n api_set_error(error, kErrorTypeException, \ + "Wrong type for argument ' + .. j + .. ' when calling ' + .. fn.name + .. ', expecting ' + .. param[1] + .. '");' + ) + output:write('\n goto cleanup;') + output:write('\n }\n') + end + args[#args + 1] = converted + end + + if fn.textlock then + output:write('\n if (text_locked()) {') + output:write('\n api_set_error(error, kErrorTypeException, "%s", get_text_locked_msg());') + output:write('\n goto cleanup;') + output:write('\n }\n') + elseif fn.textlock_allow_cmdwin then + output:write('\n if (textlock != 0 || expr_map_locked()) {') + output:write('\n api_set_error(error, kErrorTypeException, "%s", e_textlock);') + output:write('\n goto cleanup;') + output:write('\n }\n') + end + + -- function call + output:write('\n ') + if fn.return_type ~= 'void' then + -- has a return value, prefix the call with a declaration + output:write(fn.return_type .. ' rv = ') + end + + -- write the function name and the opening parenthesis + output:write(fn.name .. '(') + + local call_args = {} + if fn.receives_channel_id then + table.insert(call_args, 'channel_id') + end + + if fn.receives_array_args then + table.insert(call_args, 'args') + end + + for _, a in ipairs(args) do + table.insert(call_args, a) + end + + if fn.receives_arena then + table.insert(call_args, 'arena') + end + + if fn.has_lua_imp then + table.insert(call_args, 'NULL') + end + + if fn.can_fail then + table.insert(call_args, 'error') + end + + output:write(table.concat(call_args, ', ')) + output:write(');\n') + + if fn.can_fail then + -- if the function can fail, also pass a pointer to the local error object + -- and check for the error + output:write('\n if (ERROR_SET(error)) {') + output:write('\n goto cleanup;') + output:write('\n }\n') + end + + local ret_type = real_type(fn.return_type) + if string.match(ret_type, '^KeyDict_') then + local table = string.sub(ret_type, 9) .. '_table' + output:write( + '\n ret = DICT_OBJ(api_keydict_to_dict(&rv, ' + .. table + .. ', ARRAY_SIZE(' + .. table + .. '), arena));' + ) + elseif ret_type ~= 'void' then + output:write('\n ret = ' .. string.upper(real_type(fn.return_type)) .. '_OBJ(rv);') + end + output:write('\n\ncleanup:') + + output:write('\n return ret;\n}\n\n') + end +end + +local remote_fns = {} +for _, fn in ipairs(functions) do + if fn.remote then + remote_fns[fn.name] = fn + end +end +remote_fns.redraw = { impl_name = 'ui_client_redraw', fast = true } + +local names = vim.tbl_keys(remote_fns) +table.sort(names) +local hashorder, hashfun = hashy.hashy_hash('msgpack_rpc_get_handler_for', names, function(idx) + return 'method_handlers[' .. idx .. '].name' +end) + +output:write('const MsgpackRpcRequestHandler method_handlers[] = {\n') +for n, name in ipairs(hashorder) do + local fn = remote_fns[name] + fn.handler_id = n - 1 + output:write( + ' { .name = "' + .. name + .. '", .fn = handle_' + .. (fn.impl_name or fn.name) + .. ', .fast = ' + .. tostring(fn.fast) + .. ', .ret_alloc = ' + .. tostring(not not fn.ret_alloc) + .. '},\n' + ) +end +output:write('};\n\n') +output:write(hashfun) + +output:close() + +functions.keysets = keysets +local mpack_output = assert(io.open(mpack_outputf, 'wb')) +mpack_output:write(mpack.encode(functions)) +mpack_output:close() + +local function include_headers(output_handle, headers_to_include) + for i = 1, #headers_to_include do + if headers_to_include[i]:sub(-12) ~= '.generated.h' then + output_handle:write('\n#include "nvim/' .. headers_to_include[i] .. '"') + end + end +end + +local function write_shifted_output(str, ...) + str = str:gsub('\n ', '\n') + str = str:gsub('^ ', '') + str = str:gsub(' +$', '') + output:write(string.format(str, ...)) +end + +-- start building lua output +output = assert(io.open(lua_c_bindings_outputf, 'wb')) + +include_headers(output, headers) +output:write('\n') + +local lua_c_functions = {} + +local function process_function(fn) + local lua_c_function_name = ('nlua_api_%s'):format(fn.name) + write_shifted_output( + [[ + + static int %s(lua_State *lstate) + { + Error err = ERROR_INIT; + Arena arena = ARENA_EMPTY; + char *err_param = 0; + if (lua_gettop(lstate) != %i) { + api_set_error(&err, kErrorTypeValidation, "Expected %i argument%s"); + goto exit_0; + } + ]], + lua_c_function_name, + #fn.parameters, + #fn.parameters, + (#fn.parameters == 1) and '' or 's' + ) + lua_c_functions[#lua_c_functions + 1] = { + binding = lua_c_function_name, + api = fn.name, + } + + if not fn.fast then + write_shifted_output( + [[ + if (!nlua_is_deferred_safe()) { + return luaL_error(lstate, e_fast_api_disabled, "%s"); + } + ]], + fn.name + ) + end + + if fn.textlock then + write_shifted_output([[ + if (text_locked()) { + api_set_error(&err, kErrorTypeException, "%%s", get_text_locked_msg()); + goto exit_0; + } + ]]) + elseif fn.textlock_allow_cmdwin then + write_shifted_output([[ + if (textlock != 0 || expr_map_locked()) { + api_set_error(&err, kErrorTypeException, "%%s", e_textlock); + goto exit_0; + } + ]]) + end + + local cparams = '' + local free_code = {} + for j = #fn.parameters, 1, -1 do + local param = fn.parameters[j] + local cparam = string.format('arg%u', j) + local param_type = real_type(param[1]) + local extra = param_type == 'Dict' and 'false, ' or '' + local arg_free_code = '' + if param[1] == 'Object' then + extra = 'true, ' + arg_free_code = ' api_luarefs_free_object(' .. cparam .. ');' + elseif param[1] == 'DictOf(LuaRef)' then + extra = 'true, ' + arg_free_code = ' api_luarefs_free_dict(' .. cparam .. ');' + elseif param[1] == 'LuaRef' then + arg_free_code = ' api_free_luaref(' .. cparam .. ');' + end + local errshift = 0 + local seterr = '' + if string.match(param_type, '^KeyDict_') then + write_shifted_output( + [[ + %s %s = KEYDICT_INIT; + nlua_pop_keydict(lstate, &%s, %s_get_field, &err_param, &arena, &err); + ]], + param_type, + cparam, + cparam, + param_type + ) + cparam = '&' .. cparam + errshift = 1 -- free incomplete dict on error + arg_free_code = ' api_luarefs_free_keydict(' + .. cparam + .. ', ' + .. string.sub(param_type, 9) + .. '_table);' + else + write_shifted_output( + [[ + const %s %s = nlua_pop_%s(lstate, %s&arena, &err);]], + param[1], + cparam, + param_type, + extra + ) + seterr = '\n err_param = "' .. param[2] .. '";' + end + + write_shifted_output([[ + + if (ERROR_SET(&err)) {]] .. seterr .. [[ + + goto exit_%u; + } + + ]], #fn.parameters - j + errshift) + free_code[#free_code + 1] = arg_free_code + cparams = cparam .. ', ' .. cparams + end + if fn.receives_channel_id then + cparams = 'LUA_INTERNAL_CALL, ' .. cparams + end + if fn.receives_arena then + cparams = cparams .. '&arena, ' + end + + if fn.has_lua_imp then + cparams = cparams .. 'lstate, ' + end + + if fn.can_fail then + cparams = cparams .. '&err' + else + cparams = cparams:gsub(', $', '') + end + local free_at_exit_code = '' + for i = 1, #free_code do + local rev_i = #free_code - i + 1 + local code = free_code[rev_i] + if i == 1 and not string.match(real_type(fn.parameters[1][1]), '^KeyDict_') then + free_at_exit_code = free_at_exit_code .. ('\n%s'):format(code) + else + free_at_exit_code = free_at_exit_code .. ('\nexit_%u:\n%s'):format(rev_i, code) + end + end + local err_throw_code = [[ + +exit_0: + arena_mem_free(arena_finish(&arena)); + if (ERROR_SET(&err)) { + luaL_where(lstate, 1); + if (err_param) { + lua_pushstring(lstate, "Invalid '"); + lua_pushstring(lstate, err_param); + lua_pushstring(lstate, "': "); + } + lua_pushstring(lstate, err.msg); + api_clear_error(&err); + lua_concat(lstate, err_param ? 5 : 2); + return lua_error(lstate); + } +]] + local return_type + if fn.return_type ~= 'void' then + if fn.return_type:match('^ArrayOf') then + return_type = 'Array' + else + return_type = fn.return_type + end + local free_retval = '' + if fn.ret_alloc then + free_retval = ' api_free_' .. return_type:lower() .. '(ret);' + end + write_shifted_output(' %s ret = %s(%s);\n', fn.return_type, fn.name, cparams) + + local ret_type = real_type(fn.return_type) + local ret_mode = (ret_type == 'Object') and '&' or '' + if fn.has_lua_imp then + -- only push onto the Lua stack if we haven't already + write_shifted_output( + [[ + if (lua_gettop(lstate) == 0) { + nlua_push_%s(lstate, %sret, kNluaPushSpecial | kNluaPushFreeRefs); + } + ]], + return_type, + ret_mode + ) + elseif string.match(ret_type, '^KeyDict_') then + write_shifted_output( + ' nlua_push_keydict(lstate, &ret, %s_table);\n', + string.sub(ret_type, 9) + ) + else + local special = (fn.since ~= nil and fn.since < 11) + write_shifted_output( + ' nlua_push_%s(lstate, %sret, %s | kNluaPushFreeRefs);\n', + return_type, + ret_mode, + special and 'kNluaPushSpecial' or '0' + ) + end + + -- NOTE: we currently assume err_throw needs nothing from arena + write_shifted_output( + [[ + %s + %s + %s + return 1; + ]], + free_retval, + free_at_exit_code, + err_throw_code + ) + else + write_shifted_output( + [[ + %s(%s); + %s + %s + return 0; + ]], + fn.name, + cparams, + free_at_exit_code, + err_throw_code + ) + end + write_shifted_output([[ + } + ]]) +end + +for _, fn in ipairs(functions) do + if fn.lua or fn.name:sub(1, 4) == '_vim' then + process_function(fn) + end +end + +output:write(string.format( + [[ +void nlua_add_api_functions(lua_State *lstate) +{ + lua_createtable(lstate, 0, %u); +]], + #lua_c_functions +)) +for _, func in ipairs(lua_c_functions) do + output:write(string.format( + [[ + + lua_pushcfunction(lstate, &%s); + lua_setfield(lstate, -2, "%s");]], + func.binding, + func.api + )) +end +output:write([[ + + lua_setfield(lstate, -2, "api"); +} +]]) + +output:close() +keysets_defs:close() diff --git a/src/gen/gen_api_ui_events.lua b/src/gen/gen_api_ui_events.lua new file mode 100644 index 0000000000..8ba67dafff --- /dev/null +++ b/src/gen/gen_api_ui_events.lua @@ -0,0 +1,219 @@ +local mpack = vim.mpack + +assert(#arg == 5) +local input = io.open(arg[1], 'rb') +local call_output = io.open(arg[2], 'wb') +local remote_output = io.open(arg[3], 'wb') +local metadata_output = io.open(arg[4], 'wb') +local client_output = io.open(arg[5], 'wb') + +local c_grammar = require('gen.c_grammar') +local events = c_grammar.grammar:match(input:read('*all')) + +local hashy = require 'gen.hashy' + +local function write_signature(output, ev, prefix, notype) + output:write('(' .. prefix) + if prefix == '' and #ev.parameters == 0 then + output:write('void') + end + for j = 1, #ev.parameters do + if j > 1 or prefix ~= '' then + output:write(', ') + end + local param = ev.parameters[j] + if not notype then + output:write(param[1] .. ' ') + end + output:write(param[2]) + end + output:write(')') +end + +local function write_arglist(output, ev) + if #ev.parameters == 0 then + return + end + output:write(' MAXSIZE_TEMP_ARRAY(args, ' .. #ev.parameters .. ');\n') + for j = 1, #ev.parameters do + local param = ev.parameters[j] + local kind = string.upper(param[1]) + output:write(' ADD_C(args, ') + output:write(kind .. '_OBJ(' .. param[2] .. ')') + output:write(');\n') + end +end + +local function call_ui_event_method(output, ev) + output:write('void ui_client_event_' .. ev.name .. '(Array args)\n{\n') + + local hlattrs_args_count = 0 + if #ev.parameters > 0 then + output:write(' if (args.size < ' .. #ev.parameters) + for j = 1, #ev.parameters do + local kind = ev.parameters[j][1] + if kind ~= 'Object' then + if kind == 'HlAttrs' then + kind = 'Dict' + end + output:write('\n || args.items[' .. (j - 1) .. '].type != kObjectType' .. kind .. '') + end + end + output:write(') {\n') + output:write(' ELOG("Error handling ui event \'' .. ev.name .. '\'");\n') + output:write(' return;\n') + output:write(' }\n') + end + + for j = 1, #ev.parameters do + local param = ev.parameters[j] + local kind = param[1] + output:write(' ' .. kind .. ' arg_' .. j .. ' = ') + if kind == 'HlAttrs' then + -- The first HlAttrs argument is rgb_attrs and second is cterm_attrs + output:write( + 'ui_client_dict2hlattrs(args.items[' + .. (j - 1) + .. '].data.dict, ' + .. (hlattrs_args_count == 0 and 'true' or 'false') + .. ');\n' + ) + hlattrs_args_count = hlattrs_args_count + 1 + elseif kind == 'Object' then + output:write('args.items[' .. (j - 1) .. '];\n') + elseif kind == 'Window' then + output:write('(Window)args.items[' .. (j - 1) .. '].data.integer;\n') + else + output:write('args.items[' .. (j - 1) .. '].data.' .. string.lower(kind) .. ';\n') + end + end + + output:write(' tui_' .. ev.name .. '(tui') + for j = 1, #ev.parameters do + output:write(', arg_' .. j) + end + output:write(');\n') + + output:write('}\n\n') +end + +events = vim.tbl_filter(function(ev) + return ev[1] ~= 'empty' and ev[1] ~= 'preproc' +end, events) + +for i = 1, #events do + local ev = events[i] + assert(ev.return_type == 'void') + + if ev.since == nil and not ev.noexport then + print('Ui event ' .. ev.name .. ' lacks since field.\n') + os.exit(1) + end + ev.since = tonumber(ev.since) + + local args = #ev.parameters > 0 and 'args' or 'noargs' + if not ev.remote_only then + if not ev.remote_impl and not ev.noexport then + remote_output:write('void remote_ui_' .. ev.name) + write_signature(remote_output, ev, 'RemoteUI *ui') + remote_output:write('\n{\n') + write_arglist(remote_output, ev) + remote_output:write(' push_call(ui, "' .. ev.name .. '", ' .. args .. ');\n') + remote_output:write('}\n\n') + end + end + + if not (ev.remote_only and ev.remote_impl) then + call_output:write('void ui_call_' .. ev.name) + write_signature(call_output, ev, '') + call_output:write('\n{\n') + if ev.remote_only then + -- Lua callbacks may emit other events or the same event again. Avoid the latter + -- by adding a recursion guard to each generated function that may call a Lua callback. + call_output:write(' static bool entered = false;\n') + call_output:write(' if (entered) {\n') + call_output:write(' return;\n') + call_output:write(' }\n') + call_output:write(' entered = true;\n') + write_arglist(call_output, ev) + call_output:write((' ui_call_event("%s", %s, %s)'):format(ev.name, tostring(ev.fast), args)) + call_output:write(';\n entered = false;\n') + elseif ev.compositor_impl then + call_output:write(' ui_comp_' .. ev.name) + write_signature(call_output, ev, '', true) + call_output:write(';\n') + call_output:write(' UI_CALL') + write_signature(call_output, ev, '!ui->composed, ' .. ev.name .. ', ui', true) + call_output:write(';\n') + else + call_output:write(' UI_CALL') + write_signature(call_output, ev, 'true, ' .. ev.name .. ', ui', true) + call_output:write(';\n') + end + call_output:write('}\n\n') + end + + if ev.compositor_impl then + call_output:write('void ui_composed_call_' .. ev.name) + write_signature(call_output, ev, '') + call_output:write('\n{\n') + call_output:write(' UI_CALL') + write_signature(call_output, ev, 'ui->composed, ' .. ev.name .. ', ui', true) + call_output:write(';\n') + call_output:write('}\n\n') + end + + if (not ev.remote_only) and not ev.noexport and not ev.client_impl and not ev.client_ignore then + call_ui_event_method(client_output, ev) + end +end + +local client_events = {} +for _, ev in ipairs(events) do + if (not ev.noexport) and ((not ev.remote_only) or ev.client_impl) and not ev.client_ignore then + client_events[ev.name] = ev + end +end + +local hashorder, hashfun = hashy.hashy_hash( + 'ui_client_handler', + vim.tbl_keys(client_events), + function(idx) + return 'event_handlers[' .. idx .. '].name' + end +) + +client_output:write('static const UIClientHandler event_handlers[] = {\n') + +for _, name in ipairs(hashorder) do + client_output:write(' { .name = "' .. name .. '", .fn = ui_client_event_' .. name .. '},\n') +end + +client_output:write('\n};\n\n') +client_output:write(hashfun) + +call_output:close() +remote_output:close() +client_output:close() + +-- don't expose internal attributes like "impl_name" in public metadata +local exported_attributes = { 'name', 'parameters', 'since', 'deprecated_since' } +local exported_events = {} +for _, ev in ipairs(events) do + local ev_exported = {} + for _, attr in ipairs(exported_attributes) do + ev_exported[attr] = ev[attr] + end + for _, p in ipairs(ev_exported.parameters) do + if p[1] == 'HlAttrs' or p[1] == 'Dict' then + -- TODO(justinmk): for back-compat, but do clients actually look at this? + p[1] = 'Dictionary' + end + end + if not ev.noexport then + exported_events[#exported_events + 1] = ev_exported + end +end + +metadata_output:write(mpack.encode(exported_events)) +metadata_output:close() diff --git a/src/gen/gen_char_blob.lua b/src/gen/gen_char_blob.lua new file mode 100644 index 0000000000..c40e0d6e82 --- /dev/null +++ b/src/gen/gen_char_blob.lua @@ -0,0 +1,96 @@ +if arg[1] == '--help' then + print('Usage:') + print(' ' .. arg[0] .. ' [-c] target source varname [source varname]...') + print('') + print('Generates C file with big uint8_t blob.') + print('Blob will be stored in a static const array named varname.') + os.exit() +end + +-- Recognized options: +-- -c compile Lua bytecode +local options = {} + +while true do + local opt = string.match(arg[1], '^-(%w)') + if not opt then + break + end + + options[opt] = true + table.remove(arg, 1) +end + +assert(#arg >= 3 and (#arg - 1) % 2 == 0) + +local target_file = arg[1] or error('Need a target file') +local target = io.open(target_file, 'w') + +target:write('#include \n\n') + +local index_items = {} + +local warn_on_missing_compiler = true +local modnames = {} +for argi = 2, #arg, 2 do + local source_file = arg[argi] + local modname = arg[argi + 1] + if modnames[modname] then + error(string.format('modname %q is already specified for file %q', modname, modnames[modname])) + end + modnames[modname] = source_file + + local varname = string.gsub(modname, '%.', '_dot_') .. '_module' + target:write(('static const uint8_t %s[] = {\n'):format(varname)) + + local output + if options.c then + local luac = os.getenv('LUAC_PRG') + if luac and luac ~= '' then + output = io.popen(luac:format(source_file), 'r'):read('*a') + elseif warn_on_missing_compiler then + print('LUAC_PRG is missing, embedding raw source') + warn_on_missing_compiler = false + end + end + + if not output then + local source = io.open(source_file, 'r') + or error(string.format("source_file %q doesn't exist", source_file)) + output = source:read('*a') + source:close() + end + + local num_bytes = 0 + local MAX_NUM_BYTES = 15 -- 78 / 5: maximum number of bytes on one line + target:write(' ') + + local increase_num_bytes + increase_num_bytes = function() + num_bytes = num_bytes + 1 + if num_bytes == MAX_NUM_BYTES then + num_bytes = 0 + target:write('\n ') + end + end + + for i = 1, string.len(output) do + local byte = output:byte(i) + target:write(string.format(' %3u,', byte)) + increase_num_bytes() + end + + target:write(' 0};\n') + if modname ~= '_' then + table.insert( + index_items, + ' { "' .. modname .. '", ' .. varname .. ', sizeof ' .. varname .. ' },\n\n' + ) + end +end + +target:write('static ModuleDef builtin_modules[] = {\n') +target:write(table.concat(index_items)) +target:write('};\n') + +target:close() diff --git a/src/gen/gen_declarations.lua b/src/gen/gen_declarations.lua new file mode 100644 index 0000000000..582ac756b4 --- /dev/null +++ b/src/gen/gen_declarations.lua @@ -0,0 +1,186 @@ +local grammar = require('gen.c_grammar').grammar + +--- @param fname string +--- @return string? +local function read_file(fname) + local f = io.open(fname, 'r') + if not f then + return + end + local contents = f:read('*a') + f:close() + return contents +end + +--- @param fname string +--- @param contents string[] +local function write_file(fname, contents) + local contents_s = table.concat(contents, '\n') .. '\n' + local fcontents = read_file(fname) + if fcontents == contents_s then + return + end + local f = assert(io.open(fname, 'w')) + f:write(contents_s) + f:close() +end + +--- @param fname string +--- @param non_static_fname string +--- @return string? non_static +local function add_iwyu_non_static(fname, non_static_fname) + if fname:find('.*/src/nvim/.*%.c$') then + -- Add an IWYU pragma comment if the corresponding .h file exists. + local header_fname = fname:sub(1, -3) .. '.h' + local header_f = io.open(header_fname, 'r') + if header_f then + header_f:close() + return (header_fname:gsub('.*/src/nvim/', 'nvim/')) + end + elseif non_static_fname:find('/include/api/private/dispatch_wrappers%.h%.generated%.h$') then + return 'nvim/api/private/dispatch.h' + elseif non_static_fname:find('/include/ui_events_call%.h%.generated%.h$') then + return 'nvim/ui.h' + elseif non_static_fname:find('/include/ui_events_client%.h%.generated%.h$') then + return 'nvim/ui_client.h' + elseif non_static_fname:find('/include/ui_events_remote%.h%.generated%.h$') then + return 'nvim/api/ui.h' + end +end + +--- @param d string +local function process_decl(d) + -- Comments are really handled by preprocessor, so the following is not + -- needed + d = d:gsub('/%*.-%*/', '') + d = d:gsub('//.-\n', '\n') + d = d:gsub('# .-\n', '') + d = d:gsub('\n', ' ') + d = d:gsub('%s+', ' ') + d = d:gsub(' ?%( ?', '(') + d = d:gsub(' ?, ?', ', ') + d = d:gsub(' ?(%*+) ?', ' %1') + d = d:gsub(' ?(FUNC_ATTR_)', ' %1') + d = d:gsub(' $', '') + d = d:gsub('^ ', '') + return d .. ';' +end + +--- @param fname string +--- @param text string +--- @return string[] static +--- @return string[] non_static +--- @return boolean any_static +local function gen_declarations(fname, text) + local non_static = {} --- @type string[] + local static = {} --- @type string[] + + local neededfile = fname:match('[^/]+$') + local curfile = nil + local any_static = false + for _, node in ipairs(grammar:match(text)) do + if node[1] == 'preproc' then + curfile = node.content:match('^%a* %d+ "[^"]-/?([^"/]+)"') or curfile + elseif node[1] == 'proto' and curfile == neededfile then + local node_text = text:sub(node.pos, node.endpos - 1) + local declaration = process_decl(node_text) + + if node.static then + if not any_static and declaration:find('FUNC_ATTR_') then + any_static = true + end + static[#static + 1] = declaration + else + non_static[#non_static + 1] = 'DLLEXPORT ' .. declaration + end + end + end + + return static, non_static, any_static +end + +local usage = [[ +Usage: + + gen_declarations.lua definitions.c static.h non-static.h definitions.i + +Generates declarations for a C file definitions.c, putting declarations for +static functions into static.h and declarations for non-static functions into +non-static.h. File `definitions.i' should contain an already preprocessed +version of definitions.c and it is the only one which is actually parsed, +definitions.c is needed only to determine functions from which file out of all +functions found in definitions.i are needed and to generate an IWYU comment. +]] + +local function main() + local fname = arg[1] + local static_fname = arg[2] + local non_static_fname = arg[3] + local preproc_fname = arg[4] + local static_basename = arg[5] + + if fname == '--help' or #arg < 5 then + print(usage) + os.exit() + end + + local text = assert(read_file(preproc_fname)) + + local static_decls, non_static_decls, any_static = gen_declarations(fname, text) + + local static = {} --- @type string[] + if fname:find('.*/src/nvim/.*%.h$') then + static[#static + 1] = ('// IWYU pragma: private, include "%s"'):format( + fname:gsub('.*/src/nvim/', 'nvim/') + ) + end + vim.list_extend(static, { + '#define DEFINE_FUNC_ATTRIBUTES', + '#include "nvim/func_attr.h"', + '#undef DEFINE_FUNC_ATTRIBUTES', + }) + vim.list_extend(static, static_decls) + vim.list_extend(static, { + '#define DEFINE_EMPTY_ATTRIBUTES', + '#include "nvim/func_attr.h" // IWYU pragma: export', + '', + }) + + write_file(static_fname, static) + + if any_static then + local orig_text = assert(read_file(fname)) + local pat = '\n#%s?include%s+"' .. static_basename .. '"\n' + local pat_comment = '\n#%s?include%s+"' .. static_basename .. '"%s*//' + if not orig_text:find(pat) and not orig_text:find(pat_comment) then + error(('fail: missing include for %s in %s'):format(static_basename, fname)) + end + end + + if non_static_fname ~= 'SKIP' then + local non_static = {} --- @type string[] + local iwyu_non_static = add_iwyu_non_static(fname, non_static_fname) + if iwyu_non_static then + non_static[#non_static + 1] = ('// IWYU pragma: private, include "%s"'):format( + iwyu_non_static + ) + end + vim.list_extend(non_static, { + '#define DEFINE_FUNC_ATTRIBUTES', + '#include "nvim/func_attr.h"', + '#undef DEFINE_FUNC_ATTRIBUTES', + '#ifndef DLLEXPORT', + '# ifdef MSWIN', + '# define DLLEXPORT __declspec(dllexport)', + '# else', + '# define DLLEXPORT', + '# endif', + '#endif', + }) + vim.list_extend(non_static, non_static_decls) + non_static[#non_static + 1] = '#include "nvim/func_attr.h"' + write_file(non_static_fname, non_static) + end +end + +return main() diff --git a/src/gen/gen_eval.lua b/src/gen/gen_eval.lua new file mode 100644 index 0000000000..9d2f2f7523 --- /dev/null +++ b/src/gen/gen_eval.lua @@ -0,0 +1,112 @@ +local mpack = vim.mpack + +local autodir = arg[1] +local metadata_file = arg[2] +local funcs_file = arg[3] + +local funcsfname = autodir .. '/funcs.generated.h' + +--Will generate funcs.generated.h with definition of functions static const array. + +local hashy = require 'gen.hashy' + +local hashpipe = assert(io.open(funcsfname, 'wb')) + +hashpipe:write([[ +#include "nvim/arglist.h" +#include "nvim/cmdexpand.h" +#include "nvim/cmdhist.h" +#include "nvim/digraph.h" +#include "nvim/eval.h" +#include "nvim/eval/buffer.h" +#include "nvim/eval/deprecated.h" +#include "nvim/eval/fs.h" +#include "nvim/eval/funcs.h" +#include "nvim/eval/typval.h" +#include "nvim/eval/vars.h" +#include "nvim/eval/window.h" +#include "nvim/ex_docmd.h" +#include "nvim/ex_getln.h" +#include "nvim/fold.h" +#include "nvim/getchar.h" +#include "nvim/insexpand.h" +#include "nvim/mapping.h" +#include "nvim/match.h" +#include "nvim/mbyte.h" +#include "nvim/menu.h" +#include "nvim/mouse.h" +#include "nvim/move.h" +#include "nvim/quickfix.h" +#include "nvim/runtime.h" +#include "nvim/search.h" +#include "nvim/state.h" +#include "nvim/strings.h" +#include "nvim/sign.h" +#include "nvim/testing.h" +#include "nvim/undo.h" + +]]) + +local funcs = require('nvim.eval').funcs +for _, func in pairs(funcs) do + if func.float_func then + func.func = 'float_op_wrapper' + func.data = '{ .float_func = &' .. func.float_func .. ' }' + end +end + +local metadata = mpack.decode(io.open(metadata_file, 'rb'):read('*all')) +for _, fun in ipairs(metadata) do + if fun.eval then + funcs[fun.name] = { + args = #fun.parameters, + func = 'api_wrapper', + data = '{ .api_handler = &method_handlers[' .. fun.handler_id .. '] }', + } + end +end + +local func_names = vim.tbl_filter(function(name) + return name:match('__%d*$') == nil +end, vim.tbl_keys(funcs)) + +table.sort(func_names) + +local funcsdata = assert(io.open(funcs_file, 'w')) +funcsdata:write(mpack.encode(func_names)) +funcsdata:close() + +local neworder, hashfun = hashy.hashy_hash('find_internal_func', func_names, function(idx) + return 'functions[' .. idx .. '].name' +end) + +hashpipe:write('static const EvalFuncDef functions[] = {\n') + +for _, name in ipairs(neworder) do + local def = funcs[name] + local args = def.args or 0 + if type(args) == 'number' then + args = { args, args } + elseif #args == 1 then + args[2] = 'MAX_FUNC_ARGS' + end + local base = def.base or 'BASE_NONE' + local func = def.func or ('f_' .. name) + local data = def.data or '{ .null = NULL }' + local fast = def.fast and 'true' or 'false' + hashpipe:write( + (' { "%s", %s, %s, %s, %s, &%s, %s },\n'):format( + name, + args[1], + args[2], + base, + fast, + func, + data + ) + ) +end +hashpipe:write(' { NULL, 0, 0, BASE_NONE, false, NULL, { .null = NULL } },\n') +hashpipe:write('};\n\n') +hashpipe:write(hashfun) +hashpipe:close() diff --git a/src/gen/gen_eval_files.lua b/src/gen/gen_eval_files.lua new file mode 100755 index 0000000000..74e45507e5 --- /dev/null +++ b/src/gen/gen_eval_files.lua @@ -0,0 +1,1090 @@ +#!/usr/bin/env -S nvim -l + +-- Generator for various vimdoc and Lua type files + +local util = require('gen.util') +local fmt = string.format + +local DEP_API_METADATA = 'build/funcs_metadata.mpack' +local TEXT_WIDTH = 78 + +--- @class vim.api.metadata +--- @field name string +--- @field parameters [string,string][] +--- @field return_type string +--- @field deprecated_since integer +--- @field eval boolean +--- @field fast boolean +--- @field handler_id integer +--- @field impl_name string +--- @field lua boolean +--- @field method boolean +--- @field remote boolean +--- @field since integer + +local LUA_API_RETURN_OVERRIDES = { + nvim_buf_get_command = 'table', + nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item_by_id', + nvim_buf_get_extmarks = 'vim.api.keyset.get_extmark_item[]', + nvim_buf_get_keymap = 'vim.api.keyset.get_keymap[]', + nvim_get_autocmds = 'vim.api.keyset.get_autocmds.ret[]', + nvim_get_color_map = 'table', + nvim_get_command = 'table', + nvim_get_keymap = 'vim.api.keyset.get_keymap[]', + nvim_get_mark = 'vim.api.keyset.get_mark', + + -- Can also return table, however we need to + -- pick one to get some benefit. + -- REVISIT lewrus01 (26/01/24): we can maybe add + -- @overload fun(ns: integer, {}): table + nvim_get_hl = 'vim.api.keyset.get_hl_info', + + nvim_get_mode = 'vim.api.keyset.get_mode', + nvim_get_namespaces = 'table', + nvim_get_option_info = 'vim.api.keyset.get_option_info', + nvim_get_option_info2 = 'vim.api.keyset.get_option_info', + nvim_parse_cmd = 'vim.api.keyset.parse_cmd', + nvim_win_get_config = 'vim.api.keyset.win_config', +} + +local LUA_API_KEYSET_OVERRIDES = { + create_autocmd = { + callback = 'string|(fun(args: vim.api.keyset.create_autocmd.callback_args): boolean?)', + }, +} + +local LUA_API_PARAM_OVERRIDES = { + nvim_create_user_command = { + command = 'string|fun(args: vim.api.keyset.create_user_command.command_args)', + }, +} + +local LUA_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", +} + +local LUA_API_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", + '', + '--- This file embeds vimdoc as the function descriptions', + '--- so ignore any doc related errors.', + '--- @diagnostic disable: undefined-doc-name,luadoc-miss-symbol', + '', + 'vim.api = {}', +} + +local LUA_OPTION_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", + '', + '---@class vim.bo', + '---@field [integer] vim.bo', + 'vim.bo = vim.bo', + '', + '---@class vim.wo', + '---@field [integer] vim.wo', + 'vim.wo = vim.wo', +} + +local LUA_VVAR_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", + '', + '--- @class vim.v', + 'vim.v = ...', +} + +local LUA_KEYWORDS = { + ['and'] = true, + ['end'] = true, + ['function'] = true, + ['or'] = true, + ['if'] = true, + ['while'] = true, + ['repeat'] = true, + ['true'] = true, + ['false'] = true, +} + +local OPTION_TYPES = { + boolean = 'boolean', + number = 'integer', + string = 'string', +} + +local API_TYPES = { + Window = 'integer', + Tabpage = 'integer', + Buffer = 'integer', + Boolean = 'boolean', + Object = 'any', + Integer = 'integer', + String = 'string', + Array = 'any[]', + LuaRef = 'function', + Dict = 'table', + Float = 'number', + HLGroupID = 'integer|string', + void = '', +} + +--- @param s string +--- @return string +local function luaescape(s) + if LUA_KEYWORDS[s] then + return s .. '_' + end + return s +end + +--- @param x string +--- @param sep? string +--- @return string[] +local function split(x, sep) + return vim.split(x, sep or '\n', { plain = true }) +end + +--- Convert an API type to Lua +--- @param t string +--- @return string +local function api_type(t) + if vim.startswith(t, '*') then + return api_type(t:sub(2)) .. '?' + end + + local as0 = t:match('^ArrayOf%((.*)%)') + if as0 then + local as = split(as0, ', ') + return api_type(as[1]) .. '[]' + end + + local d = t:match('^Dict%((.*)%)') + if d then + return 'vim.api.keyset.' .. d + end + + local d0 = t:match('^DictOf%((.*)%)') + if d0 then + return 'table' + end + + local u = t:match('^Union%((.*)%)') + if u then + local us = vim.split(u, ',%s*') + return table.concat(vim.tbl_map(api_type, us), '|') + end + + local l = t:match('^LuaRefOf%((.*)%)') + if l then + --- @type string + l = l:gsub('%s+', ' ') + --- @type string?, string? + local as, r = l:match('%((.*)%),%s*(.*)') + if not as then + --- @type string + as = assert(l:match('%((.*)%)')) + end + + local as1 = {} --- @type string[] + for a in vim.gsplit(as, ',%s') do + local a1 = vim.split(a, '%s+', { trimempty = true }) + local nm = a1[2]:gsub('%*(.*)$', '%1?') + as1[#as1 + 1] = nm .. ': ' .. api_type(a1[1]) + end + + return ('fun(%s)%s'):format(table.concat(as1, ', '), r and ': ' .. api_type(r) or '') + end + + return API_TYPES[t] or t +end + +--- @param f string +--- @param params [string,string][]|true +--- @return string +local function render_fun_sig(f, params) + local param_str --- @type string + if params == true then + param_str = '...' + else + param_str = table.concat( + vim.tbl_map( + --- @param v [string,string] + --- @return string + function(v) + return luaescape(v[1]) + end, + params + ), + ', ' + ) + end + + if LUA_KEYWORDS[f] then + return fmt("vim.fn['%s'] = function(%s) end", f, param_str) + else + return fmt('function vim.fn.%s(%s) end', f, param_str) + end +end + +--- Uniquify names +--- @param params [string,string,string][] +--- @return [string,string,string][] +local function process_params(params) + local seen = {} --- @type table + local sfx = 1 + + for _, p in ipairs(params) do + if seen[p[1]] then + p[1] = p[1] .. sfx + sfx = sfx + 1 + else + seen[p[1]] = true + end + end + + return params +end + +--- @return table +local function get_api_meta() + local ret = {} --- @type table + + local cdoc_parser = require('gen.cdoc_parser') + + local f = 'src/nvim/api' + + local function include(fun) + if not vim.startswith(fun.name, 'nvim_') then + return false + end + if vim.tbl_contains(fun.attrs or {}, 'lua_only') then + return true + end + if vim.tbl_contains(fun.attrs or {}, 'remote_only') then + return false + end + return true + end + + --- @type table + local functions = {} + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + local filename = vim.fs.joinpath(f, path) + local _, funs = cdoc_parser.parse(filename) + for _, fn in ipairs(funs) do + if include(fn) then + functions[fn.name] = fn + end + end + end + end + + for _, fun in pairs(functions) do + local deprecated = fun.deprecated_since ~= nil + + local notes = {} --- @type string[] + for _, note in ipairs(fun.notes or {}) do + notes[#notes + 1] = note.desc + end + + local sees = {} --- @type string[] + for _, see in ipairs(fun.see or {}) do + sees[#sees + 1] = see.desc + end + + local pty_overrides = LUA_API_PARAM_OVERRIDES[fun.name] or {} + + local params = {} --- @type [string,string][] + for _, p in ipairs(fun.params) do + params[#params + 1] = { + p.name, + api_type(pty_overrides[p.name] or p.type), + not deprecated and p.desc or nil, + } + end + + local r = { + signature = 'NA', + name = fun.name, + params = params, + notes = notes, + see = sees, + returns = api_type(fun.returns[1].type), + deprecated = deprecated, + } + + if not deprecated then + r.desc = fun.desc + r.returns_desc = fun.returns[1].desc + end + + ret[fun.name] = r + end + return ret +end + +--- Convert vimdoc references to markdown literals +--- Convert vimdoc codeblocks to markdown codeblocks +--- +--- Ensure code blocks have one empty line before the start fence and after the closing fence. +--- +--- @param x string +--- @param special string? +--- | 'see-api-meta' Normalize `@see` for API meta docstrings. +--- @return string +local function norm_text(x, special) + if special == 'see-api-meta' then + -- Try to guess a symbol that actually works in @see. + -- "nvim_xx()" => "vim.api.nvim_xx" + x = x:gsub([=[%|?(nvim_[^.()| ]+)%(?%)?%|?]=], 'vim.api.%1') + -- TODO: Remove backticks when LuaLS resolves: https://github.com/LuaLS/lua-language-server/issues/2889 + -- "|foo|" => "`:help foo`" + x = x:gsub([=[|([^ ]+)|]=], '`:help %1`') + end + + return ( + x:gsub('|([^ ]+)|', '`%1`') + :gsub('\n*>lua', '\n\n```lua') + :gsub('\n*>vim', '\n\n```vim') + :gsub('\n+<$', '\n```') + :gsub('\n+<\n+', '\n```\n\n') + :gsub('%s+>\n+', '\n```\n') + :gsub('\n+<%s+\n?', '\n```\n') + ) +end + +--- Generates LuaLS docstring for an API function. +--- @param _f string +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_api_meta(_f, fun, write) + write('') + + if vim.startswith(fun.name, 'nvim__') then + write('--- @private') + end + + if fun.deprecated then + write('--- @deprecated') + end + + local desc = fun.desc + if desc then + write(util.prefix_lines('--- ', norm_text(desc))) + end + + -- LuaLS doesn't support @note. Render @note items as a markdown list. + if fun.notes and #fun.notes > 0 then + write('--- Note:') + write(util.prefix_lines('--- ', table.concat(fun.notes, '\n'))) + write('---') + end + + for _, see in ipairs(fun.see or {}) do + write(util.prefix_lines('--- @see ', norm_text(see, 'see-api-meta'))) + end + + local param_names = {} --- @type string[] + local params = process_params(fun.params) + for _, p in ipairs(params) do + local pname, ptype, pdesc = luaescape(p[1]), p[2], p[3] + param_names[#param_names + 1] = pname + if pdesc then + local s = '--- @param ' .. pname .. ' ' .. ptype .. ' ' + local pdesc_a = split(vim.trim(norm_text(pdesc))) + write(s .. pdesc_a[1]) + for i = 2, #pdesc_a do + if not pdesc_a[i] then + break + end + write('--- ' .. pdesc_a[i]) + end + else + write('--- @param ' .. pname .. ' ' .. ptype) + end + end + + if fun.returns ~= '' then + local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or '' + local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns + write(util.prefix_lines('--- ', '@return ' .. ret .. ret_desc)) + end + local param_str = table.concat(param_names, ', ') + + write(fmt('function vim.api.%s(%s) end', fun.name, param_str)) +end + +--- @return table +local function get_api_keysets_meta() + local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) + local metadata = assert(vim.mpack.decode(mpack_f:read('*all'))) + + local ret = {} --- @type table + + --- @type {name: string, keys: string[], types: table}[] + local keysets = metadata.keysets + + for _, k in ipairs(keysets) do + local pty_overrides = LUA_API_KEYSET_OVERRIDES[k.name] or {} + local params = {} + for _, key in ipairs(k.keys) do + local pty = pty_overrides[key] or k.types[key] or 'any' + table.insert(params, { key .. '?', api_type(pty) }) + end + ret[k.name] = { + signature = 'NA', + name = k.name, + params = params, + } + end + + return ret +end + +--- Generates LuaLS docstring for an API keyset. +--- @param _f string +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_api_keyset_meta(_f, fun, write) + if string.sub(fun.name, 1, 1) == '_' then + return -- not exported + end + write('') + write('--- @class vim.api.keyset.' .. fun.name) + for _, p in ipairs(fun.params) do + write('--- @field ' .. p[1] .. ' ' .. p[2]) + end +end + +--- @return table +local function get_eval_meta() + return require('nvim.eval').funcs +end + +--- Generates LuaLS docstring for a Vimscript "eval" function. +--- @param f string +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_eval_meta(f, fun, write) + if fun.lua == false then + return + end + + local funname = fun.name or f + local params = process_params(fun.params) + + write('') + if fun.deprecated then + write('--- @deprecated') + end + + local desc = fun.desc + + if desc then + --- @type string + desc = desc:gsub('\n%s*\n%s*$', '\n') + for _, l in ipairs(split(desc)) do + l = l:gsub('^ ', ''):gsub('\t', ' '):gsub('@', '\\@') + write('--- ' .. l) + end + end + + for _, text in ipairs(vim.fn.reverse(fun.generics or {})) do + write(fmt('--- @generic %s', text)) + end + + local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 + + for i, param in ipairs(params) do + local pname, ptype = luaescape(param[1]), param[2] + local optional = (pname ~= '...' and i > req_args) and '?' or '' + write(fmt('--- @param %s%s %s', pname, optional, ptype)) + end + + if fun.returns ~= false then + local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or '' + write('--- @return ' .. (fun.returns or 'any') .. ret_desc) + end + + write(render_fun_sig(funname, params)) +end + +--- Generates vimdoc heading for a Vimscript "eval" function signature. +--- @param name string +--- @param name_tag boolean +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_sig_and_tag(name, name_tag, fun, write) + if not fun.signature then + return + end + + local tags = name_tag and { '*' .. name .. '()*' } or {} + + if fun.tags then + for _, t in ipairs(fun.tags) do + tags[#tags + 1] = '*' .. t .. '*' + end + end + + if #tags == 0 then + write(fun.signature) + return + end + + local tag = table.concat(tags, ' ') + local siglen = #fun.signature + local conceal_offset = 2 * (#tags - 1) + local tag_pad_len = math.max(1, 80 - #tag + conceal_offset) + + if siglen + #tag > 80 then + write(string.rep(' ', tag_pad_len) .. tag) + write(fun.signature) + else + write(fmt('%s%s%s', fun.signature, string.rep(' ', tag_pad_len - siglen), tag)) + end +end + +--- Generates vimdoc for a Vimscript "eval" function. +--- @param f string +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_eval_doc(f, fun, write) + if fun.deprecated or not fun.signature then + return + end + + render_sig_and_tag(fun.name or f, not f:find('__%d+$'), fun, write) + + if not fun.desc then + return + end + + local params = process_params(fun.params) + local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 + + local desc_l = split(vim.trim(fun.desc)) + for _, l in ipairs(desc_l) do + l = l:gsub('^ ', '') + if vim.startswith(l, '<') and not l:match('^<[^ \t]+>') then + write('<\t\t' .. l:sub(2)) + elseif l:match('^>[a-z0-9]*$') then + write(l) + else + write('\t\t' .. l) + end + end + + if #desc_l > 0 and not desc_l[#desc_l]:match('^ 0 then + write(util.md_to_vimdoc('Parameters: ~', 16, 16, TEXT_WIDTH)) + for i, param in ipairs(params) do + local pname, ptype = param[1], param[2] + local optional = (pname ~= '...' and i > req_args) and '?' or '' + local s = fmt('- %-14s (`%s%s`)', fmt('{%s}', pname), ptype, optional) + write(util.md_to_vimdoc(s, 16, 18, TEXT_WIDTH)) + end + write('') + end + + if fun.returns ~= false then + write(util.md_to_vimdoc('Return: ~', 16, 16, TEXT_WIDTH)) + local ret = ('(`%s`)'):format((fun.returns or 'any')) + ret = ret .. (fun.returns_desc and ' ' .. fun.returns_desc or '') + ret = util.md_to_vimdoc(ret, 18, 18, TEXT_WIDTH) + write(ret) + write('') + end +end + +--- @param d vim.option_defaults +--- @param vimdoc? boolean +--- @return string +local function render_option_default(d, vimdoc) + local dt --- @type integer|boolean|string|fun(): string + if d.if_false ~= nil then + dt = d.if_false + else + dt = d.if_true + end + + if vimdoc then + if d.doc then + return d.doc + end + if type(dt) == 'boolean' then + return dt and 'on' or 'off' + end + end + + if dt == '' or dt == nil or type(dt) == 'function' then + dt = d.meta + end + + local v --- @type string + if not vimdoc then + v = vim.inspect(dt) --[[@as string]] + else + v = type(dt) == 'string' and '"' .. dt .. '"' or tostring(dt) + end + + --- @type table + local envvars = { + TMPDIR = false, + VIMRUNTIME = false, + XDG_CONFIG_HOME = vim.env.HOME .. '/.local/config', + XDG_DATA_HOME = vim.env.HOME .. '/.local/share', + XDG_STATE_HOME = vim.env.HOME .. '/.local/state', + } + + for name, default in pairs(envvars) do + local value = vim.env[name] or default + if value then + v = v:gsub(vim.pesc(value), '$' .. name) + end + end + + return v +end + +--- @param _f string +--- @param opt vim.option_meta +--- @param write fun(line: string) +local function render_option_meta(_f, opt, write) + write('') + for _, l in ipairs(split(norm_text(opt.desc))) do + write('--- ' .. l) + end + + if opt.type == 'string' and not opt.list and opt.values then + local values = {} --- @type string[] + for _, e in ipairs(opt.values) do + values[#values + 1] = fmt("'%s'", e) + end + write('--- @type ' .. table.concat(values, '|')) + else + write('--- @type ' .. OPTION_TYPES[opt.type]) + end + + write('vim.o.' .. opt.full_name .. ' = ' .. render_option_default(opt.defaults)) + if opt.abbreviation then + write('vim.o.' .. opt.abbreviation .. ' = vim.o.' .. opt.full_name) + end + + for _, s in pairs { + { 'wo', 'win' }, + { 'bo', 'buf' }, + { 'go', 'global' }, + } do + local id, scope = s[1], s[2] + if vim.list_contains(opt.scope, scope) or (id == 'go' and #opt.scope > 1) then + local pfx = 'vim.' .. id .. '.' + write(pfx .. opt.full_name .. ' = vim.o.' .. opt.full_name) + if opt.abbreviation then + write(pfx .. opt.abbreviation .. ' = ' .. pfx .. opt.full_name) + end + end + end +end + +--- @param _f string +--- @param opt vim.option_meta +--- @param write fun(line: string) +local function render_vvar_meta(_f, opt, write) + write('') + + local desc = split(norm_text(opt.desc)) + while desc[#desc]:match('^%s*$') do + desc[#desc] = nil + end + + for _, l in ipairs(desc) do + write('--- ' .. l) + end + + write('--- @type ' .. (opt.type or 'any')) + + if LUA_KEYWORDS[opt.full_name] then + write("vim.v['" .. opt.full_name .. "'] = ...") + else + write('vim.v.' .. opt.full_name .. ' = ...') + end +end + +--- @param s string[] +--- @return string +local function scope_to_doc(s) + local m = { + global = 'global', + buf = 'local to buffer', + win = 'local to window', + tab = 'local to tab page', + } + + if #s == 1 then + return m[s[1]] + end + assert(s[1] == 'global') + return 'global or ' .. m[s[2]] .. (s[2] ~= 'tab' and ' |global-local|' or '') +end + +-- @param o vim.option_meta +-- @return string +local function scope_more_doc(o) + if + vim.list_contains({ + 'bufhidden', + 'buftype', + 'filetype', + 'modified', + 'previewwindow', + 'readonly', + 'scroll', + 'syntax', + 'winfixheight', + 'winfixwidth', + }, o.full_name) + then + return ' |local-noglobal|' + end + + return '' +end + +--- @param x string +--- @return string +local function dedent(x) + local xs = split(x) + local leading_ws = xs[1]:match('^%s*') --[[@as string]] + local leading_ws_pat = '^' .. leading_ws + + for i in ipairs(xs) do + local strip_pat = xs[i]:match(leading_ws_pat) and leading_ws_pat or '^%s*' + xs[i] = xs[i]:gsub(strip_pat, '') + end + + return table.concat(xs, '\n') +end + +--- @return table +local function get_option_meta() + local opts = require('nvim.options').options + local optinfo = vim.api.nvim_get_all_options_info() + local ret = {} --- @type table + for _, o in ipairs(opts) do + local is_window_option = #o.scope == 1 and o.scope[1] == 'win' + local is_option_hidden = o.immutable and not o.varname and not is_window_option + if not is_option_hidden and o.desc then + if o.full_name == 'cmdheight' then + table.insert(o.scope, 'tab') + end + local r = vim.deepcopy(o) --[[@as vim.option_meta]] + r.desc = o.desc:gsub('^ ', ''):gsub('\n ', '\n') + r.defaults = r.defaults or {} + if r.defaults.meta == nil then + r.defaults.meta = optinfo[o.full_name].default + end + ret[o.full_name] = r + end + end + return ret +end + +--- @return table +local function get_vvar_meta() + local info = require('nvim.vvars').vars + local ret = {} --- @type table + for name, o in pairs(info) do + o.desc = dedent(o.desc) + o.full_name = name + ret[name] = o + end + return ret +end + +--- @param opt vim.option_meta +--- @return string[] +local function build_option_tags(opt) + --- @type string[] + local tags = { opt.full_name } + + tags[#tags + 1] = opt.abbreviation + if opt.type == 'boolean' then + for i = 1, #tags do + tags[#tags + 1] = 'no' .. tags[i] + end + end + + for i, t in ipairs(tags) do + tags[i] = "'" .. t .. "'" + end + + for _, t in ipairs(opt.tags or {}) do + tags[#tags + 1] = t + end + + for i, t in ipairs(tags) do + tags[i] = '*' .. t .. '*' + end + + return tags +end + +--- @param _f string +--- @param opt vim.option_meta +--- @param write fun(line: string) +local function render_option_doc(_f, opt, write) + local tags = build_option_tags(opt) + local tag_str = table.concat(tags, ' ') + local conceal_offset = 2 * (#tags - 1) + local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) + -- local pad = string.rep(' ', 80 - #tag_str + conceal_offset) + write(tag_pad .. tag_str) + + local name_str --- @type string + if opt.abbreviation then + name_str = fmt("'%s' '%s'", opt.full_name, opt.abbreviation) + else + name_str = fmt("'%s'", opt.full_name) + end + + local otype = opt.type == 'boolean' and 'boolean' or opt.type + if opt.defaults.doc or opt.defaults.if_true ~= nil or opt.defaults.meta ~= nil then + local v = render_option_default(opt.defaults, true) + local pad = string.rep('\t', math.max(1, math.ceil((24 - #name_str) / 8))) + if opt.defaults.doc then + local deflen = #fmt('%s%s%s (', name_str, pad, otype) + --- @type string + v = v:gsub('\n', '\n' .. string.rep(' ', deflen - 2)) + end + write(fmt('%s%s%s\t(default %s)', name_str, pad, otype, v)) + else + write(fmt('%s\t%s', name_str, otype)) + end + + write('\t\t\t' .. scope_to_doc(opt.scope) .. scope_more_doc(opt)) + for _, l in ipairs(split(opt.desc)) do + if l == '<' or l:match('^<%s') then + write(l) + else + write('\t' .. l:gsub('\\<', '<')) + end + end +end + +--- @param _f string +--- @param vvar vim.option_meta +--- @param write fun(line: string) +local function render_vvar_doc(_f, vvar, write) + local name = vvar.full_name + + local tags = { 'v:' .. name, name .. '-variable' } + if vvar.tags then + vim.list_extend(tags, vvar.tags) + end + + for i, t in ipairs(tags) do + tags[i] = '*' .. t .. '*' + end + + local tag_str = table.concat(tags, ' ') + local conceal_offset = 2 * (#tags - 1) + + local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) + write(tag_pad .. tag_str) + + local desc = split(vvar.desc) + + if (#desc == 1 or #desc == 2 and desc[2]:match('^%s*$')) and #name < 10 then + -- single line + write('v:' .. name .. '\t' .. desc[1]:gsub('^%s*', '')) + write('') + else + write('v:' .. name) + for _, l in ipairs(desc) do + if l == '<' or l:match('^<%s') then + write(l) + else + write('\t\t' .. l:gsub('\\<', '<')) + end + end + end +end + +--- @class nvim.gen_eval_files.elem +--- @field path string +--- @field from? string Skip lines in path until this pattern is reached. +--- @field funcs fun(): table +--- @field render fun(f:string,obj:table,write:fun(line:string)) +--- @field header? string[] +--- @field footer? string[] + +--- @type nvim.gen_eval_files.elem[] +local CONFIG = { + { + path = 'runtime/lua/vim/_meta/vimfn.lua', + header = LUA_META_HEADER, + funcs = get_eval_meta, + render = render_eval_meta, + }, + { + path = 'runtime/lua/vim/_meta/api.lua', + header = LUA_API_META_HEADER, + funcs = get_api_meta, + render = render_api_meta, + }, + { + path = 'runtime/lua/vim/_meta/api_keysets.lua', + header = LUA_META_HEADER, + funcs = get_api_keysets_meta, + render = render_api_keyset_meta, + }, + { + path = 'runtime/doc/builtin.txt', + funcs = get_eval_meta, + render = render_eval_doc, + header = { + '*builtin.txt* Nvim', + '', + '', + '\t\t NVIM REFERENCE MANUAL', + '', + '', + 'Builtin functions\t\t*vimscript-functions* *builtin-functions*', + '', + 'For functions grouped by what they are used for see |function-list|.', + '', + '\t\t\t\t Type |gO| to see the table of contents.', + '==============================================================================', + '1. Details *builtin-function-details*', + '', + }, + footer = { + '==============================================================================', + '2. Matching a pattern in a String *string-match*', + '', + 'This is common between several functions. A regexp pattern as explained at', + '|pattern| is normally used to find a match in the buffer lines. When a', + 'pattern is used to find a match in a String, almost everything works in the', + 'same way. The difference is that a String is handled like it is one line.', + 'When it contains a "\\n" character, this is not seen as a line break for the', + 'pattern. It can be matched with a "\\n" in the pattern, or with ".". Example:', + '>vim', + '\tlet a = "aaaa\\nxxxx"', + '\techo matchstr(a, "..\\n..")', + '\t" aa', + '\t" xx', + '\techo matchstr(a, "a.x")', + '\t" a', + '\t" x', + '', + 'Don\'t forget that "^" will only match at the first character of the String and', + '"$" at the last character of the string. They don\'t match after or before a', + '"\\n".', + '', + ' vim:tw=78:ts=8:noet:ft=help:norl:', + }, + }, + { + path = 'runtime/lua/vim/_meta/options.lua', + header = LUA_OPTION_META_HEADER, + funcs = get_option_meta, + render = render_option_meta, + }, + { + path = 'runtime/doc/options.txt', + header = { '' }, + from = 'A jump table for the options with a short description can be found at |Q_op|.', + footer = { + ' vim:tw=78:ts=8:noet:ft=help:norl:', + }, + funcs = get_option_meta, + render = render_option_doc, + }, + { + path = 'runtime/lua/vim/_meta/vvars.lua', + header = LUA_VVAR_META_HEADER, + funcs = get_vvar_meta, + render = render_vvar_meta, + }, + { + path = 'runtime/doc/vvars.txt', + header = { '' }, + from = 'Type |gO| to see the table of contents.', + footer = { + ' vim:tw=78:ts=8:noet:ft=help:norl:', + }, + funcs = get_vvar_meta, + render = render_vvar_doc, + }, +} + +--- @param elem nvim.gen_eval_files.elem +local function render(elem) + print('Rendering ' .. elem.path) + local from_lines = {} --- @type string[] + local from = elem.from + if from then + for line in io.lines(elem.path) do + from_lines[#from_lines + 1] = line + if line:match(from) then + break + end + end + end + + local o = assert(io.open(elem.path, 'w')) + + --- @param l string + local function write(l) + local l1 = l:gsub('%s+$', '') + o:write(l1) + o:write('\n') + end + + for _, l in ipairs(from_lines) do + write(l) + end + + for _, l in ipairs(elem.header or {}) do + write(l) + end + + local funcs = elem.funcs() + + --- @type string[] + local fnames = vim.tbl_keys(funcs) + table.sort(fnames) + + for _, f in ipairs(fnames) do + elem.render(f, funcs[f], write) + end + + for _, l in ipairs(elem.footer or {}) do + write(l) + end + + o:close() +end + +local function main() + for _, c in ipairs(CONFIG) do + render(c) + end +end + +main() diff --git a/src/gen/gen_events.lua b/src/gen/gen_events.lua new file mode 100644 index 0000000000..77f766bb28 --- /dev/null +++ b/src/gen/gen_events.lua @@ -0,0 +1,42 @@ +local fileio_enum_file = arg[1] +local names_file = arg[2] + +local auevents = require('nvim.auevents') +local events = auevents.events + +local enum_tgt = io.open(fileio_enum_file, 'w') +local names_tgt = io.open(names_file, 'w') + +enum_tgt:write([[ +// IWYU pragma: private, include "nvim/autocmd_defs.h" + +typedef enum auto_event {]]) +names_tgt:write([[ +static const struct event_name { + size_t len; + char *name; + int event; +} event_names[] = {]]) + +local aliases = 0 +for i, event in ipairs(events) do + enum_tgt:write(('\n EVENT_%s = %u,'):format(event[1]:upper(), i + aliases - 1)) + -- Events with positive keys aren't allowed in 'eventignorewin'. + local event_int = ('%sEVENT_%s'):format(event[3] and '-' or '', event[1]:upper()) + names_tgt:write(('\n {%u, "%s", %s},'):format(#event[1], event[1], event_int)) + for _, alias in ipairs(event[2]) do + aliases = aliases + 1 + names_tgt:write(('\n {%u, "%s", %s},'):format(#alias, alias, event_int)) + enum_tgt:write(('\n EVENT_%s = %u,'):format(alias:upper(), i + aliases - 1)) + end + if i == #events then -- Last item. + enum_tgt:write(('\n NUM_EVENTS = %u,'):format(i + aliases)) + end +end + +names_tgt:write('\n {0, NULL, (event_T)0},\n};\n') +names_tgt:write('\nstatic AutoCmdVec autocmds[NUM_EVENTS] = { 0 };\n') +names_tgt:close() + +enum_tgt:write('\n} event_T;\n') +enum_tgt:close() diff --git a/src/gen/gen_ex_cmds.lua b/src/gen/gen_ex_cmds.lua new file mode 100644 index 0000000000..6c03e8fc4d --- /dev/null +++ b/src/gen/gen_ex_cmds.lua @@ -0,0 +1,194 @@ +local includedir = arg[1] +local autodir = arg[2] + +-- Will generate files ex_cmds_enum.generated.h with cmdidx_T enum +-- and ex_cmds_defs.generated.h with main Ex commands definitions. + +local enumfname = includedir .. '/ex_cmds_enum.generated.h' +local defsfname = autodir .. '/ex_cmds_defs.generated.h' + +local enumfile = io.open(enumfname, 'w') +local defsfile = io.open(defsfname, 'w') + +local bit = require 'bit' +local ex_cmds = require('nvim.ex_cmds') +local defs = ex_cmds.cmds +local flags = ex_cmds.flags + +local byte_a = string.byte('a') +local byte_z = string.byte('z') +local a_to_z = byte_z - byte_a + 1 + +-- Table giving the index of the first command in cmdnames[] to lookup +-- based on the first letter of a command. +local cmdidxs1_out = string.format( + [[ +static const uint16_t cmdidxs1[%u] = { +]], + a_to_z +) +-- Table giving the index of the first command in cmdnames[] to lookup +-- based on the first 2 letters of a command. +-- Values in cmdidxs2[c1][c2] are relative to cmdidxs1[c1] so that they +-- fit in a byte. +local cmdidxs2_out = string.format( + [[ +static const uint8_t cmdidxs2[%u][%u] = { + /* a b c d e f g h i j k l m n o p q r s t u v w x y z */ +]], + a_to_z, + a_to_z +) + +enumfile:write([[ +// IWYU pragma: private, include "nvim/ex_cmds_defs.h" + +typedef enum CMD_index { +]]) +defsfile:write(string.format( + [[ +#include "nvim/arglist.h" +#include "nvim/autocmd.h" +#include "nvim/buffer.h" +#include "nvim/cmdhist.h" +#include "nvim/debugger.h" +#include "nvim/diff.h" +#include "nvim/digraph.h" +#include "nvim/eval.h" +#include "nvim/eval/userfunc.h" +#include "nvim/eval/vars.h" +#include "nvim/ex_cmds.h" +#include "nvim/ex_cmds2.h" +#include "nvim/ex_docmd.h" +#include "nvim/ex_eval.h" +#include "nvim/ex_session.h" +#include "nvim/help.h" +#include "nvim/indent.h" +#include "nvim/lua/executor.h" +#include "nvim/lua/secure.h" +#include "nvim/mapping.h" +#include "nvim/mark.h" +#include "nvim/match.h" +#include "nvim/menu.h" +#include "nvim/message.h" +#include "nvim/ops.h" +#include "nvim/option.h" +#include "nvim/os/lang.h" +#include "nvim/profile.h" +#include "nvim/quickfix.h" +#include "nvim/runtime.h" +#include "nvim/sign.h" +#include "nvim/spell.h" +#include "nvim/spellfile.h" +#include "nvim/syntax.h" +#include "nvim/undo.h" +#include "nvim/usercmd.h" +#include "nvim/version.h" + +static const int command_count = %u; +static CommandDefinition cmdnames[%u] = { +]], + #defs, + #defs +)) +local cmds, cmdidxs1, cmdidxs2 = {}, {}, {} +for _, cmd in ipairs(defs) do + if bit.band(cmd.flags, flags.RANGE) == flags.RANGE then + assert( + cmd.addr_type ~= 'ADDR_NONE', + string.format('ex_cmds.lua:%s: Using RANGE with ADDR_NONE\n', cmd.command) + ) + else + assert( + cmd.addr_type == 'ADDR_NONE', + string.format('ex_cmds.lua:%s: Missing ADDR_NONE\n', cmd.command) + ) + end + if bit.band(cmd.flags, flags.DFLALL) == flags.DFLALL then + assert( + cmd.addr_type ~= 'ADDR_OTHER' and cmd.addr_type ~= 'ADDR_NONE', + string.format('ex_cmds.lua:%s: Missing misplaced DFLALL\n', cmd.command) + ) + end + if bit.band(cmd.flags, flags.PREVIEW) == flags.PREVIEW then + assert( + cmd.preview_func ~= nil, + string.format('ex_cmds.lua:%s: Missing preview_func\n', cmd.command) + ) + end + local enumname = cmd.enum or ('CMD_' .. cmd.command) + local byte_cmd = cmd.command:sub(1, 1):byte() + if byte_a <= byte_cmd and byte_cmd <= byte_z then + table.insert(cmds, cmd.command) + end + local preview_func + if cmd.preview_func then + preview_func = string.format('&%s', cmd.preview_func) + else + preview_func = 'NULL' + end + enumfile:write(' ' .. enumname .. ',\n') + defsfile:write(string.format( + [[ + [%s] = { + .cmd_name = "%s", + .cmd_func = (ex_func_T)&%s, + .cmd_preview_func = %s, + .cmd_argt = %uL, + .cmd_addr_type = %s + }, +]], + enumname, + cmd.command, + cmd.func, + preview_func, + cmd.flags, + cmd.addr_type + )) +end +for i = #cmds, 1, -1 do + local cmd = cmds[i] + -- First and second characters of the command + local c1 = cmd:sub(1, 1) + cmdidxs1[c1] = i - 1 + if cmd:len() >= 2 then + local c2 = cmd:sub(2, 2) + local byte_c2 = string.byte(c2) + if byte_a <= byte_c2 and byte_c2 <= byte_z then + if not cmdidxs2[c1] then + cmdidxs2[c1] = {} + end + cmdidxs2[c1][c2] = i - 1 + end + end +end +for i = byte_a, byte_z do + local c1 = string.char(i) + cmdidxs1_out = cmdidxs1_out .. ' /* ' .. c1 .. ' */ ' .. cmdidxs1[c1] .. ',\n' + cmdidxs2_out = cmdidxs2_out .. ' /* ' .. c1 .. ' */ {' + for j = byte_a, byte_z do + local c2 = string.char(j) + cmdidxs2_out = cmdidxs2_out + .. ((cmdidxs2[c1] and cmdidxs2[c1][c2]) and string.format( + '%3d', + cmdidxs2[c1][c2] - cmdidxs1[c1] + ) or ' 0') + .. ',' + end + cmdidxs2_out = cmdidxs2_out .. ' },\n' +end +enumfile:write([[ + CMD_SIZE, + CMD_USER = -1, + CMD_USER_BUF = -2 +} cmdidx_T; +]]) +defsfile:write(string.format( + [[ +}; +%s}; +%s}; +]], + cmdidxs1_out, + cmdidxs2_out +)) diff --git a/src/gen/gen_filetype.lua b/src/gen/gen_filetype.lua new file mode 100644 index 0000000000..18b53f1ea4 --- /dev/null +++ b/src/gen/gen_filetype.lua @@ -0,0 +1,209 @@ +local do_not_run = true +if do_not_run then + print([[ + This script was used to bootstrap the filetype patterns in runtime/lua/vim/filetype.lua. It + should no longer be used except for testing purposes. New filetypes, or changes to existing + filetypes, should be ported manually as part of the vim-patch process. + ]]) + return +end + +local filetype_vim = 'runtime/filetype.vim' +local filetype_lua = 'runtime/lua/vim/filetype.lua' + +local keywords = { + ['for'] = true, + ['or'] = true, + ['and'] = true, + ['end'] = true, + ['do'] = true, + ['if'] = true, + ['while'] = true, + ['repeat'] = true, +} + +local sections = { + extension = { str = {}, func = {} }, + filename = { str = {}, func = {} }, + pattern = { str = {}, func = {} }, +} + +local specialchars = '%*%?\\%$%[%]%{%}' + +local function add_pattern(pat, ft) + local ok = true + + -- Patterns that start or end with { or } confuse splitting on commas and make parsing harder, so just skip those + if not string.find(pat, '^%{') and not string.find(pat, '%}$') then + for part in string.gmatch(pat, '[^,]+') do + if not string.find(part, '[' .. specialchars .. ']') then + if type(ft) == 'string' then + sections.filename.str[part] = ft + else + sections.filename.func[part] = ft + end + elseif string.match(part, '^%*%.[^%./' .. specialchars .. ']+$') then + if type(ft) == 'string' then + sections.extension.str[part:sub(3)] = ft + else + sections.extension.func[part:sub(3)] = ft + end + else + if string.match(part, '^%*/[^' .. specialchars .. ']+$') then + -- For patterns matching */some/pattern we want to easily match files + -- with path /some/pattern, so include those in filename detection + if type(ft) == 'string' then + sections.filename.str[part:sub(2)] = ft + else + sections.filename.func[part:sub(2)] = ft + end + end + + if string.find(part, '^[%w-_.*?%[%]/]+$') then + local p = part:gsub('%.', '%%.'):gsub('%*', '.*'):gsub('%?', '.') + -- Insert into array to maintain order rather than setting + -- key-value directly + if type(ft) == 'string' then + sections.pattern.str[p] = ft + else + sections.pattern.func[p] = ft + end + else + ok = false + end + end + end + end + + return ok +end + +local function parse_line(line) + local pat, ft + pat, ft = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)') + if pat then + return add_pattern(pat, ft) + else + local func + pat, func = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)') + if pat then + return add_pattern(pat, function() + return func + end) + end + end +end + +local unparsed = {} +local full_line +for line in io.lines(filetype_vim) do + local cont = string.match(line, '^%s*\\%s*(.*)$') + if cont then + full_line = full_line .. ' ' .. cont + else + if full_line then + if not parse_line(full_line) and string.find(full_line, '^%s*au%a* Buf') then + table.insert(unparsed, full_line) + end + end + full_line = line + end +end + +if #unparsed > 0 then + print('Failed to parse the following patterns:') + for _, v in ipairs(unparsed) do + print(v) + end +end + +local function add_item(indent, key, ft) + if type(ft) == 'string' then + if string.find(key, '%A') or keywords[key] then + key = string.format('["%s"]', key) + end + return string.format([[%s%s = "%s",]], indent, key, ft) + elseif type(ft) == 'function' then + local func = ft() + if string.find(key, '%A') or keywords[key] then + key = string.format('["%s"]', key) + end + -- Right now only a single argument is supported, which covers + -- everything in filetype.vim as of this writing + local arg = string.match(func, '%((.*)%)$') + func = string.gsub(func, '%(.*$', '') + if arg == '' then + -- Function with no arguments, call the function directly + return string.format([[%s%s = function() vim.fn["%s"]() end,]], indent, key, func) + elseif string.match(arg, [[^(["']).*%1$]]) then + -- String argument + if func == 's:StarSetf' then + return string.format([[%s%s = starsetf(%s),]], indent, key, arg) + else + return string.format([[%s%s = function() vim.fn["%s"](%s) end,]], indent, key, func, arg) + end + elseif string.find(arg, '%(') then + -- Function argument + return string.format( + [[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]], + indent, + key, + func, + arg + ) + else + assert(false, arg) + end + end +end + +do + local lines = {} + local start = false + for line in io.lines(filetype_lua) do + if line:match('^%s+-- END [A-Z]+$') then + start = false + end + + if not start then + table.insert(lines, line) + end + + local indent, section = line:match('^(%s+)-- BEGIN ([A-Z]+)$') + if section then + start = true + local t = sections[string.lower(section)] + + local sorted = {} + for k, v in pairs(t.str) do + table.insert(sorted, { [k] = v }) + end + + table.sort(sorted, function(a, b) + return a[next(a)] < b[next(b)] + end) + + for _, v in ipairs(sorted) do + local k = next(v) + table.insert(lines, add_item(indent, k, v[k])) + end + + sorted = {} + for k, v in pairs(t.func) do + table.insert(sorted, { [k] = v }) + end + + table.sort(sorted, function(a, b) + return next(a) < next(b) + end) + + for _, v in ipairs(sorted) do + local k = next(v) + table.insert(lines, add_item(indent, k, v[k])) + end + end + end + local f = io.open(filetype_lua, 'w') + f:write(table.concat(lines, '\n') .. '\n') + f:close() +end diff --git a/src/gen/gen_help_html.lua b/src/gen/gen_help_html.lua new file mode 100644 index 0000000000..53a65fd65f --- /dev/null +++ b/src/gen/gen_help_html.lua @@ -0,0 +1,1491 @@ +--- Converts Nvim :help files to HTML. Validates |tag| links and document syntax (parser errors). +-- +-- USAGE (For CI/local testing purposes): Simply `make lintdoc` or `scripts/lintdoc.lua`, which +-- basically does the following: +-- 1. :helptags ALL +-- 2. nvim -V1 -es +"lua require('scripts.gen_help_html').run_validate()" +q +-- 3. nvim -V1 -es +"lua require('scripts.gen_help_html').test_gen()" +q +-- +-- USAGE (GENERATE HTML): +-- 1. `:helptags ALL` first; this script depends on vim.fn.taglist(). +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./runtime/doc', 'target/dir/')" +q +-- - Read the docstring at gen(). +-- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 +-- 4. Visit http://localhost:4000/…/help.txt.html +-- +-- USAGE (VALIDATE): +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate('./runtime/doc')" +q +-- - validate() is 10x faster than gen(), so it is used in CI. +-- +-- SELF-TEST MODE: +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +q +-- +-- NOTES: +-- * This script is used by the automation repo: https://github.com/neovim/doc +-- * :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). +-- * gen() and validate() are the primary (programmatic) entrypoints. validate() only exists +-- because gen() is too slow (~1 min) to run in per-commit CI. +-- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. +-- * visit_validate() is the core function used by validate(). +-- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. + +local tagmap = nil ---@type table +local helpfiles = nil ---@type string[] +local invalid_links = {} ---@type table +local invalid_urls = {} ---@type table +local invalid_spelling = {} ---@type table> +local spell_dict = { + Neovim = 'Nvim', + NeoVim = 'Nvim', + neovim = 'Nvim', + lua = 'Lua', + VimL = 'Vimscript', + vimL = 'Vimscript', + viml = 'Vimscript', + ['tree-sitter'] = 'treesitter', + ['Tree-sitter'] = 'Treesitter', +} +--- specify the list of keywords to ignore (i.e. allow), or true to disable spell check completely. +--- @type table +local spell_ignore_files = { + ['credits.txt'] = { 'Neovim' }, + ['news.txt'] = { 'tree-sitter' }, -- in news, may refer to the upstream "tree-sitter" library + ['news-0.10.txt'] = { 'tree-sitter' }, +} +local language = nil + +local M = {} + +-- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). +-- All other files are "legacy" files which require fixed-width layout. +local new_layout = { + ['api.txt'] = true, + ['lsp.txt'] = true, + ['channel.txt'] = true, + ['deprecated.txt'] = true, + ['develop.txt'] = true, + ['dev_style.txt'] = true, + ['dev_theme.txt'] = true, + ['dev_tools.txt'] = true, + ['dev_vimpatch.txt'] = true, + ['editorconfig.txt'] = true, + ['faq.txt'] = true, + ['gui.txt'] = true, + ['intro.txt'] = true, + ['lua.txt'] = true, + ['luaref.txt'] = true, + ['news.txt'] = true, + ['news-0.9.txt'] = true, + ['news-0.10.txt'] = true, + ['nvim.txt'] = true, + ['provider.txt'] = true, + ['tui.txt'] = true, + ['ui.txt'] = true, + ['vim_diff.txt'] = true, +} + +-- Map of new:old pages, to redirect renamed pages. +local redirects = { + ['credits'] = 'backers', + ['tui'] = 'term', + ['terminal'] = 'nvim_terminal_emulator', +} + +-- TODO: These known invalid |links| require an update to the relevant docs. +local exclude_invalid = { + ["'string'"] = 'eval.txt', + Query = 'treesitter.txt', + matchit = 'vim_diff.txt', + ['set!'] = 'treesitter.txt', +} + +-- False-positive "invalid URLs". +local exclude_invalid_urls = { + ['http://'] = 'usr_23.txt', + ['http://.'] = 'usr_23.txt', + ['http://aspell.net/man-html/Affix-Compression.html'] = 'spell.txt', + ['http://aspell.net/man-html/Phonetic-Code.html'] = 'spell.txt', + ['http://canna.sourceforge.jp/'] = 'mbyte.txt', + ['http://gnuada.sourceforge.net'] = 'ft_ada.txt', + ['http://lua-users.org/wiki/StringLibraryTutorial'] = 'lua.txt', + ['http://michael.toren.net/code/'] = 'pi_tar.txt', + ['http://papp.plan9.de'] = 'syntax.txt', + ['http://wiki.services.openoffice.org/wiki/Dictionaries'] = 'spell.txt', + ['http://www.adapower.com'] = 'ft_ada.txt', + ['http://www.jclark.com/'] = 'quickfix.txt', + ['http://oldblog.antirez.com/post/redis-and-scripting.html'] = 'faq.txt', +} + +-- Deprecated, brain-damaged files that I don't care about. +local ignore_errors = { + ['pi_netrw.txt'] = true, + ['credits.txt'] = true, +} + +local function tofile(fname, text) + local f = io.open(fname, 'w') + if not f then + error(('failed to write: %s'):format(f)) + else + f:write(text) + f:close() + end +end + +---@type fun(s: string): string +local function html_esc(s) + return (s:gsub('&', '&'):gsub('<', '<'):gsub('>', '>')) +end + +local function url_encode(s) + -- Credit: tpope / vim-unimpaired + -- NOTE: these chars intentionally *not* escaped: ' ( ) + return vim.fn.substitute( + vim.fn.iconv(s, 'latin1', 'utf-8'), + [=[[^A-Za-z0-9()'_.~-]]=], + [=[\="%".printf("%02X",char2nr(submatch(0)))]=], + 'g' + ) +end + +local function expandtabs(s) + return s:gsub('\t', (' '):rep(8)) --[[ @as string ]] +end + +local function to_titlecase(s) + local text = '' + for w in vim.gsplit(s, '[ \t]+') do + text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2)) + end + return text +end + +local function to_heading_tag(text) + -- Prepend "_" to avoid conflicts with actual :help tags. + return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown' +end + +local function basename_noext(f) + return vim.fs.basename(f:gsub('%.txt', '')) +end + +local function is_blank(s) + return not not s:find([[^[\t ]*$]]) +end + +---@type fun(s: string, dir?:0|1|2): string +local function trim(s, dir) + return vim.fn.trim(s, '\r\t\n ', dir or 0) +end + +--- Removes common punctuation from URLs. +--- +--- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc +--- +--- @param url string +--- @return string, string (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. +local function fix_url(url) + local removed_chars = '' + local fixed_url = url + -- Remove up to one of each char from end of the URL, in this order. + for _, c in ipairs({ '.', ')' }) do + if fixed_url:sub(-1) == c then + removed_chars = c .. removed_chars + fixed_url = fixed_url:sub(1, -2) + end + end + return fixed_url, removed_chars +end + +--- Checks if a given line is a "noise" line that doesn't look good in HTML form. +local function is_noise(line, noise_lines) + if + -- First line is always noise. + (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0) + or line:find('Type .*gO.* to see the table of contents') + -- Title line of traditional :help pages. + -- Example: "NVIM REFERENCE MANUAL by ..." + or line:find([[^%s*N?VIM[ \t]*REFERENCE[ \t]*MANUAL]]) + -- First line of traditional :help pages. + -- Example: "*api.txt* Nvim" + or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') + -- modeline + -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" + or line:find('^%s*vim?%:.*ft=help') + or line:find('^%s*vim?%:.*filetype=help') + or line:find('[*>]local%-additions[*<]') + then + -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + table.insert(noise_lines or {}, line) + return true + end + return false +end + +--- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. +--- @return string +local function get_bug_url_vimdoc(fname, to_fname, sample_text) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ( + 'https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + .. vim.fs.basename(fname) + .. '+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' + .. this_url + .. '%0D%0DContext%3A%0D%0D%60%60%60%0D' + .. url_encode(sample_text) + .. '%0D%60%60%60' + ) + return bug_url +end + +--- Creates a github issue URL at neovim/neovim with prefilled content. +--- @return string +local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ( + 'https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' + .. vim.fs.basename(fname) + .. '+&body=%60gen_help_html.lua%60+problem+at%3A+' + .. this_url + .. '%0D' + .. (token_name and '+unhandled+token%3A+%60' .. token_name .. '%60' or '') + .. '%0DContext%3A%0D%0D%60%60%60%0D' + .. url_encode(sample_text) + .. '%0D%60%60%60' + ) + return bug_url +end + +--- Gets a "foo.html" name from a "foo.txt" helpfile name. +local function get_helppage(f) + if not f then + return nil + end + -- Special case: help.txt is the "main landing page" of :help files, not index.txt. + if f == 'index.txt' then + return 'vimindex.html' + elseif f == 'help.txt' then + return 'index.html' + end + + return (f:gsub('%.txt$', '')) .. '.html' +end + +--- Counts leading spaces (tab=8) to decide the indent size of multiline text. +--- +--- Blank lines (empty or whitespace-only) are ignored. +local function get_indent(s) + local min_indent = nil + for line in vim.gsplit(s, '\n') do + if line and not is_blank(line) then + local ws = expandtabs(line:match('^%s+') or '') + min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent + end + end + return min_indent or 0 +end + +--- Removes the common indent level, after expanding tabs to 8 spaces. +local function trim_indent(s) + local indent_size = get_indent(s) + local trimmed = '' + for line in vim.gsplit(s, '\n') do + line = expandtabs(line) + trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1)) + end + return trimmed:sub(1, -2) +end + +--- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. +---@param node TSNode +---@param bufnr integer +---@param offset integer +local function getbuflinestr(node, bufnr, offset) + local line1, _, line2, _ = node:range() + line1 = line1 - offset + line2 = line2 + offset + local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1) + return table.concat(lines, '\n') +end + +--- Gets the whitespace just before `node` from the raw buffer text. +--- Needed for preformatted `old` lines. +---@param node TSNode +---@param bufnr integer +---@return string +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + ---@type string + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + +local function get_tagname(node, bufnr) + local text = vim.treesitter.get_node_text(node, bufnr) + local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') + and ("'%s'"):format(text) + or text + local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" + return helppage, tag +end + +--- Returns true if the given invalid tagname is a false positive. +local function ignore_invalid(s) + return not not ( + exclude_invalid[s] + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + or s:find('===') + or s:find('%-%-%-') + ) +end + +local function ignore_parse_error(fname, s) + if ignore_errors[vim.fs.basename(fname)] then + return true + end + -- Ignore parse errors for unclosed tag. + -- This is common in vimdocs and is treated as plaintext by :help. + return s:find("^[`'|*]") +end + +---@param node TSNode +local function has_ancestor(node, ancestor_name) + local p = node ---@type TSNode? + while p do + p = p:parent() + if not p or p:type() == 'help_file' then + break + elseif p:type() == ancestor_name then + return true + end + end + return false +end + +--- Gets the first matching child node matching `name`. +---@param node TSNode +local function first(node, name) + for c, _ in node:iter_children() do + if c:named() and c:type() == name then + return c + end + end + return nil +end + +local function validate_link(node, bufnr, fname) + local helppage, tagname = get_tagname(node:child(1), bufnr) + local ignored = false + if not tagmap[tagname] then + ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname) + if not ignored then + invalid_links[tagname] = vim.fs.basename(fname) + end + end + return helppage, tagname, ignored +end + +--- TODO: port the logic from scripts/check_urls.vim +local function validate_url(text, fname) + local ignored = false + if ignore_errors[vim.fs.basename(fname)] then + ignored = true + elseif text:find('http%:') and not exclude_invalid_urls[text] then + invalid_urls[text] = vim.fs.basename(fname) + end + return ignored +end + +--- Traverses the tree at `root` and checks that |tag| links point to valid helptags. +---@param root TSNode +---@param level integer +---@param lang_tree TSTree +---@param opt table +---@param stats table +local function visit_validate(root, level, lang_tree, opt, stats) + level = level or 0 + local node_name = (root.named and root:named()) and root:type() or nil + -- Parent kind (string). + local parent = root:parent() and root:parent():type() or nil + local toplevel = level < 1 + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) + end + local text = trim(node_text()) + + if root:child_count() > 0 then + for node, _ in root:iter_children() do + if node:named() then + visit_validate(node, level + 1, lang_tree, opt, stats) + end + end + end + + if node_name == 'ERROR' then + if ignore_parse_error(opt.fname, text) then + return + end + -- Store the raw text to give context to the error report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 0) or '[top level!]' + -- Flatten the sample text to a single, truncated line. + sample_text = vim.trim(sample_text):gsub('[\t\n]', ' '):sub(1, 80) + table.insert(stats.parse_errors, sample_text) + elseif + (node_name == 'word' or node_name == 'uppercase_name') + and (not vim.tbl_contains({ 'codespan', 'taglink', 'tag' }, parent)) + then + local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. + local fname_basename = assert(vim.fs.basename(opt.fname)) + if spell_dict[text_nopunct] then + local should_ignore = ( + spell_ignore_files[fname_basename] == true + or vim.tbl_contains( + (spell_ignore_files[fname_basename] or {}) --[[ @as string[] ]], + text_nopunct + ) + ) + if not should_ignore then + invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} + invalid_spelling[text_nopunct][fname_basename] = node_text(root:parent()) + end + end + elseif node_name == 'url' then + local fixed_url, _ = fix_url(trim(text)) + validate_url(fixed_url, opt.fname) + elseif node_name == 'taglink' or node_name == 'optionlink' then + local _, _, _ = validate_link(root, opt.buf, opt.fname) + end +end + +-- Fix tab alignment issues caused by concealed characters like |, `, * in tags +-- and code blocks. +---@param text string +---@param next_node_text string +local function fix_tab_after_conceal(text, next_node_text) + -- Vim tabs take into account the two concealed characters even though they + -- are invisible, so we need to add back in the two spaces if this is + -- followed by a tab to make the tab alignment to match Vim's behavior. + if string.sub(next_node_text, 1, 1) == '\t' then + text = text .. ' ' + end + return text +end + +---@class (exact) nvim.gen_help_html.heading +---@field name string +---@field subheadings nvim.gen_help_html.heading[] +---@field tag string + +-- Generates HTML from node `root` recursively. +---@param root TSNode +---@param level integer +---@param lang_tree TSTree +---@param headings nvim.gen_help_html.heading[] +---@param opt table +---@param stats table +local function visit_node(root, level, lang_tree, headings, opt, stats) + level = level or 0 + + local node_name = (root.named and root:named()) and root:type() or nil + -- Previous sibling kind (string). + local prev = root:prev_sibling() + and (root:prev_sibling().named and root:prev_sibling():named()) + and root:prev_sibling():type() + or nil + -- Next sibling kind (string). + local next_ = root:next_sibling() + and (root:next_sibling().named and root:next_sibling():named()) + and root:next_sibling():type() + or nil + -- Parent kind (string). + local parent = root:parent() and root:parent():type() or nil + -- Gets leading whitespace of `node`. + local function ws(node) + node = node or root + local ws_ = getws(node, opt.buf) + -- XXX: first node of a (line) includes whitespace, even after + -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ? + if ws_ == '' then + ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or '' + end + return ws_ + end + local function node_text(node, ws_) + node = node or root + ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or '' + return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) + end + + local text = '' + local trimmed ---@type string + if root:named_child_count() == 0 or node_name == 'ERROR' then + text = node_text() + trimmed = html_esc(trim(text)) + text = html_esc(text) + else + -- Process children and join them with whitespace. + for node, _ in root:iter_children() do + if node:named() then + local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) + text = string.format('%s%s', text, r) + end + end + trimmed = trim(text) + end + + if node_name == 'help_file' then -- root node + return text + elseif node_name == 'url' then + local fixed_url, removed_chars = fix_url(trimmed) + return ('%s%s%s'):format(ws(), fixed_url, fixed_url, removed_chars) + elseif node_name == 'word' or node_name == 'uppercase_name' then + return text + elseif node_name == 'note' then + return ('%s'):format(text) + elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then + if is_noise(text, stats.noise_lines) then + return '' -- Discard common "noise" lines. + end + -- Remove tags from ToC text. + local heading_node = first(root, 'heading') + local hname = trim(node_text(heading_node):gsub('%*.*%*', '')) + if not heading_node or hname == '' then + return '' -- Spurious "===" or "---" in the help doc. + end + + -- Generate an anchor id from the heading text. + local tagname = to_heading_tag(hname) + if node_name == 'h1' or #headings == 0 then + ---@type nvim.gen_help_html.heading + local heading = { name = hname, subheadings = {}, tag = tagname } + headings[#headings + 1] = heading + else + table.insert( + headings[#headings].subheadings, + { name = hname, subheadings = {}, tag = tagname } + ) + end + local el = node_name == 'h1' and 'h2' or 'h3' + return ('<%s id="%s" class="help-heading">%s\n'):format(el, tagname, trimmed, el) + elseif node_name == 'heading' then + return trimmed + elseif node_name == 'column_heading' or node_name == 'column_name' then + if root:has_error() then + return text + end + return ('
%s
'):format(text) + elseif node_name == 'block' then + if is_blank(text) then + return '' + end + if opt.old then + -- XXX: Treat "old" docs as preformatted: they use indentation for layout. + -- Trim trailing newlines to avoid too much whitespace between divs. + return ('
%s
\n'):format(trim(text, 2)) + end + return string.format('
\n%s\n
\n', text) + elseif node_name == 'line' then + if + (parent ~= 'codeblock' or parent ~= 'code') + and (is_blank(text) or is_noise(text, stats.noise_lines)) + then + return '' -- Discard common "noise" lines. + end + -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. + local div = opt.old + and root:child(0) + and vim.list_contains({ 'column_heading', 'h1', 'h2', 'h3' }, root:child(0):type()) + return string.format('%s%s', div and trim(text) or text, div and '' or '\n') + elseif node_name == 'line_li' then + local sib = root:prev_sibling() + local prev_li = sib and sib:type() == 'line_li' + + if not prev_li then + opt.indent = 1 + else + -- The previous listitem _sibling_ is _logically_ the _parent_ if it is indented less. + local parent_indent = get_indent(node_text(sib)) + local this_indent = get_indent(node_text()) + if this_indent > parent_indent then + opt.indent = opt.indent + 1 + elseif this_indent < parent_indent then + opt.indent = math.max(1, opt.indent - 1) + end + end + local margin = opt.indent == 1 and '' or ('margin-left: %drem;'):format((1.5 * opt.indent)) + + return string.format('
%s
', margin, text) + elseif node_name == 'taglink' or node_name == 'optionlink' then + local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname) + if ignored then + return text + end + local s = ('%s%s'):format( + ws(), + helppage, + url_encode(tagname), + html_esc(tagname) + ) + if opt.old and node_name == 'taglink' then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + return s + elseif vim.list_contains({ 'codespan', 'keycode' }, node_name) then + if root:has_error() then + return text + end + local s = ('%s%s'):format(ws(), trimmed) + if opt.old and node_name == 'codespan' then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + return s + elseif node_name == 'argument' then + return ('%s{%s}'):format(ws(), text) + elseif node_name == 'codeblock' then + return text + elseif node_name == 'language' then + language = node_text(root) + return '' + elseif node_name == 'code' then -- Highlighted codeblock (child). + if is_blank(text) then + return '' + end + local code ---@type string + if language then + code = ('
%s
'):format( + language, + trim(trim_indent(text), 2) + ) + language = nil + else + code = ('
%s
'):format(trim(trim_indent(text), 2)) + end + return code + elseif node_name == 'tag' then -- anchor, h4 pseudo-heading + if root:has_error() then + return text + end + local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent) + local h4 = not in_heading and not next_ and get_indent(node_text()) > 8 -- h4 pseudo-heading + local cssclass = h4 and 'help-tag-right' or 'help-tag' + local tagname = node_text(root:child(1), false) + if vim.tbl_count(stats.first_tags) < 2 then + -- Force the first 2 tags in the doc to be anchored at the main heading. + table.insert(stats.first_tags, tagname) + return '' + end + local el = 'span' + local encoded_tagname = url_encode(tagname) + local s = ('%s<%s id="%s" class="%s">%s'):format( + ws(), + el, + encoded_tagname, + cssclass, + encoded_tagname, + trimmed, + el + ) + if opt.old then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + + if in_heading and prev ~= 'tag' then + -- Start the container for tags in a heading. + -- This makes "justify-content:space-between" right-align the tags. + --

foo bartag1 tag2

+ return string.format('%s', s) + elseif in_heading and next_ == nil then + -- End the container for tags in a heading. + return string.format('%s', s) + end + return s .. (h4 and '
' or '') -- HACK:
avoids h4 pseudo-heading mushing with text. + elseif node_name == 'delimiter' or node_name == 'modeline' then + return '' + elseif node_name == 'ERROR' then + if ignore_parse_error(opt.fname, trimmed) then + return text + end + + -- Store the raw text to give context to the bug report. + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + return ('%s'):format( + get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), + trimmed + ) + else -- Unknown token. + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' + return ('%s'):format( + node_name, + get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), + trimmed + ), + ('unknown-token:"%s"'):format(node_name) + end +end + +--- @param dir string e.g. '$VIMRUNTIME/doc' +--- @param include string[]|nil +--- @return string[] +local function get_helpfiles(dir, include) + local rv = {} + for f, type in vim.fs.dir(dir) do + if + vim.endswith(f, '.txt') + and type == 'file' + and (not include or vim.list_contains(include, f)) + then + local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') + table.insert(rv, fullpath) + end + end + return rv +end + +--- Populates the helptags map. +local function get_helptags(help_dir) + local m = {} + -- Load a random help file to convince taglist() to do its job. + vim.cmd(string.format('split %s/api.txt', help_dir)) + vim.cmd('lcd %:p:h') + for _, item in ipairs(vim.fn.taglist('.*')) do + if vim.endswith(item.filename, '.txt') then + m[item.name] = item.filename + end + end + vim.cmd('q!') + return m +end + +--- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system. +local function ensure_runtimepath() + if not vim.o.runtimepath:find('build/lib/nvim/') then + vim.cmd [[set runtimepath^=./build/lib/nvim/]] + end +end + +--- Opens `fname` (or `text`, if given) in a buffer and gets a treesitter parser for the buffer contents. +--- +--- @param fname string :help file to parse +--- @param text string? :help file contents +--- @param parser_path string? path to non-default vimdoc.so +--- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr) +local function parse_buf(fname, text, parser_path) + local buf ---@type integer + if text then + vim.cmd('split new') -- Text contents. + vim.api.nvim_put(vim.split(text, '\n'), '', false, false) + vim.cmd('setfiletype help') + -- vim.treesitter.language.add('vimdoc') + buf = vim.api.nvim_get_current_buf() + elseif type(fname) == 'string' then + vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename. + buf = vim.api.nvim_get_current_buf() + else + -- Left for debugging + ---@diagnostic disable-next-line: no-unknown + buf = fname + vim.cmd('sbuffer ' .. tostring(fname)) -- Buffer number. + end + if parser_path then + vim.treesitter.language.add('vimdoc', { path = parser_path }) + end + local lang_tree = assert(vim.treesitter.get_parser(buf, nil, { error = false })) + return lang_tree, buf +end + +--- Validates one :help file `fname`: +--- - checks that |tag| links point to valid helptags. +--- - recursively counts parse errors ("ERROR" nodes) +--- +--- @param fname string help file to validate +--- @param parser_path string? path to non-default vimdoc.so +--- @return { invalid_links: number, parse_errors: string[] } +local function validate_one(fname, parser_path) + local stats = { + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname, nil, parser_path) + for _, tree in ipairs(lang_tree:trees()) do + visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats) + end + lang_tree:destroy() + vim.cmd.close() + return stats +end + +--- Generates HTML from one :help file `fname` and writes the result to `to_fname`. +--- +--- @param fname string Source :help file. +--- @param text string|nil Source :help file contents, or nil to read `fname`. +--- @param to_fname string Destination .html file +--- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) +--- @param parser_path string? path to non-default vimdoc.so +--- +--- @return string html +--- @return table stats +local function gen_one(fname, text, to_fname, old, commit, parser_path) + local stats = { + noise_lines = {}, + parse_errors = {}, + first_tags = {}, -- Track the first few tags in doc. + } + local lang_tree, buf = parse_buf(fname, text, parser_path) + ---@type nvim.gen_help_html.heading[] + local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. + local title = to_titlecase(basename_noext(fname)) + + local html = ([[ + + + + + + + + + + + + + + + + + + + + %s - Neovim docs + + + ]]):format(title) + + local logo_svg = [[ + + Neovim + + + + + + + + + + + + + + + + + + + + + + + + + + ]] + + local main = '' + for _, tree in ipairs(lang_tree:trees()) do + main = main + .. ( + visit_node( + tree:root(), + 0, + tree, + headings, + { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1 }, + stats + ) + ) + end + + main = ([[ +
+ +
+ +
+
+

%s

 +

+ + Nvim :help pages, generated + from source + using the tree-sitter-vimdoc parser. + +

+
+ %s +
+ ]]):format( + logo_svg, + stats.first_tags[1] or '', + stats.first_tags[2] or '', + stats.first_tags[2] or '', + title, + vim.fs.basename(fname), + main + ) + + ---@type string + local toc = [[ +
+ + + +
+ ]] + + local n = 0 -- Count of all headings + subheadings. + for _, h1 in ipairs(headings) do + n = n + 1 + #h1.subheadings + end + for _, h1 in ipairs(headings) do + ---@type string + toc = toc .. ('
%s\n'):format(h1.tag, h1.name) + if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. + for _, h2 in ipairs(h1.subheadings) do + toc = toc + .. ('
%s
\n'):format(h2.tag, h2.name) + end + end + toc = toc .. '
' + end + toc = toc .. '
\n' + + local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil) + local bug_link = string.format('(report docs bug...)', bug_url) + + local footer = ([[ +
+
+
+ Generated at %s from %s +
+
+ parse_errors: %d %s | noise_lines: %d +
+
+ + + + + +
+ ]]):format( + os.date('%Y-%m-%d %H:%M'), + commit, + commit:sub(1, 7), + #stats.parse_errors, + bug_link, + html_esc(table.concat(stats.noise_lines, '\n')), + #stats.noise_lines + ) + + html = ('%s%s%s
\n%s\n\n'):format(html, main, toc, footer) + vim.cmd('q!') + lang_tree:destroy() + return html, stats +end + +local function gen_css(fname) + local css = [[ + :root { + --code-color: #004b4b; + --tag-color: #095943; + } + @media (prefers-color-scheme: dark) { + :root { + --code-color: #00c243; + --tag-color: #00b7b7; + } + } + @media (min-width: 40em) { + .toc { + position: fixed; + left: 67%; + } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + } + @media (max-width: 40em) { + .golden-grid { + /* Disable grid for narrow viewport (mobile phone). */ + display: block; + } + } + .toc { + /* max-width: 12rem; */ + height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + } + .toc > div { + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + } + html { + scroll-behavior: auto; + } + body { + font-size: 18px; + line-height: 1.5; + } + h1, h2, h3, h4, h5 { + font-family: sans-serif; + border-bottom: 1px solid var(--tag-color); /*rgba(0, 0, 0, .9);*/ + } + h3, h4, h5 { + border-bottom-style: dashed; + } + .help-column_heading { + color: var(--code-color); + } + .help-body { + padding-bottom: 2em; + } + .help-line { + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + } + .help-li { + white-space: normal; + display: list-item; + margin-left: 1.5rem; /* padding-left: 1rem; */ + } + .help-para { + padding-top: 10px; + padding-bottom: 10px; + } + + .old-help-para { + padding-top: 10px; + padding-bottom: 10px; + /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */ + tab-size: 8; + white-space: pre-wrap; + font-size: 16px; + font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; + word-wrap: break-word; + } + .old-help-para pre, .old-help-para pre:hover { + /* Text following 
 is already visually separated by the linebreak. */
+      margin-bottom: 0;
+      /* Long lines that exceed the textwidth should not be wrapped (no "pre-wrap").
+         Since text may overflow horizontally, we make the contents to be scrollable
+         (only if necessary) to prevent overlapping with the navigation bar at the right. */
+      white-space: pre;
+      overflow-x: auto;
+    }
+
+    /* TODO: should this rule be deleted? help tags are rendered as  or , not  */
+    a.help-tag, a.help-tag:focus, a.help-tag:hover {
+      color: inherit;
+      text-decoration: none;
+    }
+    .help-tag {
+      color: var(--tag-color);
+    }
+    /* Tag pseudo-header common in :help docs. */
+    .help-tag-right {
+      color: var(--tag-color);
+      margin-left: auto;
+      margin-right: 0;
+      float: right;
+      display: block;
+    }
+    .help-tag a,
+    .help-tag-right a {
+      color: inherit;
+    }
+    .help-tag a:not(:hover),
+    .help-tag-right a:not(:hover) {
+      text-decoration: none;
+    }
+    h1 .help-tag, h2 .help-tag, h3 .help-tag {
+      font-size: smaller;
+    }
+    .help-heading {
+      white-space: normal;
+      display: flex;
+      flex-flow: row wrap;
+      justify-content: space-between;
+      gap: 0 15px;
+    }
+    /* The (right-aligned) "tags" part of a section heading. */
+    .help-heading-tags {
+      margin-right: 10px;
+    }
+    .help-toc-h1 {
+    }
+    .help-toc-h2 {
+      margin-left: 1em;
+    }
+    .parse-error {
+      background-color: red;
+    }
+    .unknown-token {
+      color: black;
+      background-color: yellow;
+    }
+    code {
+      color: var(--code-color);
+      font-size: 16px;
+    }
+    pre {
+      /* Tabs are used in codeblocks only for indentation, not alignment, so we can aggressively shrink them. */
+      tab-size: 2;
+      white-space: pre-wrap;
+      line-height: 1.3;  /* Important for ascii art. */
+      overflow: visible;
+      /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */
+      font-size: 16px;
+      margin-top: 10px;
+    }
+    pre:last-child {
+      margin-bottom: 0;
+    }
+    pre:hover {
+      overflow: visible;
+    }
+    .generator-stats {
+      color: gray;
+      font-size: smaller;
+    }
+  ]]
+  tofile(fname, css)
+end
+
+-- Testing
+
+local function ok(cond, expected, actual, message)
+  assert(
+    (not expected and not actual) or (expected and actual),
+    'if "expected" is given, "actual" is also required'
+  )
+  if expected then
+    assert(
+      cond,
+      ('%sexpected %s, got: %s'):format(
+        message and (message .. '\n') or '',
+        vim.inspect(expected),
+        vim.inspect(actual)
+      )
+    )
+    return cond
+  else
+    return assert(cond)
+  end
+end
+local function eq(expected, actual, message)
+  return ok(vim.deep_equal(expected, actual), expected, actual, message)
+end
+
+function M._test()
+  tagmap = get_helptags('$VIMRUNTIME/doc')
+  helpfiles = get_helpfiles(vim.fs.normalize('$VIMRUNTIME/doc'))
+
+  ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap))
+  ok(
+    vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'),
+    tagmap['vim.diagnostic.set()'],
+    'diagnostic.txt'
+  )
+  ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt')
+  ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]]))
+  ok(is_noise([[          NVIM  REFERENCE  MANUAL     by  Thiago  de  Arruda      ]]))
+  ok(not is_noise([[vim:tw=78]]))
+
+  eq(0, get_indent('a'))
+  eq(1, get_indent(' a'))
+  eq(2, get_indent('  a\n  b\n  c\n'))
+  eq(5, get_indent('     a\n      \n        b\n      c\n      d\n      e\n'))
+  eq(
+    'a\n        \n   b\n c\n d\n e\n',
+    trim_indent('     a\n             \n        b\n      c\n      d\n      e\n')
+  )
+
+  local fixed_url, removed_chars = fix_url('https://example.com).')
+  eq('https://example.com', fixed_url)
+  eq(').', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com.)')
+  eq('https://example.com.', fixed_url)
+  eq(')', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com.')
+  eq('https://example.com', fixed_url)
+  eq('.', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com)')
+  eq('https://example.com', fixed_url)
+  eq(')', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com')
+  eq('https://example.com', fixed_url)
+  eq('', removed_chars)
+
+  print('all tests passed.\n')
+end
+
+--- @class nvim.gen_help_html.gen_result
+--- @field helpfiles string[] list of generated HTML files, from the source docs {include}
+--- @field err_count integer number of parse errors in :help docs
+--- @field invalid_links table
+
+--- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`.
+---
+--- Example:
+---
+---   gen('$VIMRUNTIME/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil)
+---
+--- @param help_dir string Source directory containing the :help files. Must run `make helptags` first.
+--- @param to_dir string Target directory where the .html files will be written.
+--- @param include string[]|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'}
+---
+--- @return nvim.gen_help_html.gen_result result
+function M.gen(help_dir, to_dir, include, commit, parser_path)
+  vim.validate('help_dir', help_dir, function(d)
+    return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
+  end, 'valid directory')
+  vim.validate('to_dir', to_dir, 'string')
+  vim.validate('include', include, 'table', true)
+  vim.validate('commit', commit, 'string', true)
+  vim.validate('parser_path', parser_path, function(f)
+    return vim.fn.filereadable(vim.fs.normalize(f)) == 1
+  end, true, 'valid vimdoc.{so,dll} filepath')
+
+  local err_count = 0
+  local redirects_count = 0
+  ensure_runtimepath()
+  tagmap = get_helptags(vim.fs.normalize(help_dir))
+  helpfiles = get_helpfiles(help_dir, include)
+  to_dir = vim.fs.normalize(to_dir)
+  parser_path = parser_path and vim.fs.normalize(parser_path) or nil
+
+  print(('output dir: %s\n\n'):format(to_dir))
+  vim.fn.mkdir(to_dir, 'p')
+  gen_css(('%s/help.css'):format(to_dir))
+
+  for _, f in ipairs(helpfiles) do
+    -- "foo.txt"
+    local helpfile = vim.fs.basename(f)
+    -- "to/dir/foo.html"
+    local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile))
+    local html, stats =
+      gen_one(f, nil, to_fname, not new_layout[helpfile], commit or '?', parser_path)
+    tofile(to_fname, html)
+    print(
+      ('generated (%-2s errors): %-15s => %s'):format(
+        #stats.parse_errors,
+        helpfile,
+        vim.fs.basename(to_fname)
+      )
+    )
+
+    -- Generate redirect pages for renamed help files.
+    local helpfile_tag = (helpfile:gsub('%.txt$', ''))
+    local redirect_from = redirects[helpfile_tag]
+    if redirect_from then
+      local redirect_text = ([[
+*%s*      Nvim
+
+This document moved to: |%s|
+
+==============================================================================
+This document moved to: |%s|
+
+This document moved to: |%s|
+
+==============================================================================
+ vim:tw=78:ts=8:ft=help:norl:
+      ]]):format(
+        redirect_from,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag
+      )
+      local redirect_to = ('%s/%s'):format(to_dir, get_helppage(redirect_from))
+      local redirect_html, _ =
+        gen_one(redirect_from, redirect_text, redirect_to, false, commit or '?', parser_path)
+      assert(redirect_html:find(helpfile_tag))
+      tofile(redirect_to, redirect_html)
+
+      print(
+        ('generated (redirect) : %-15s => %s'):format(
+          redirect_from .. '.txt',
+          vim.fs.basename(to_fname)
+        )
+      )
+      redirects_count = redirects_count + 1
+    end
+
+    err_count = err_count + #stats.parse_errors
+  end
+
+  print(('\ngenerated %d html pages'):format(#helpfiles + redirects_count))
+  print(('total errors: %d'):format(err_count))
+  print(('invalid tags: %s'):format(vim.inspect(invalid_links)))
+  assert(#(include or {}) > 0 or redirects_count == vim.tbl_count(redirects)) -- sanity check
+  print(('redirects: %d'):format(redirects_count))
+  print('\n')
+
+  --- @type nvim.gen_help_html.gen_result
+  return {
+    helpfiles = helpfiles,
+    err_count = err_count,
+    invalid_links = invalid_links,
+  }
+end
+
+--- @class nvim.gen_help_html.validate_result
+--- @field helpfiles integer number of generated helpfiles
+--- @field err_count integer number of parse errors
+--- @field parse_errors table
+--- @field invalid_links table invalid tags in :help docs
+--- @field invalid_urls table invalid URLs in :help docs
+--- @field invalid_spelling table> invalid spelling in :help docs
+
+--- Validates all :help files found in `help_dir`:
+---  - checks that |tag| links point to valid helptags.
+---  - recursively counts parse errors ("ERROR" nodes)
+---
+--- This is 10x faster than gen(), for use in CI.
+---
+--- @return nvim.gen_help_html.validate_result result
+function M.validate(help_dir, include, parser_path)
+  vim.validate('help_dir', help_dir, function(d)
+    return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
+  end, 'valid directory')
+  vim.validate('include', include, 'table', true)
+  vim.validate('parser_path', parser_path, function(f)
+    return vim.fn.filereadable(vim.fs.normalize(f)) == 1
+  end, true, 'valid vimdoc.{so,dll} filepath')
+  local err_count = 0 ---@type integer
+  local files_to_errors = {} ---@type table
+  ensure_runtimepath()
+  tagmap = get_helptags(vim.fs.normalize(help_dir))
+  helpfiles = get_helpfiles(help_dir, include)
+  parser_path = parser_path and vim.fs.normalize(parser_path) or nil
+
+  for _, f in ipairs(helpfiles) do
+    local helpfile = vim.fs.basename(f)
+    local rv = validate_one(f, parser_path)
+    print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile))
+    if #rv.parse_errors > 0 then
+      files_to_errors[helpfile] = rv.parse_errors
+      vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v)
+        return s .. '\n    ' .. v
+      end)))
+    end
+    err_count = err_count + #rv.parse_errors
+  end
+
+  ---@type nvim.gen_help_html.validate_result
+  return {
+    helpfiles = #helpfiles,
+    err_count = err_count,
+    parse_errors = files_to_errors,
+    invalid_links = invalid_links,
+    invalid_urls = invalid_urls,
+    invalid_spelling = invalid_spelling,
+  }
+end
+
+--- Validates vimdoc files on $VIMRUNTIME. and print human-readable error messages if fails.
+---
+--- If this fails, try these steps (in order):
+--- 1. Fix/cleanup the :help docs.
+--- 2. Fix the parser: https://github.com/neovim/tree-sitter-vimdoc
+--- 3. File a parser bug, and adjust the tolerance of this test in the meantime.
+---
+--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
+function M.run_validate(help_dir)
+  help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
+  print('doc path = ' .. vim.uv.fs_realpath(help_dir))
+
+  local rv = M.validate(help_dir)
+
+  -- Check that we actually found helpfiles.
+  ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles)
+
+  eq({}, rv.parse_errors, 'no parse errors')
+  eq(0, rv.err_count, 'no parse errors')
+  eq({}, rv.invalid_links, 'invalid tags in :help docs')
+  eq({}, rv.invalid_urls, 'invalid URLs in :help docs')
+  eq(
+    {},
+    rv.invalid_spelling,
+    'invalid spelling in :help docs (see spell_dict in scripts/gen_help_html.lua)'
+  )
+end
+
+--- Test-generates HTML from docs.
+---
+--- 1. Test that gen_help_html.lua actually works.
+--- 2. Test that parse errors did not increase wildly. Because we explicitly test only a few
+---    :help files, we can be precise about the tolerances here.
+--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
+function M.test_gen(help_dir)
+  local tmpdir = vim.fs.dirname(vim.fn.tempname())
+  help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
+  print('doc path = ' .. vim.uv.fs_realpath(help_dir))
+
+  -- Because gen() is slow (~30s), this test is limited to a few files.
+  local input = { 'help.txt', 'index.txt', 'nvim.txt' }
+  local rv = M.gen(help_dir, tmpdir, input)
+  eq(#input, #rv.helpfiles)
+  eq(0, rv.err_count, 'parse errors in :help docs')
+  eq({}, rv.invalid_links, 'invalid tags in :help docs')
+end
+
+return M
diff --git a/src/gen/gen_lsp.lua b/src/gen/gen_lsp.lua
new file mode 100644
index 0000000000..38792307e4
--- /dev/null
+++ b/src/gen/gen_lsp.lua
@@ -0,0 +1,514 @@
+-- Generates lua-ls annotations for lsp.
+
+local USAGE = [[
+Generates lua-ls annotations for lsp.
+
+USAGE:
+nvim -l src/gen/gen_lsp.lua gen  # by default, this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua
+nvim -l src/gen/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua
+nvim -l src/gen/gen_lsp.lua gen --version 3.18 --methods --capabilities
+]]
+
+local DEFAULT_LSP_VERSION = '3.18'
+
+local M = {}
+
+local function tofile(fname, text)
+  local f = io.open(fname, 'w')
+  if not f then
+    error(('failed to write: %s'):format(f))
+  else
+    print(('Written to: %s'):format(fname))
+    f:write(text)
+    f:close()
+  end
+end
+
+--- The LSP protocol JSON data (it's partial, non-exhaustive).
+--- https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/3.18/metaModel/metaModel.schema.json
+--- @class vim._gen_lsp.Protocol
+--- @field requests vim._gen_lsp.Request[]
+--- @field notifications vim._gen_lsp.Notification[]
+--- @field structures vim._gen_lsp.Structure[]
+--- @field enumerations vim._gen_lsp.Enumeration[]
+--- @field typeAliases vim._gen_lsp.TypeAlias[]
+
+---@param opt vim._gen_lsp.opt
+---@return vim._gen_lsp.Protocol
+local function read_json(opt)
+  local uri = 'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/'
+    .. opt.version
+    .. '/metaModel/metaModel.json'
+  print('Reading ' .. uri)
+
+  local res = vim.system({ 'curl', '--no-progress-meter', uri, '-o', '-' }):wait()
+  if res.code ~= 0 or (res.stdout or ''):len() < 999 then
+    print(('URL failed: %s'):format(uri))
+    vim.print(res)
+    error(res.stdout)
+  end
+  return vim.json.decode(res.stdout)
+end
+
+-- Gets the Lua symbol for a given fully-qualified LSP method name.
+local function to_luaname(s)
+  -- "$/" prefix is special: https://microsoft.github.io/language-server-protocol/specification/#dollarRequests
+  return s:gsub('^%$', 'dollar'):gsub('/', '_')
+end
+
+---@param protocol vim._gen_lsp.Protocol
+---@param gen_methods boolean
+---@param gen_capabilities boolean
+local function write_to_protocol(protocol, gen_methods, gen_capabilities)
+  if not gen_methods and not gen_capabilities then
+    return
+  end
+
+  local indent = (' '):rep(2)
+
+  --- @class vim._gen_lsp.Request
+  --- @field deprecated? string
+  --- @field documentation? string
+  --- @field messageDirection string
+  --- @field clientCapability? string
+  --- @field serverCapability? string
+  --- @field method string
+  --- @field params? any
+  --- @field proposed? boolean
+  --- @field registrationMethod? string
+  --- @field registrationOptions? any
+  --- @field since? string
+
+  --- @class vim._gen_lsp.Notification
+  --- @field deprecated? string
+  --- @field documentation? string
+  --- @field errorData? any
+  --- @field messageDirection string
+  --- @field clientCapability? string
+  --- @field serverCapability? string
+  --- @field method string
+  --- @field params? any[]
+  --- @field partialResult? any
+  --- @field proposed? boolean
+  --- @field registrationMethod? string
+  --- @field registrationOptions? any
+  --- @field result any
+  --- @field since? string
+
+  ---@type (vim._gen_lsp.Request|vim._gen_lsp.Notification)[]
+  local all = vim.list_extend(protocol.requests, protocol.notifications)
+  table.sort(all, function(a, b)
+    return to_luaname(a.method) < to_luaname(b.method)
+  end)
+
+  local output = { '-- Generated by gen_lsp.lua, keep at end of file.' }
+
+  if gen_methods then
+    output[#output + 1] = '--- @alias vim.lsp.protocol.Method.ClientToServer'
+
+    for _, item in ipairs(all) do
+      if item.method and item.messageDirection == 'clientToServer' then
+        output[#output + 1] = ("--- | '%s',"):format(item.method)
+      end
+    end
+
+    vim.list_extend(output, {
+      '',
+      '--- @alias vim.lsp.protocol.Method.ServerToClient',
+    })
+    for _, item in ipairs(all) do
+      if item.method and item.messageDirection == 'serverToClient' then
+        output[#output + 1] = ("--- | '%s',"):format(item.method)
+      end
+    end
+
+    vim.list_extend(output, {
+      '',
+      '--- @alias vim.lsp.protocol.Method',
+      '--- | vim.lsp.protocol.Method.ClientToServer',
+      '--- | vim.lsp.protocol.Method.ServerToClient',
+      '',
+      '-- Generated by gen_lsp.lua, keep at end of file.',
+      '--- @enum vim.lsp.protocol.Methods',
+      '--- @see https://microsoft.github.io/language-server-protocol/specification/#metaModel',
+      '--- LSP method names.',
+      'protocol.Methods = {',
+    })
+
+    for _, item in ipairs(all) do
+      if item.method then
+        if item.documentation then
+          local document = vim.split(item.documentation, '\n?\n', { trimempty = true })
+          for _, docstring in ipairs(document) do
+            output[#output + 1] = indent .. '--- ' .. docstring
+          end
+        end
+        output[#output + 1] = ("%s%s = '%s',"):format(indent, to_luaname(item.method), item.method)
+      end
+    end
+    output[#output + 1] = '}'
+  end
+
+  if gen_capabilities then
+    vim.list_extend(output, {
+      '',
+      '-- stylua: ignore start',
+      '-- Generated by gen_lsp.lua, keep at end of file.',
+      '--- Maps method names to the required server capability',
+      'protocol._request_name_to_capability = {',
+    })
+
+    for _, item in ipairs(all) do
+      if item.serverCapability then
+        output[#output + 1] = ("%s['%s'] = { %s },"):format(
+          indent,
+          item.method,
+          table.concat(
+            vim
+              .iter(vim.split(item.serverCapability, '.', { plain = true }))
+              :map(function(segment)
+                return "'" .. segment .. "'"
+              end)
+              :totable(),
+            ', '
+          )
+        )
+      end
+    end
+
+    output[#output + 1] = '}'
+    output[#output + 1] = '-- stylua: ignore end'
+  end
+
+  output[#output + 1] = ''
+  output[#output + 1] = 'return protocol'
+
+  local fname = './runtime/lua/vim/lsp/protocol.lua'
+  local bufnr = vim.fn.bufadd(fname)
+  vim.fn.bufload(bufnr)
+  vim.api.nvim_set_current_buf(bufnr)
+  local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false)
+  local index = vim.iter(ipairs(lines)):find(function(key, item)
+    return vim.startswith(item, '-- Generated by') and key or nil
+  end)
+  index = index and index - 1 or vim.api.nvim_buf_line_count(bufnr) - 1
+  vim.api.nvim_buf_set_lines(bufnr, index, -1, true, output)
+  vim.cmd.write()
+end
+
+---@class vim._gen_lsp.opt
+---@field output_file string
+---@field version string
+---@field methods boolean
+---@field capabilities boolean
+
+---@param opt vim._gen_lsp.opt
+function M.gen(opt)
+  --- @type vim._gen_lsp.Protocol
+  local protocol = read_json(opt)
+
+  write_to_protocol(protocol, opt.methods, opt.capabilities)
+
+  local output = {
+    '--' .. '[[',
+    'THIS FILE IS GENERATED by scr/gen/gen_lsp.lua',
+    'DO NOT EDIT MANUALLY',
+    '',
+    'Based on LSP protocol ' .. opt.version,
+    '',
+    'Regenerate:',
+    ([=[nvim -l scr/gen/gen_lsp.lua gen --version %s]=]):format(DEFAULT_LSP_VERSION),
+    '--' .. ']]',
+    '',
+    '---@meta',
+    "error('Cannot require a meta file')",
+    '',
+    '---@alias lsp.null nil',
+    '---@alias uinteger integer',
+    '---@alias decimal number',
+    '---@alias lsp.DocumentUri string',
+    '---@alias lsp.URI string',
+    '',
+  }
+
+  local anonymous_num = 0
+
+  ---@type string[]
+  local anonym_classes = {}
+
+  local simple_types = {
+    'string',
+    'boolean',
+    'integer',
+    'uinteger',
+    'decimal',
+  }
+
+  ---@param documentation string
+  local _process_documentation = function(documentation)
+    documentation = documentation:gsub('\n', '\n---')
+    -- Remove <200b> (zero-width space) unicode characters: e.g., `**/<200b>*`
+    documentation = documentation:gsub('\226\128\139', '')
+    -- Escape annotations that are not recognized by lua-ls
+    documentation = documentation:gsub('%^---@sample', '---\\@sample')
+    return '---' .. documentation
+  end
+
+  --- @class vim._gen_lsp.Type
+  --- @field kind string a common field for all Types.
+  --- @field name? string for ReferenceType, BaseType
+  --- @field element? any for ArrayType
+  --- @field items? vim._gen_lsp.Type[] for OrType, AndType
+  --- @field key? vim._gen_lsp.Type for MapType
+  --- @field value? string|vim._gen_lsp.Type for StringLiteralType, MapType, StructureLiteralType
+
+  ---@param type vim._gen_lsp.Type
+  ---@param prefix? string Optional prefix associated with the this type, made of (nested) field name.
+  ---             Used to generate class name for structure literal types.
+  ---@return string
+  local function parse_type(type, prefix)
+    -- ReferenceType | BaseType
+    if type.kind == 'reference' or type.kind == 'base' then
+      if vim.tbl_contains(simple_types, type.name) then
+        return type.name
+      end
+      return 'lsp.' .. type.name
+
+    -- ArrayType
+    elseif type.kind == 'array' then
+      local parsed_items = parse_type(type.element, prefix)
+      if type.element.items and #type.element.items > 1 then
+        parsed_items = '(' .. parsed_items .. ')'
+      end
+      return parsed_items .. '[]'
+
+    -- OrType
+    elseif type.kind == 'or' then
+      local val = ''
+      for _, item in ipairs(type.items) do
+        val = val .. parse_type(item, prefix) .. '|' --[[ @as string ]]
+      end
+      val = val:sub(0, -2)
+      return val
+
+    -- StringLiteralType
+    elseif type.kind == 'stringLiteral' then
+      return '"' .. type.value .. '"'
+
+    -- MapType
+    elseif type.kind == 'map' then
+      local key = assert(type.key)
+      local value = type.value --[[ @as vim._gen_lsp.Type ]]
+      return 'table<' .. parse_type(key, prefix) .. ', ' .. parse_type(value, prefix) .. '>'
+
+    -- StructureLiteralType
+    elseif type.kind == 'literal' then
+      -- can I use ---@param disabled? {reason: string}
+      -- use | to continue the inline class to be able to add docs
+      -- https://github.com/LuaLS/lua-language-server/issues/2128
+      anonymous_num = anonymous_num + 1
+      local anonymous_classname = 'lsp._anonym' .. anonymous_num
+      if prefix then
+        anonymous_classname = anonymous_classname .. '.' .. prefix
+      end
+      local anonym = vim
+        .iter({
+          (anonymous_num > 1 and { '' } or {}),
+          { '---@class ' .. anonymous_classname },
+        })
+        :flatten()
+        :totable()
+
+      --- @class vim._gen_lsp.StructureLiteral translated to anonymous @class.
+      --- @field deprecated? string
+      --- @field description? string
+      --- @field properties vim._gen_lsp.Property[]
+      --- @field proposed? boolean
+      --- @field since? string
+
+      ---@type vim._gen_lsp.StructureLiteral
+      local structural_literal = assert(type.value) --[[ @as vim._gen_lsp.StructureLiteral ]]
+      for _, field in ipairs(structural_literal.properties) do
+        anonym[#anonym + 1] = '---'
+        if field.documentation then
+          anonym[#anonym + 1] = _process_documentation(field.documentation)
+        end
+        anonym[#anonym + 1] = '---@field '
+          .. field.name
+          .. (field.optional and '?' or '')
+          .. ' '
+          .. parse_type(field.type, prefix .. '.' .. field.name)
+      end
+      -- anonym[#anonym + 1] = ''
+      for _, line in ipairs(anonym) do
+        if line then
+          anonym_classes[#anonym_classes + 1] = line
+        end
+      end
+      return anonymous_classname
+
+    -- TupleType
+    elseif type.kind == 'tuple' then
+      local tuple = '['
+      for _, value in ipairs(type.items) do
+        tuple = tuple .. parse_type(value, prefix) .. ', '
+      end
+      -- remove , at the end
+      tuple = tuple:sub(0, -3)
+      return tuple .. ']'
+    end
+
+    vim.print('WARNING: Unknown type ', type)
+    return ''
+  end
+
+  --- @class vim._gen_lsp.Structure translated to @class
+  --- @field deprecated? string
+  --- @field documentation? string
+  --- @field extends? { kind: string, name: string }[]
+  --- @field mixins? { kind: string, name: string }[]
+  --- @field name string
+  --- @field properties? vim._gen_lsp.Property[]  members, translated to @field
+  --- @field proposed? boolean
+  --- @field since? string
+  for _, structure in ipairs(protocol.structures) do
+    -- output[#output + 1] = ''
+    if structure.documentation then
+      output[#output + 1] = _process_documentation(structure.documentation)
+    end
+    local class_string = ('---@class lsp.%s'):format(structure.name)
+    if structure.extends or structure.mixins then
+      local inherits_from = table.concat(
+        vim.list_extend(
+          vim.tbl_map(parse_type, structure.extends or {}),
+          vim.tbl_map(parse_type, structure.mixins or {})
+        ),
+        ', '
+      )
+      class_string = class_string .. ': ' .. inherits_from
+    end
+    output[#output + 1] = class_string
+
+    --- @class vim._gen_lsp.Property translated to @field
+    --- @field deprecated? string
+    --- @field documentation? string
+    --- @field name string
+    --- @field optional? boolean
+    --- @field proposed? boolean
+    --- @field since? string
+    --- @field type { kind: string, name: string }
+    for _, field in ipairs(structure.properties or {}) do
+      output[#output + 1] = '---' -- Insert a single newline between @fields (and after @class)
+      if field.documentation then
+        output[#output + 1] = _process_documentation(field.documentation)
+      end
+      output[#output + 1] = '---@field '
+        .. field.name
+        .. (field.optional and '?' or '')
+        .. ' '
+        .. parse_type(field.type, field.name)
+    end
+    output[#output + 1] = ''
+  end
+
+  --- @class vim._gen_lsp.Enumeration translated to @enum
+  --- @field deprecated string?
+  --- @field documentation string?
+  --- @field name string?
+  --- @field proposed boolean?
+  --- @field since string?
+  --- @field suportsCustomValues boolean?
+  --- @field values { name: string, value: string, documentation?: string, since?: string }[]
+  for _, enum in ipairs(protocol.enumerations) do
+    if enum.documentation then
+      output[#output + 1] = _process_documentation(enum.documentation)
+    end
+    local enum_type = '---@alias lsp.' .. enum.name
+    for _, value in ipairs(enum.values) do
+      enum_type = enum_type
+        .. '\n---| '
+        .. (type(value.value) == 'string' and '"' .. value.value .. '"' or value.value)
+        .. ' # '
+        .. value.name
+    end
+    output[#output + 1] = enum_type
+    output[#output + 1] = ''
+  end
+
+  --- @class vim._gen_lsp.TypeAlias translated to @alias
+  --- @field deprecated? string?
+  --- @field documentation? string
+  --- @field name string
+  --- @field proposed? boolean
+  --- @field since? string
+  --- @field type vim._gen_lsp.Type
+  for _, alias in ipairs(protocol.typeAliases) do
+    if alias.documentation then
+      output[#output + 1] = _process_documentation(alias.documentation)
+    end
+    if alias.type.kind == 'or' then
+      local alias_type = '---@alias lsp.' .. alias.name .. ' '
+      for _, item in ipairs(alias.type.items) do
+        alias_type = alias_type .. parse_type(item, alias.name) .. '|'
+      end
+      alias_type = alias_type:sub(0, -2)
+      output[#output + 1] = alias_type
+    else
+      output[#output + 1] = '---@alias lsp.'
+        .. alias.name
+        .. ' '
+        .. parse_type(alias.type, alias.name)
+    end
+    output[#output + 1] = ''
+  end
+
+  -- anonymous classes
+  for _, line in ipairs(anonym_classes) do
+    output[#output + 1] = line
+  end
+
+  tofile(opt.output_file, table.concat(output, '\n') .. '\n')
+end
+
+---@type vim._gen_lsp.opt
+local opt = {
+  output_file = 'runtime/lua/vim/lsp/_meta/protocol.lua',
+  version = DEFAULT_LSP_VERSION,
+  methods = false,
+  capabilities = false,
+}
+
+local command = nil
+local i = 1
+while i <= #_G.arg do
+  if _G.arg[i] == '--out' then
+    opt.output_file = assert(_G.arg[i + 1], '--out  needed')
+    i = i + 1
+  elseif _G.arg[i] == '--version' then
+    opt.version = assert(_G.arg[i + 1], '--version  needed')
+    i = i + 1
+  elseif _G.arg[i] == '--methods' then
+    opt.methods = true
+  elseif _G.arg[i] == '--capabilities' then
+    opt.capabilities = true
+  elseif vim.startswith(_G.arg[i], '-') then
+    error('Unrecognized args: ' .. _G.arg[i])
+  else
+    if command then
+      error('More than one command was given: ' .. _G.arg[i])
+    else
+      command = _G.arg[i]
+    end
+  end
+  i = i + 1
+end
+
+if not command then
+  print(USAGE)
+elseif M[command] then
+  M[command](opt) -- see M.gen()
+else
+  error('Unknown command: ' .. command)
+end
+
+return M
diff --git a/src/gen/gen_options.lua b/src/gen/gen_options.lua
new file mode 100644
index 0000000000..1947297a0e
--- /dev/null
+++ b/src/gen/gen_options.lua
@@ -0,0 +1,535 @@
+--- @module 'nvim.options'
+local options = require('nvim.options')
+local options_meta = options.options
+local cstr = options.cstr
+local valid_scopes = options.valid_scopes
+
+--- @param o vim.option_meta
+--- @return string
+local function get_values_var(o)
+  return ('opt_%s_values'):format(o.abbreviation or o.full_name)
+end
+
+--- @param s string
+--- @return string
+local function lowercase_to_titlecase(s)
+  return table.concat(vim.tbl_map(function(word) --- @param word string
+    return word:sub(1, 1):upper() .. word:sub(2)
+  end, vim.split(s, '[-_]')))
+end
+
+--- @param scope string
+--- @param option_name string
+--- @return string
+local function get_scope_option(scope, option_name)
+  return ('k%sOpt%s'):format(lowercase_to_titlecase(scope), lowercase_to_titlecase(option_name))
+end
+
+local redraw_flags = {
+  ui_option = 'kOptFlagUIOption',
+  tabline = 'kOptFlagRedrTabl',
+  statuslines = 'kOptFlagRedrStat',
+  current_window = 'kOptFlagRedrWin',
+  current_buffer = 'kOptFlagRedrBuf',
+  all_windows = 'kOptFlagRedrAll',
+  curswant = 'kOptFlagCurswant',
+  highlight_only = 'kOptFlagHLOnly',
+}
+
+local list_flags = {
+  comma = 'kOptFlagComma',
+  onecomma = 'kOptFlagOneComma',
+  commacolon = 'kOptFlagComma|kOptFlagColon',
+  onecommacolon = 'kOptFlagOneComma|kOptFlagColon',
+  flags = 'kOptFlagFlagList',
+  flagscomma = 'kOptFlagComma|kOptFlagFlagList',
+}
+
+--- @param o vim.option_meta
+--- @return string
+local function get_flags(o)
+  --- @type string[]
+  local flags = { '0' }
+
+  --- @param f string
+  local function add_flag(f)
+    table.insert(flags, f)
+  end
+
+  if o.list then
+    add_flag(list_flags[o.list])
+  end
+
+  for _, r_flag in ipairs(o.redraw or {}) do
+    add_flag(redraw_flags[r_flag])
+  end
+
+  if o.expand then
+    add_flag('kOptFlagExpand')
+    if o.expand == 'nodefault' then
+      add_flag('kOptFlagNoDefExp')
+    end
+  end
+
+  for _, flag_desc in ipairs({
+    { 'nodefault', 'NoDefault' },
+    { 'no_mkrc', 'NoMkrc' },
+    { 'secure' },
+    { 'gettext' },
+    { 'noglob', 'NoGlob' },
+    { 'normal_fname_chars', 'NFname' },
+    { 'normal_dname_chars', 'NDname' },
+    { 'pri_mkrc', 'PriMkrc' },
+    { 'deny_in_modelines', 'NoML' },
+    { 'deny_duplicates', 'NoDup' },
+    { 'modelineexpr', 'MLE' },
+    { 'func' },
+  }) do
+    local key_name, flag_suffix = flag_desc[1], flag_desc[2]
+    if o[key_name] then
+      local def_name = 'kOptFlag' .. (flag_suffix or lowercase_to_titlecase(key_name))
+      add_flag(def_name)
+    end
+  end
+
+  return table.concat(flags, '|')
+end
+
+--- @param opt_type vim.option_type
+--- @return string
+local function opt_type_enum(opt_type)
+  return ('kOptValType%s'):format(lowercase_to_titlecase(opt_type))
+end
+
+--- @param scope vim.option_scope
+--- @return string
+local function opt_scope_enum(scope)
+  return ('kOptScope%s'):format(lowercase_to_titlecase(scope))
+end
+
+--- @param o vim.option_meta
+--- @return string
+local function get_scope_flags(o)
+  local scope_flags = '0'
+
+  for _, scope in ipairs(o.scope) do
+    scope_flags = ('%s | (1 << %s)'):format(scope_flags, opt_scope_enum(scope))
+  end
+
+  return scope_flags
+end
+
+--- @param o vim.option_meta
+--- @return string
+local function get_scope_idx(o)
+  --- @type string[]
+  local strs = {}
+
+  for _, scope in pairs(valid_scopes) do
+    local has_scope = vim.tbl_contains(o.scope, scope)
+    strs[#strs + 1] = ('      [%s] = %s'):format(
+      opt_scope_enum(scope),
+      get_scope_option(scope, has_scope and o.full_name or 'Invalid')
+    )
+  end
+
+  return ('{\n%s\n    }'):format(table.concat(strs, ',\n'))
+end
+
+--- @param s string
+--- @return string
+local function static_cstr_as_string(s)
+  return ('{ .data = %s, .size = sizeof(%s) - 1 }'):format(s, s)
+end
+
+--- @param v vim.option_value|function
+--- @return string
+local function get_opt_val(v)
+  --- @type vim.option_type
+  local v_type
+
+  if type(v) == 'function' then
+    v, v_type = v() --[[ @as string, vim.option_type ]]
+
+    if v_type == 'string' then
+      v = static_cstr_as_string(v)
+    end
+  else
+    v_type = type(v) --[[ @as vim.option_type ]]
+
+    if v_type == 'boolean' then
+      v = v and 'true' or 'false'
+    elseif v_type == 'number' then
+      v = ('%iL'):format(v)
+    elseif v_type == 'string' then
+      --- @cast v string
+      v = static_cstr_as_string(cstr(v))
+    end
+  end
+
+  return ('{ .type = %s, .data.%s = %s }'):format(opt_type_enum(v_type), v_type, v)
+end
+
+--- @param d vim.option_value|function
+--- @param n string
+--- @return string
+local function get_defaults(d, n)
+  if d == nil then
+    error("option '" .. n .. "' should have a default value")
+  end
+  return get_opt_val(d)
+end
+
+--- @param i integer
+--- @param o vim.option_meta
+--- @param write fun(...: string)
+local function dump_option(i, o, write)
+  write('  [', ('%u'):format(i - 1) .. ']={')
+  write('    .fullname=', cstr(o.full_name))
+  if o.abbreviation then
+    write('    .shortname=', cstr(o.abbreviation))
+  end
+  write('    .type=', opt_type_enum(o.type))
+  write('    .flags=', get_flags(o))
+  write('    .scope_flags=', get_scope_flags(o))
+  write('    .scope_idx=', get_scope_idx(o))
+  write('    .values=', (o.values and get_values_var(o) or 'NULL'))
+  write('    .values_len=', (o.values and #o.values or '0'))
+  write('    .flags_var=', (o.flags_varname and ('&%s'):format(o.flags_varname) or 'NULL'))
+  if o.enable_if then
+    write(('#if defined(%s)'):format(o.enable_if))
+  end
+
+  local is_window_local = #o.scope == 1 and o.scope[1] == 'win'
+
+  if is_window_local then
+    write('    .var=NULL')
+  elseif o.varname then
+    write('    .var=&', o.varname)
+  elseif o.immutable then
+    -- Immutable options can directly point to the default value.
+    write(('    .var=&options[%u].def_val.data'):format(i - 1))
+  else
+    error('Option must be immutable or have a variable.')
+  end
+
+  write('    .immutable=', (o.immutable and 'true' or 'false'))
+  write('    .opt_did_set_cb=', o.cb or 'NULL')
+  write('    .opt_expand_cb=', o.expand_cb or 'NULL')
+
+  if o.enable_if then
+    write('#else')
+    -- Hidden option directly points to default value.
+    write(('    .var=&options[%u].def_val.data'):format(i - 1))
+    -- Option is always immutable on the false branch of `enable_if`.
+    write('    .immutable=true')
+    write('#endif')
+  end
+
+  if not o.defaults then
+    write('    .def_val=NIL_OPTVAL')
+  elseif o.defaults.condition then
+    write(('#if defined(%s)'):format(o.defaults.condition))
+    write('    .def_val=', get_defaults(o.defaults.if_true, o.full_name))
+    if o.defaults.if_false then
+      write('#else')
+      write('    .def_val=', get_defaults(o.defaults.if_false, o.full_name))
+    end
+    write('#endif')
+  else
+    write('    .def_val=', get_defaults(o.defaults.if_true, o.full_name))
+  end
+
+  write('  },')
+end
+
+--- @param prefix string
+--- @param values vim.option_valid_values
+local function preorder_traversal(prefix, values)
+  local out = {} --- @type string[]
+
+  local function add(s)
+    table.insert(out, s)
+  end
+
+  add('')
+  add(('EXTERN const char *(%s_values[%s]) INIT( = {'):format(prefix, #vim.tbl_keys(values) + 1))
+
+  --- @type [string,vim.option_valid_values][]
+  local children = {}
+
+  for _, value in ipairs(values) do
+    if type(value) == 'string' then
+      add(('  "%s",'):format(value))
+    else
+      assert(type(value) == 'table' and type(value[1]) == 'string' and type(value[2]) == 'table')
+      add(('  "%s",'):format(value[1]))
+      table.insert(children, value)
+    end
+  end
+
+  add('  NULL')
+  add('});')
+
+  for _, value in pairs(children) do
+    -- Remove trailing colon from the added prefix to prevent syntax errors.
+    add(preorder_traversal(prefix .. '_' .. value[1]:gsub(':$', ''), value[2]))
+  end
+
+  return table.concat(out, '\n')
+end
+
+--- @param o vim.option_meta
+--- @return string
+local function gen_opt_enum(o)
+  local out = {} --- @type string[]
+
+  local function add(s)
+    table.insert(out, s)
+  end
+
+  add('')
+  add('typedef enum {')
+
+  local opt_name = lowercase_to_titlecase(o.abbreviation or o.full_name)
+  --- @type table
+  local enum_values
+
+  if type(o.flags) == 'table' then
+    enum_values = o.flags --[[ @as table ]]
+  else
+    enum_values = {}
+    for i, flag_name in ipairs(o.values) do
+      assert(type(flag_name) == 'string')
+      enum_values[flag_name] = math.pow(2, i - 1)
+    end
+  end
+
+  -- Sort the keys by the flag value so that the enum can be generated in order.
+  --- @type string[]
+  local flag_names = vim.tbl_keys(enum_values)
+  table.sort(flag_names, function(a, b)
+    return enum_values[a] < enum_values[b]
+  end)
+
+  for _, flag_name in pairs(flag_names) do
+    add(
+      ('  kOpt%sFlag%s = 0x%02x,'):format(
+        opt_name,
+        lowercase_to_titlecase(flag_name:gsub(':$', '')),
+        enum_values[flag_name]
+      )
+    )
+  end
+
+  add(('} Opt%sFlags;'):format(opt_name))
+
+  return table.concat(out, '\n')
+end
+
+--- @param output_file string
+--- @return table options_index Map of option name to option index
+local function gen_enums(output_file)
+  --- Options for each scope.
+  --- @type table
+  local scope_options = {}
+  for _, scope in ipairs(valid_scopes) do
+    scope_options[scope] = {}
+  end
+
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  -- Generate options enum file
+  write('// IWYU pragma: private, include "nvim/option_defs.h"')
+  write('')
+
+  --- Map of option name to option index
+  --- @type table
+  local option_index = {}
+
+  -- Generate option index enum and populate the `option_index` and `scope_option` dicts.
+  write('typedef enum {')
+  write('  kOptInvalid = -1,')
+
+  for i, o in ipairs(options_meta) do
+    local enum_val_name = 'kOpt' .. lowercase_to_titlecase(o.full_name)
+    write(('  %s = %u,'):format(enum_val_name, i - 1))
+
+    option_index[o.full_name] = enum_val_name
+
+    if o.abbreviation then
+      option_index[o.abbreviation] = enum_val_name
+    end
+
+    local alias = o.alias or {} --[[@as string[] ]]
+    for _, v in ipairs(alias) do
+      option_index[v] = enum_val_name
+    end
+
+    for _, scope in ipairs(o.scope) do
+      table.insert(scope_options[scope], o)
+    end
+  end
+
+  write('  // Option count')
+  write('#define kOptCount ' .. tostring(#options_meta))
+  write('} OptIndex;')
+
+  -- Generate option index enum for each scope
+  for _, scope in ipairs(valid_scopes) do
+    write('')
+
+    local scope_name = lowercase_to_titlecase(scope)
+    write('typedef enum {')
+    write(('  %s = -1,'):format(get_scope_option(scope, 'Invalid')))
+
+    for idx, option in ipairs(scope_options[scope]) do
+      write(('  %s = %u,'):format(get_scope_option(scope, option.full_name), idx - 1))
+    end
+
+    write(('  // %s option count'):format(scope_name))
+    write(('#define %s %d'):format(get_scope_option(scope, 'Count'), #scope_options[scope]))
+    write(('} %sOptIndex;'):format(scope_name))
+  end
+
+  -- Generate reverse lookup from option scope index to option index for each scope.
+  for _, scope in ipairs(valid_scopes) do
+    write('')
+    write(('EXTERN const OptIndex %s_opt_idx[] INIT( = {'):format(scope))
+    for _, option in ipairs(scope_options[scope]) do
+      local idx = option_index[option.full_name]
+      write(('  [%s] = %s,'):format(get_scope_option(scope, option.full_name), idx))
+    end
+    write('});')
+  end
+
+  fd:close()
+
+  return option_index
+end
+
+--- @param output_file string
+--- @param option_index table
+local function gen_map(output_file, option_index)
+  -- Generate option index map.
+  local hashy = require('gen.hashy')
+
+  local neworder, hashfun = hashy.hashy_hash(
+    'find_option',
+    vim.tbl_keys(option_index),
+    function(idx)
+      return ('option_hash_elems[%s].name'):format(idx)
+    end
+  )
+
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  write('static const struct { const char *name; OptIndex opt_idx; } option_hash_elems[] = {')
+
+  for _, name in ipairs(neworder) do
+    assert(option_index[name] ~= nil)
+    write(('  { .name = "%s", .opt_idx = %s },'):format(name, option_index[name]))
+  end
+
+  write('};')
+  write('')
+  write('static ' .. hashfun)
+
+  fd:close()
+end
+
+--- @param output_file string
+local function gen_vars(output_file)
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param s string
+  local function write(s)
+    fd:write(s)
+    fd:write('\n')
+  end
+
+  write('// IWYU pragma: private, include "nvim/option_vars.h"')
+
+  -- Generate enums for option flags.
+  for _, o in ipairs(options_meta) do
+    if o.flags and (type(o.flags) == 'table' or o.values) then
+      write(gen_opt_enum(o))
+    end
+  end
+
+  -- Generate valid values for each option.
+  for _, option in ipairs(options_meta) do
+    -- Since option values can be nested, we need to do preorder traversal to generate the values.
+    if option.values then
+      local values_var = ('opt_%s'):format(option.abbreviation or option.full_name)
+      write(preorder_traversal(values_var, option.values))
+    end
+  end
+
+  fd:close()
+end
+
+--- @param output_file string
+local function gen_options(output_file)
+  local fd = assert(io.open(output_file, 'w'))
+
+  --- @param ... string
+  local function write(...)
+    local s = table.concat({ ... }, '')
+    fd:write(s)
+    if s:match('^    %.') then
+      fd:write(',')
+    end
+    fd:write('\n')
+  end
+
+  -- Generate options[] array.
+  write([[
+  #include "nvim/ex_docmd.h"
+  #include "nvim/ex_getln.h"
+  #include "nvim/insexpand.h"
+  #include "nvim/mapping.h"
+  #include "nvim/ops.h"
+  #include "nvim/option.h"
+  #include "nvim/optionstr.h"
+  #include "nvim/quickfix.h"
+  #include "nvim/runtime.h"
+  #include "nvim/tag.h"
+  #include "nvim/window.h"
+
+  static vimoption_T options[] = {]])
+
+  for i, o in ipairs(options_meta) do
+    dump_option(i, o, write)
+  end
+
+  write('};')
+
+  fd:close()
+end
+
+local function main()
+  local options_file = arg[1]
+  local options_enum_file = arg[2]
+  local options_map_file = arg[3]
+  local option_vars_file = arg[4]
+
+  local option_index = gen_enums(options_enum_file)
+  gen_map(options_map_file, option_index)
+  gen_vars(option_vars_file)
+  gen_options(options_file)
+end
+
+main()
diff --git a/src/gen/gen_vimdoc.lua b/src/gen/gen_vimdoc.lua
new file mode 100755
index 0000000000..2fe7224ea5
--- /dev/null
+++ b/src/gen/gen_vimdoc.lua
@@ -0,0 +1,1041 @@
+#!/usr/bin/env -S nvim -l
+--- Generates Nvim :help docs from Lua/C docstrings
+---
+--- The generated :help text for each function is formatted as follows:
+--- - Max width of 78 columns (`TEXT_WIDTH`).
+--- - Indent with spaces (not tabs).
+--- - Indent of 4 columns for body text (`INDENTATION`).
+--- - Function signature and helptag (right-aligned) on the same line.
+---   - Signature and helptag must have a minimum of 8 spaces between them.
+---   - If the signature is too long, it is placed on the line after the helptag.
+---     Signature wraps with subsequent lines indented to the open parenthesis.
+---   - Subsection bodies are indented an additional 4 spaces.
+--- - Body consists of function description, parameters, return description, and
+---   C declaration (`INCLUDE_C_DECL`).
+--- - Parameters are omitted for the `void` and `Error *` types, or if the
+---   parameter is marked as [out].
+--- - Each function documentation is separated by a single line.
+
+local luacats_parser = require('gen.luacats_parser')
+local cdoc_parser = require('gen.cdoc_parser')
+local util = require('gen.util')
+
+local fmt = string.format
+
+local wrap = util.wrap
+local md_to_vimdoc = util.md_to_vimdoc
+
+local TEXT_WIDTH = 78
+local INDENTATION = 4
+
+--- @class (exact) nvim.gen_vimdoc.Config
+---
+--- Generated documentation target, e.g. api.txt
+--- @field filename string
+---
+--- @field section_order string[]
+---
+--- List of files/directories for doxygen to read, relative to `base_dir`.
+--- @field files string[]
+---
+--- @field exclude_types? true
+---
+--- Section name overrides. Key: filename (e.g., vim.c)
+--- @field section_name? table
+---
+--- @field fn_name_pat? string
+---
+--- @field fn_xform? fun(fun: nvim.luacats.parser.fun)
+---
+--- For generated section names.
+--- @field section_fmt fun(name: string): string
+---
+--- @field helptag_fmt fun(name: string): string|string[]
+---
+--- Per-function helptag.
+--- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string
+---
+--- @field append_only? string[]
+
+local function contains(t, xs)
+  return vim.tbl_contains(xs, t)
+end
+
+--- @type {level:integer, prerelease:boolean}?
+local nvim_api_info_
+
+--- @return {level: integer, prerelease:boolean}
+local function nvim_api_info()
+  if not nvim_api_info_ then
+    --- @type integer?, boolean?
+    local level, prerelease
+    for l in io.lines('CMakeLists.txt') do
+      --- @cast l string
+      if level and prerelease then
+        break
+      end
+      local m1 = l:match('^set%(NVIM_API_LEVEL%s+(%d+)%)')
+      if m1 then
+        level = tonumber(m1) --[[@as integer]]
+      end
+      local m2 = l:match('^set%(NVIM_API_PRERELEASE%s+(%w+)%)')
+      if m2 then
+        prerelease = m2 == 'true'
+      end
+    end
+    nvim_api_info_ = { level = level, prerelease = prerelease }
+  end
+
+  return nvim_api_info_
+end
+
+--- @param fun nvim.luacats.parser.fun
+--- @return string
+local function fn_helptag_fmt_common(fun)
+  local fn_sfx = fun.table and '' or '()'
+  if fun.classvar then
+    return fmt('%s:%s%s', fun.classvar, fun.name, fn_sfx)
+  end
+  if fun.module then
+    return fmt('%s.%s%s', fun.module, fun.name, fn_sfx)
+  end
+  return fun.name .. fn_sfx
+end
+
+--- @type table
+local config = {
+  api = {
+    filename = 'api.txt',
+    section_order = {
+      'vim.c',
+      'vimscript.c',
+      'command.c',
+      'options.c',
+      'buffer.c',
+      'extmark.c',
+      'window.c',
+      'win_config.c',
+      'tabpage.c',
+      'autocmd.c',
+      'ui.c',
+    },
+    exclude_types = true,
+    fn_name_pat = 'nvim_.*',
+    files = { 'src/nvim/api' },
+    section_name = {
+      ['vim.c'] = 'Global',
+    },
+    section_fmt = function(name)
+      return name .. ' Functions'
+    end,
+    helptag_fmt = function(name)
+      return fmt('api-%s', name:lower())
+    end,
+  },
+  lua = {
+    filename = 'lua.txt',
+    section_order = {
+      'hl.lua',
+      'diff.lua',
+      'mpack.lua',
+      'json.lua',
+      'base64.lua',
+      'spell.lua',
+      'builtin.lua',
+      '_options.lua',
+      '_editor.lua',
+      '_inspector.lua',
+      'shared.lua',
+      'loader.lua',
+      'uri.lua',
+      'ui.lua',
+      'filetype.lua',
+      'keymap.lua',
+      'fs.lua',
+      'glob.lua',
+      'lpeg.lua',
+      're.lua',
+      'regex.lua',
+      'secure.lua',
+      'version.lua',
+      'iter.lua',
+      'snippet.lua',
+      'text.lua',
+      'tohtml.lua',
+    },
+    files = {
+      'runtime/lua/vim/iter.lua',
+      'runtime/lua/vim/_editor.lua',
+      'runtime/lua/vim/_options.lua',
+      'runtime/lua/vim/shared.lua',
+      'runtime/lua/vim/loader.lua',
+      'runtime/lua/vim/uri.lua',
+      'runtime/lua/vim/ui.lua',
+      'runtime/lua/vim/filetype.lua',
+      'runtime/lua/vim/keymap.lua',
+      'runtime/lua/vim/fs.lua',
+      'runtime/lua/vim/hl.lua',
+      'runtime/lua/vim/secure.lua',
+      'runtime/lua/vim/version.lua',
+      'runtime/lua/vim/_inspector.lua',
+      'runtime/lua/vim/snippet.lua',
+      'runtime/lua/vim/text.lua',
+      'runtime/lua/vim/glob.lua',
+      'runtime/lua/vim/_meta/builtin.lua',
+      'runtime/lua/vim/_meta/diff.lua',
+      'runtime/lua/vim/_meta/mpack.lua',
+      'runtime/lua/vim/_meta/json.lua',
+      'runtime/lua/vim/_meta/base64.lua',
+      'runtime/lua/vim/_meta/regex.lua',
+      'runtime/lua/vim/_meta/lpeg.lua',
+      'runtime/lua/vim/_meta/re.lua',
+      'runtime/lua/vim/_meta/spell.lua',
+      'runtime/lua/tohtml.lua',
+    },
+    fn_xform = function(fun)
+      if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then
+        fun.module = 'vim'
+      end
+
+      if fun.module == 'vim' and contains(fun.name, { 'cmd', 'inspect' }) then
+        fun.table = nil
+      end
+
+      if fun.classvar or vim.startswith(fun.name, 'vim.') or fun.module == 'vim.iter' then
+        return
+      end
+
+      fun.name = fmt('%s.%s', fun.module, fun.name)
+    end,
+    section_name = {
+      ['_inspector.lua'] = 'inspector',
+    },
+    section_fmt = function(name)
+      name = name:lower()
+      if name == '_editor' then
+        return 'Lua module: vim'
+      elseif name == '_options' then
+        return 'LUA-VIMSCRIPT BRIDGE'
+      elseif name == 'builtin' then
+        return 'VIM'
+      end
+      if
+        contains(name, {
+          'hl',
+          'mpack',
+          'json',
+          'base64',
+          'diff',
+          'spell',
+          'regex',
+          'lpeg',
+          're',
+        })
+      then
+        return 'VIM.' .. name:upper()
+      end
+      if name == 'tohtml' then
+        return 'Lua module: tohtml'
+      end
+      return 'Lua module: vim.' .. name
+    end,
+    helptag_fmt = function(name)
+      if name == '_editor' then
+        return 'lua-vim'
+      elseif name == '_options' then
+        return 'lua-vimscript'
+      elseif name == 'tohtml' then
+        return 'tohtml'
+      end
+      return 'vim.' .. name:lower()
+    end,
+    fn_helptag_fmt = function(fun)
+      local name = fun.name
+
+      if vim.startswith(name, 'vim.') then
+        local fn_sfx = fun.table and '' or '()'
+        return name .. fn_sfx
+      elseif fun.classvar == 'Option' then
+        return fmt('vim.opt:%s()', name)
+      end
+
+      return fn_helptag_fmt_common(fun)
+    end,
+    append_only = {
+      'shared.lua',
+    },
+  },
+  lsp = {
+    filename = 'lsp.txt',
+    section_order = {
+      'lsp.lua',
+      'client.lua',
+      'buf.lua',
+      'diagnostic.lua',
+      'codelens.lua',
+      'completion.lua',
+      'folding_range.lua',
+      'inlay_hint.lua',
+      'tagfunc.lua',
+      'semantic_tokens.lua',
+      'handlers.lua',
+      'util.lua',
+      'log.lua',
+      'rpc.lua',
+      'protocol.lua',
+    },
+    files = {
+      'runtime/lua/vim/lsp',
+      'runtime/lua/vim/lsp.lua',
+    },
+    fn_xform = function(fun)
+      fun.name = fun.name:gsub('result%.', '')
+      if fun.module == 'vim.lsp.protocol' then
+        fun.classvar = nil
+      end
+    end,
+    section_fmt = function(name)
+      if name:lower() == 'lsp' then
+        return 'Lua module: vim.lsp'
+      end
+      return 'Lua module: vim.lsp.' .. name:lower()
+    end,
+    helptag_fmt = function(name)
+      if name:lower() == 'lsp' then
+        return 'lsp-core'
+      end
+      return fmt('lsp-%s', name:lower())
+    end,
+  },
+  diagnostic = {
+    filename = 'diagnostic.txt',
+    section_order = {
+      'diagnostic.lua',
+    },
+    files = { 'runtime/lua/vim/diagnostic.lua' },
+    section_fmt = function()
+      return 'Lua module: vim.diagnostic'
+    end,
+    helptag_fmt = function()
+      return 'diagnostic-api'
+    end,
+  },
+  treesitter = {
+    filename = 'treesitter.txt',
+    section_order = {
+      'tstree.lua',
+      'tsnode.lua',
+      'treesitter.lua',
+      'language.lua',
+      'query.lua',
+      'highlighter.lua',
+      'languagetree.lua',
+      'dev.lua',
+    },
+    files = {
+      'runtime/lua/vim/treesitter/_meta/',
+      'runtime/lua/vim/treesitter.lua',
+      'runtime/lua/vim/treesitter/',
+    },
+    section_fmt = function(name)
+      if name:lower() == 'treesitter' then
+        return 'Lua module: vim.treesitter'
+      elseif name:lower() == 'tstree' then
+        return 'TREESITTER TREES'
+      elseif name:lower() == 'tsnode' then
+        return 'TREESITTER NODES'
+      end
+      return 'Lua module: vim.treesitter.' .. name:lower()
+    end,
+    helptag_fmt = function(name)
+      if name:lower() == 'treesitter' then
+        return 'lua-treesitter-core'
+      elseif name:lower() == 'query' then
+        return 'lua-treesitter-query'
+      elseif name:lower() == 'tstree' then
+        return { 'treesitter-tree', 'TSTree' }
+      elseif name:lower() == 'tsnode' then
+        return { 'treesitter-node', 'TSNode' }
+      end
+      return 'treesitter-' .. name:lower()
+    end,
+  },
+  editorconfig = {
+    filename = 'editorconfig.txt',
+    files = {
+      'runtime/lua/editorconfig.lua',
+    },
+    section_order = {
+      'editorconfig.lua',
+    },
+    section_fmt = function(_name)
+      return 'EditorConfig integration'
+    end,
+    helptag_fmt = function(name)
+      return name:lower()
+    end,
+    fn_xform = function(fun)
+      fun.table = true
+      fun.name = vim.split(fun.name, '.', { plain = true })[2]
+    end,
+  },
+  health = {
+    filename = 'health.txt',
+    files = {
+      'runtime/lua/vim/health.lua',
+    },
+    section_order = {
+      'health.lua',
+    },
+    section_fmt = function(_name)
+      return 'Checkhealth'
+    end,
+    helptag_fmt = function()
+      return { 'vim.health', 'health' }
+    end,
+  },
+}
+
+--- @param ty string
+--- @param generics table
+--- @return string
+local function replace_generics(ty, generics)
+  if ty:sub(-2) == '[]' then
+    local ty0 = ty:sub(1, -3)
+    if generics[ty0] then
+      return generics[ty0] .. '[]'
+    end
+  elseif ty:sub(-1) == '?' then
+    local ty0 = ty:sub(1, -2)
+    if generics[ty0] then
+      return generics[ty0] .. '?'
+    end
+  end
+
+  return generics[ty] or ty
+end
+
+--- @param name string
+local function fmt_field_name(name)
+  local name0, opt = name:match('^([^?]*)(%??)$')
+  return fmt('{%s}%s', name0, opt)
+end
+
+--- @param ty string
+--- @param generics? table
+--- @param default? string
+local function render_type(ty, generics, default)
+  if generics then
+    ty = replace_generics(ty, generics)
+  end
+  ty = ty:gsub('%s*|%s*nil', '?')
+  ty = ty:gsub('nil%s*|%s*(.*)', '%1?')
+  ty = ty:gsub('%s*|%s*', '|')
+  if default then
+    return fmt('(`%s`, default: %s)', ty, default)
+  end
+  return fmt('(`%s`)', ty)
+end
+
+--- @param p nvim.luacats.parser.param|nvim.luacats.parser.field
+local function should_render_field_or_param(p)
+  return not p.nodoc
+    and not p.access
+    and not contains(p.name, { '_', 'self' })
+    and not vim.startswith(p.name, '_')
+end
+
+--- @param desc? string
+--- @return string?, string?
+local function get_default(desc)
+  if not desc then
+    return
+  end
+
+  local default = desc:match('\n%s*%([dD]efault: ([^)]+)%)')
+  if default then
+    desc = desc:gsub('\n%s*%([dD]efault: [^)]+%)', '')
+  end
+
+  return desc, default
+end
+
+--- @param ty string
+--- @param classes? table
+--- @return nvim.luacats.parser.class?
+local function get_class(ty, classes)
+  if not classes then
+    return
+  end
+
+  local cty = ty:gsub('%s*|%s*nil', '?'):gsub('?$', ''):gsub('%[%]$', '')
+
+  return classes[cty]
+end
+
+--- @param obj nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.field
+--- @param classes? table
+local function inline_type(obj, classes)
+  local ty = obj.type
+  if not ty then
+    return
+  end
+
+  local cls = get_class(ty, classes)
+
+  if not cls or cls.nodoc then
+    return
+  end
+
+  if not cls.inlinedoc then
+    -- Not inlining so just add a: "See |tag|."
+    local tag = fmt('|%s|', cls.name)
+    if obj.desc and obj.desc:find(tag) then
+      -- Tag already there
+      return
+    end
+
+    -- TODO(lewis6991): Aim to remove this. Need this to prevent dead
+    -- references to types defined in runtime/lua/vim/lsp/_meta/protocol.lua
+    if not vim.startswith(cls.name, 'vim.') then
+      return
+    end
+
+    obj.desc = obj.desc or ''
+    local period = (obj.desc == '' or vim.endswith(obj.desc, '.')) and '' or '.'
+    obj.desc = obj.desc .. fmt('%s See %s.', period, tag)
+    return
+  end
+
+  local ty_isopt = (ty:match('%?$') or ty:match('%s*|%s*nil')) ~= nil
+  local ty_islist = (ty:match('%[%]$')) ~= nil
+  ty = ty_isopt and 'table?' or ty_islist and 'table[]' or 'table'
+
+  local desc = obj.desc or ''
+  if cls.desc then
+    desc = desc .. cls.desc
+  elseif desc == '' then
+    if ty_islist then
+      desc = desc .. 'A list of objects with the following fields:'
+    elseif cls.parent then
+      desc = desc .. fmt('Extends |%s| with the additional fields:', cls.parent)
+    else
+      desc = desc .. 'A table with the following fields:'
+    end
+  end
+
+  local desc_append = {}
+  for _, f in ipairs(cls.fields) do
+    if not f.access then
+      local fdesc, default = get_default(f.desc)
+      local fty = render_type(f.type, nil, default)
+      local fnm = fmt_field_name(f.name)
+      table.insert(desc_append, table.concat({ '-', fnm, fty, fdesc }, ' '))
+    end
+  end
+
+  desc = desc .. '\n' .. table.concat(desc_append, '\n')
+  obj.type = ty
+  obj.desc = desc
+end
+
+--- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[]
+--- @param generics? table
+--- @param classes? table
+--- @param exclude_types? true
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_fields_or_params(xs, generics, classes, exclude_types, cfg)
+  local ret = {} --- @type string[]
+
+  xs = vim.tbl_filter(should_render_field_or_param, xs)
+
+  local indent = 0
+  for _, p in ipairs(xs) do
+    if p.type or p.desc then
+      indent = math.max(indent, #p.name + 3)
+    end
+    if exclude_types then
+      p.type = nil
+    end
+  end
+
+  for _, p in ipairs(xs) do
+    local pdesc, default = get_default(p.desc)
+    p.desc = pdesc
+
+    inline_type(p, classes)
+    local nm, ty = p.name, p.type
+
+    local desc = p.classvar and string.format('See |%s|.', cfg.fn_helptag_fmt(p)) or p.desc
+
+    local fnm = p.kind == 'operator' and fmt('op(%s)', nm) or fmt_field_name(nm)
+    local pnm = fmt('      • %-' .. indent .. 's', fnm)
+
+    if ty then
+      local pty = render_type(ty, generics, default)
+
+      if desc then
+        table.insert(ret, pnm)
+        if #pty > TEXT_WIDTH - indent then
+          vim.list_extend(ret, { ' ', pty, '\n' })
+          table.insert(ret, md_to_vimdoc(desc, 9 + indent, 9 + indent, TEXT_WIDTH, true))
+        else
+          desc = fmt('%s %s', pty, desc)
+          table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true))
+        end
+      else
+        table.insert(ret, fmt('%s %s\n', pnm, pty))
+      end
+    else
+      if desc then
+        table.insert(ret, pnm)
+        table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true))
+      end
+    end
+  end
+
+  return table.concat(ret)
+end
+
+--- @param class nvim.luacats.parser.class
+--- @param classes table
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_class(class, classes, cfg)
+  if class.access or class.nodoc or class.inlinedoc then
+    return
+  end
+
+  local ret = {} --- @type string[]
+
+  table.insert(ret, fmt('*%s*\n', class.name))
+
+  if class.parent then
+    local txt = fmt('Extends: |%s|', class.parent)
+    table.insert(ret, md_to_vimdoc(txt, INDENTATION, INDENTATION, TEXT_WIDTH))
+    table.insert(ret, '\n')
+  end
+
+  if class.desc then
+    table.insert(ret, md_to_vimdoc(class.desc, INDENTATION, INDENTATION, TEXT_WIDTH))
+  end
+
+  local fields_txt = render_fields_or_params(class.fields, nil, classes, nil, cfg)
+  if not fields_txt:match('^%s*$') then
+    table.insert(ret, '\n    Fields: ~\n')
+    table.insert(ret, fields_txt)
+  end
+  table.insert(ret, '\n')
+
+  return table.concat(ret)
+end
+
+--- @param classes table
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_classes(classes, cfg)
+  local ret = {} --- @type string[]
+
+  for _, class in vim.spairs(classes) do
+    ret[#ret + 1] = render_class(class, classes, cfg)
+  end
+
+  return table.concat(ret)
+end
+
+--- @param fun nvim.luacats.parser.fun
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_fun_header(fun, cfg)
+  local ret = {} --- @type string[]
+
+  local args = {} --- @type string[]
+  for _, p in ipairs(fun.params or {}) do
+    if p.name ~= 'self' then
+      args[#args + 1] = fmt_field_name(p.name)
+    end
+  end
+
+  local nm = fun.name
+  if fun.classvar then
+    nm = fmt('%s:%s', fun.classvar, nm)
+  end
+  if nm == 'vim.bo' then
+    nm = 'vim.bo[{bufnr}]'
+  end
+  if nm == 'vim.wo' then
+    nm = 'vim.wo[{winid}][{bufnr}]'
+  end
+
+  local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')'
+
+  local tag = '*' .. cfg.fn_helptag_fmt(fun) .. '*'
+
+  if #proto + #tag > TEXT_WIDTH - 8 then
+    table.insert(ret, fmt('%78s\n', tag))
+    local name, pargs = proto:match('([^(]+%()(.*)')
+    table.insert(ret, name)
+    table.insert(ret, wrap(pargs, 0, #name, TEXT_WIDTH))
+  else
+    local pad = TEXT_WIDTH - #proto - #tag
+    table.insert(ret, proto .. string.rep(' ', pad) .. tag)
+  end
+
+  return table.concat(ret)
+end
+
+--- @param returns nvim.luacats.parser.return[]
+--- @param generics? table
+--- @param classes? table
+--- @param exclude_types boolean
+local function render_returns(returns, generics, classes, exclude_types)
+  local ret = {} --- @type string[]
+
+  returns = vim.deepcopy(returns)
+  if exclude_types then
+    for _, r in ipairs(returns) do
+      r.type = nil
+    end
+  end
+
+  if #returns > 1 then
+    table.insert(ret, '    Return (multiple): ~\n')
+  elseif #returns == 1 and next(returns[1]) then
+    table.insert(ret, '    Return: ~\n')
+  end
+
+  for _, p in ipairs(returns) do
+    inline_type(p, classes)
+    local rnm, ty, desc = p.name, p.type, p.desc
+
+    local blk = {} --- @type string[]
+    if ty then
+      blk[#blk + 1] = render_type(ty, generics)
+    end
+    blk[#blk + 1] = rnm
+    blk[#blk + 1] = desc
+
+    table.insert(ret, md_to_vimdoc(table.concat(blk, ' '), 8, 8, TEXT_WIDTH, true))
+  end
+
+  return table.concat(ret)
+end
+
+--- @param fun nvim.luacats.parser.fun
+--- @param classes table
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_fun(fun, classes, cfg)
+  if fun.access or fun.deprecated or fun.nodoc then
+    return
+  end
+
+  if cfg.fn_name_pat and not fun.name:match(cfg.fn_name_pat) then
+    return
+  end
+
+  if vim.startswith(fun.name, '_') or fun.name:find('[:.]_') then
+    return
+  end
+
+  local ret = {} --- @type string[]
+
+  table.insert(ret, render_fun_header(fun, cfg))
+  table.insert(ret, '\n')
+
+  if fun.since then
+    local since = assert(tonumber(fun.since), 'invalid @since on ' .. fun.name)
+    local info = nvim_api_info()
+    if since == 0 or (info.prerelease and since == info.level) then
+      -- Experimental = (since==0 or current prerelease)
+      local s = 'WARNING: This feature is experimental/unstable.'
+      table.insert(ret, md_to_vimdoc(s, INDENTATION, INDENTATION, TEXT_WIDTH))
+      table.insert(ret, '\n')
+    else
+      local v = assert(util.version_level[since], 'invalid @since on ' .. fun.name)
+      fun.attrs = fun.attrs or {}
+      table.insert(fun.attrs, ('Since: %s'):format(v))
+    end
+  end
+
+  if fun.desc then
+    table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH))
+  end
+
+  if fun.notes then
+    table.insert(ret, '\n    Note: ~\n')
+    for _, p in ipairs(fun.notes) do
+      table.insert(ret, '      • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true))
+    end
+  end
+
+  if fun.attrs then
+    table.insert(ret, '\n    Attributes: ~\n')
+    for _, attr in ipairs(fun.attrs) do
+      local attr_str = ({
+        textlock = 'not allowed when |textlock| is active or in the |cmdwin|',
+        textlock_allow_cmdwin = 'not allowed when |textlock| is active',
+        fast = '|api-fast|',
+        remote_only = '|RPC| only',
+        lua_only = 'Lua |vim.api| only',
+      })[attr] or attr
+      table.insert(ret, fmt('        %s\n', attr_str))
+    end
+  end
+
+  if fun.params and #fun.params > 0 then
+    local param_txt =
+      render_fields_or_params(fun.params, fun.generics, classes, cfg.exclude_types, cfg)
+    if not param_txt:match('^%s*$') then
+      table.insert(ret, '\n    Parameters: ~\n')
+      ret[#ret + 1] = param_txt
+    end
+  end
+
+  if fun.returns then
+    local txt = render_returns(fun.returns, fun.generics, classes, cfg.exclude_types)
+    if not txt:match('^%s*$') then
+      table.insert(ret, '\n')
+      ret[#ret + 1] = txt
+    end
+  end
+
+  if fun.see then
+    table.insert(ret, '\n    See also: ~\n')
+    for _, p in ipairs(fun.see) do
+      table.insert(ret, '      • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true))
+    end
+  end
+
+  table.insert(ret, '\n')
+  return table.concat(ret)
+end
+
+--- @param funs nvim.luacats.parser.fun[]
+--- @param classes table
+--- @param cfg nvim.gen_vimdoc.Config
+local function render_funs(funs, classes, cfg)
+  local ret = {} --- @type string[]
+
+  for _, f in ipairs(funs) do
+    if cfg.fn_xform then
+      cfg.fn_xform(f)
+    end
+    ret[#ret + 1] = render_fun(f, classes, cfg)
+  end
+
+  -- Sort via prototype. Experimental API functions ("nvim__") sort last.
+  table.sort(ret, function(a, b)
+    local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n')
+    local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n')
+
+    local a1__ = a1:find('^%s*nvim__') and 1 or 0
+    local b1__ = b1:find('^%s*nvim__') and 1 or 0
+    if a1__ ~= b1__ then
+      return a1__ < b1__
+    end
+
+    return a1:lower() < b1:lower()
+  end)
+
+  return table.concat(ret)
+end
+
+--- @return string
+local function get_script_path()
+  local str = debug.getinfo(2, 'S').source:sub(2)
+  return str:match('(.*[/\\])') or './'
+end
+
+local script_path = get_script_path()
+local base_dir = vim.fs.dirname(vim.fs.dirname(vim.fs.dirname(script_path)))
+
+local function delete_lines_below(doc_file, tokenstr)
+  local lines = {} --- @type string[]
+  local found = false
+  for line in io.lines(doc_file) do
+    if line:find(vim.pesc(tokenstr)) then
+      found = true
+      break
+    end
+    lines[#lines + 1] = line
+  end
+  if not found then
+    error(fmt('not found: %s in %s', tokenstr, doc_file))
+  end
+  lines[#lines] = nil
+  local fp = assert(io.open(doc_file, 'w'))
+  fp:write(table.concat(lines, '\n'))
+  fp:write('\n')
+  fp:close()
+end
+
+--- @param x string
+local function mktitle(x)
+  if x == 'ui' then
+    return 'UI'
+  end
+  return x:sub(1, 1):upper() .. x:sub(2)
+end
+
+--- @class nvim.gen_vimdoc.Section
+--- @field name string
+--- @field title string
+--- @field help_tag string
+--- @field funs_txt string
+--- @field doc? string[]
+
+--- @param filename string
+--- @param cfg nvim.gen_vimdoc.Config
+--- @param section_docs table
+--- @param funs_txt string
+--- @return nvim.gen_vimdoc.Section?
+local function make_section(filename, cfg, section_docs, funs_txt)
+  -- filename: e.g., 'autocmd.c'
+  -- name: e.g. 'autocmd'
+  local name = filename:match('(.*)%.[a-z]+')
+
+  -- Formatted (this is what's going to be written in the vimdoc)
+  -- e.g., "Autocmd Functions"
+  local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name)
+
+  -- section tag: e.g., "*api-autocmd*"
+  local help_labels = cfg.helptag_fmt(sectname)
+  if type(help_labels) == 'table' then
+    help_labels = table.concat(help_labels, '* *')
+  end
+  local help_tags = '*' .. help_labels .. '*'
+
+  if funs_txt == '' and #section_docs == 0 then
+    return
+  end
+
+  return {
+    name = sectname,
+    title = cfg.section_fmt(sectname),
+    help_tag = help_tags,
+    funs_txt = funs_txt,
+    doc = section_docs,
+  }
+end
+
+--- @param section nvim.gen_vimdoc.Section
+--- @param add_header? boolean
+local function render_section(section, add_header)
+  local doc = {} --- @type string[]
+
+  if add_header ~= false then
+    vim.list_extend(doc, {
+      string.rep('=', TEXT_WIDTH),
+      '\n',
+      section.title,
+      fmt('%' .. (TEXT_WIDTH - section.title:len()) .. 's', section.help_tag),
+    })
+  end
+
+  local sdoc = '\n\n' .. table.concat(section.doc or {}, '\n')
+  if sdoc:find('[^%s]') then
+    doc[#doc + 1] = sdoc
+  end
+
+  if section.funs_txt then
+    table.insert(doc, '\n\n')
+    table.insert(doc, section.funs_txt)
+  end
+
+  return table.concat(doc)
+end
+
+local parsers = {
+  lua = luacats_parser.parse,
+  c = cdoc_parser.parse,
+  h = cdoc_parser.parse,
+}
+
+--- @param files string[]
+local function expand_files(files)
+  for k, f in pairs(files) do
+    if vim.fn.isdirectory(f) == 1 then
+      table.remove(files, k)
+      for path, ty in vim.fs.dir(f) do
+        if ty == 'file' then
+          table.insert(files, vim.fs.joinpath(f, path))
+        end
+      end
+    end
+  end
+end
+
+--- @param cfg nvim.gen_vimdoc.Config
+local function gen_target(cfg)
+  cfg.fn_helptag_fmt = cfg.fn_helptag_fmt or fn_helptag_fmt_common
+  print('Target:', cfg.filename)
+  local sections = {} --- @type table
+
+  expand_files(cfg.files)
+
+  --- @type table, nvim.luacats.parser.fun[], string[]]>
+  local file_results = {}
+
+  --- @type table
+  local all_classes = {}
+
+  --- First pass so we can collect all classes
+  for _, f in vim.spairs(cfg.files) do
+    local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']]
+    local parser = assert(parsers[ext])
+    local classes, funs, briefs = parser(f)
+    file_results[f] = { classes, funs, briefs }
+    all_classes = vim.tbl_extend('error', all_classes, classes)
+  end
+
+  for f, r in vim.spairs(file_results) do
+    local classes, funs, briefs = r[1], r[2], r[3]
+
+    local briefs_txt = {} --- @type string[]
+    for _, b in ipairs(briefs) do
+      briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH)
+    end
+    print('    Processing file:', f)
+    local funs_txt = render_funs(funs, all_classes, cfg)
+    if next(classes) then
+      local classes_txt = render_classes(classes, cfg)
+      if vim.trim(classes_txt) ~= '' then
+        funs_txt = classes_txt .. '\n' .. funs_txt
+      end
+    end
+    -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua`
+    local f_base = vim.fs.basename(f)
+    sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt)
+  end
+
+  local first_section_tag = sections[cfg.section_order[1]].help_tag
+  local docs = {} --- @type string[]
+  for _, f in ipairs(cfg.section_order) do
+    local section = sections[f]
+    if section then
+      print(string.format("    Rendering section: '%s'", section.title))
+      local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f)
+      docs[#docs + 1] = render_section(section, add_sep_and_header)
+    end
+  end
+
+  table.insert(
+    docs,
+    fmt(' vim:tw=78:ts=8:sw=%d:sts=%d:et:ft=help:norl:\n', INDENTATION, INDENTATION)
+  )
+
+  local doc_file = vim.fs.joinpath(base_dir, 'runtime', 'doc', cfg.filename)
+
+  if vim.uv.fs_stat(doc_file) then
+    delete_lines_below(doc_file, first_section_tag)
+  end
+
+  local fp = assert(io.open(doc_file, 'a'))
+  fp:write(table.concat(docs, '\n'))
+  fp:close()
+end
+
+local function run()
+  for _, cfg in vim.spairs(config) do
+    gen_target(cfg)
+  end
+end
+
+run()
diff --git a/src/gen/gen_vimvim.lua b/src/gen/gen_vimvim.lua
new file mode 100644
index 0000000000..d2b1f48a4c
--- /dev/null
+++ b/src/gen/gen_vimvim.lua
@@ -0,0 +1,156 @@
+local mpack = vim.mpack
+
+local syntax_file = arg[1]
+local funcs_file = arg[2]
+
+local lld = {}
+local syn_fd = io.open(syntax_file, 'w')
+lld.line_length = 0
+local function w(s)
+  syn_fd:write(s)
+  if s:find('\n') then
+    lld.line_length = #(s:gsub('.*\n', ''))
+  else
+    lld.line_length = lld.line_length + #s
+  end
+end
+
+local options = require('nvim.options')
+local auevents = require('nvim.auevents')
+local ex_cmds = require('nvim.ex_cmds')
+
+local function cmd_kw(prev_cmd, cmd)
+  if not prev_cmd then
+    return cmd:sub(1, 1) .. '[' .. cmd:sub(2) .. ']'
+  else
+    local shift = 1
+    while cmd:sub(shift, shift) == prev_cmd:sub(shift, shift) do
+      shift = shift + 1
+    end
+    if cmd:sub(1, shift) == 'def' then
+      shift = shift + 1
+    end
+    if shift >= #cmd then
+      return cmd
+    else
+      return cmd:sub(1, shift) .. '[' .. cmd:sub(shift + 1) .. ']'
+    end
+  end
+end
+
+-- Exclude these from the vimCommand keyword list, they are handled specially
+-- in syntax/vim.vim (vimAugroupKey, vimAutoCmd, vimGlobal, vimSubst). #9327
+local function is_special_cased_cmd(cmd)
+  return (
+    cmd == 'augroup'
+    or cmd == 'autocmd'
+    or cmd == 'doautocmd'
+    or cmd == 'doautoall'
+    or cmd == 'global'
+    or cmd == 'substitute'
+  )
+end
+
+local vimcmd_start = 'syn keyword vimCommand contained '
+local vimcmd_end = ' nextgroup=vimBang'
+w(vimcmd_start)
+
+local prev_cmd = nil
+for _, cmd_desc in ipairs(ex_cmds.cmds) do
+  if lld.line_length > 850 then
+    w(vimcmd_end .. '\n' .. vimcmd_start)
+  end
+  local cmd = cmd_desc.command
+  if cmd:match('%w') and cmd ~= 'z' and not is_special_cased_cmd(cmd) then
+    w(' ' .. cmd_kw(prev_cmd, cmd))
+  end
+  if cmd == 'delete' then
+    -- Add special abbreviations of :delete
+    w(' ' .. cmd_kw('d', 'dl'))
+    w(' ' .. cmd_kw('del', 'dell'))
+    w(' ' .. cmd_kw('dele', 'delel'))
+    w(' ' .. cmd_kw('delet', 'deletl'))
+    w(' ' .. cmd_kw('delete', 'deletel'))
+    w(' ' .. cmd_kw('d', 'dp'))
+    w(' ' .. cmd_kw('de', 'dep'))
+    w(' ' .. cmd_kw('del', 'delp'))
+    w(' ' .. cmd_kw('dele', 'delep'))
+    w(' ' .. cmd_kw('delet', 'deletp'))
+    w(' ' .. cmd_kw('delete', 'deletep'))
+  end
+  prev_cmd = cmd
+end
+
+w(vimcmd_end .. '\n')
+
+local vimopt_start = 'syn keyword vimOption contained '
+local vimopt_end = ' skipwhite nextgroup=vimSetEqual,vimSetMod'
+w('\n' .. vimopt_start)
+
+for _, opt_desc in ipairs(options.options) do
+  if not opt_desc.immutable then
+    if lld.line_length > 850 then
+      w(vimopt_end .. '\n' .. vimopt_start)
+    end
+    w(' ' .. opt_desc.full_name)
+    if opt_desc.abbreviation then
+      w(' ' .. opt_desc.abbreviation)
+    end
+    if opt_desc.type == 'boolean' then
+      w(' inv' .. opt_desc.full_name)
+      w(' no' .. opt_desc.full_name)
+      if opt_desc.abbreviation then
+        w(' inv' .. opt_desc.abbreviation)
+        w(' no' .. opt_desc.abbreviation)
+      end
+    end
+  end
+end
+
+w(vimopt_end .. '\n')
+
+w('\nsyn case ignore')
+local vimau_start = 'syn keyword vimAutoEvent contained '
+w('\n\n' .. vimau_start)
+
+for _, au in ipairs(auevents.events) do
+  if not auevents.nvim_specific[au[1]] then
+    if lld.line_length > 850 then
+      w('\n' .. vimau_start)
+    end
+    w(' ' .. au[1])
+    for _, alias in ipairs(au[2]) do
+      if lld.line_length > 850 then
+        w('\n' .. vimau_start)
+      end
+      -- au[1] is aliased to alias
+      w(' ' .. alias)
+    end
+  end
+end
+
+local nvimau_start = 'syn keyword nvimAutoEvent contained '
+w('\n\n' .. nvimau_start)
+
+for au, _ in vim.spairs(auevents.nvim_specific) do
+  if lld.line_length > 850 then
+    w('\n' .. nvimau_start)
+  end
+  w(' ' .. au)
+end
+
+w('\n\nsyn case match')
+local vimfun_start = 'syn keyword vimFuncName contained '
+w('\n\n' .. vimfun_start)
+local funcs = mpack.decode(io.open(funcs_file, 'rb'):read('*all'))
+for _, name in ipairs(funcs) do
+  if name then
+    if lld.line_length > 850 then
+      w('\n' .. vimfun_start)
+    end
+    w(' ' .. name)
+  end
+end
+
+w('\n')
+syn_fd:close()
diff --git a/src/gen/hashy.lua b/src/gen/hashy.lua
new file mode 100644
index 0000000000..74b7655324
--- /dev/null
+++ b/src/gen/hashy.lua
@@ -0,0 +1,145 @@
+-- HASHY McHASHFACE
+
+local M = {}
+_G.d = M
+
+local function setdefault(table, key)
+  local val = table[key]
+  if val == nil then
+    val = {}
+    table[key] = val
+  end
+  return val
+end
+
+function M.build_pos_hash(strings)
+  local len_buckets = {}
+  local maxlen = 0
+  for _, s in ipairs(strings) do
+    table.insert(setdefault(len_buckets, #s), s)
+    if #s > maxlen then
+      maxlen = #s
+    end
+  end
+
+  local len_pos_buckets = {}
+  local worst_buck_size = 0
+
+  for len = 1, maxlen do
+    local strs = len_buckets[len]
+    if strs then
+      -- the best position so far generates `best_bucket`
+      -- with `minsize` worst case collisions
+      local bestpos, minsize, best_bucket = nil, #strs * 2, nil
+      for pos = 1, len do
+        local try_bucket = {}
+        for _, str in ipairs(strs) do
+          local poschar = string.sub(str, pos, pos)
+          table.insert(setdefault(try_bucket, poschar), str)
+        end
+        local maxsize = 1
+        for _, pos_strs in pairs(try_bucket) do
+          maxsize = math.max(maxsize, #pos_strs)
+        end
+        if maxsize < minsize then
+          bestpos = pos
+          minsize = maxsize
+          best_bucket = try_bucket
+        end
+      end
+      len_pos_buckets[len] = { bestpos, best_bucket }
+      worst_buck_size = math.max(worst_buck_size, minsize)
+    end
+  end
+  return len_pos_buckets, maxlen, worst_buck_size
+end
+
+function M.switcher(put, tab, maxlen, worst_buck_size)
+  local neworder = {} --- @type string[]
+  put '  switch (len) {\n'
+  local bucky = worst_buck_size > 1
+  for len = 1, maxlen do
+    local vals = tab[len]
+    if vals then
+      put('    case ' .. len .. ': ')
+      local pos, posbuck = unpack(vals)
+      local keys = vim.tbl_keys(posbuck)
+      if #keys > 1 then
+        table.sort(keys)
+        put('switch (str[' .. (pos - 1) .. ']) {\n')
+        for _, c in ipairs(keys) do
+          local buck = posbuck[c]
+          local startidx = #neworder
+          vim.list_extend(neworder, buck)
+          local endidx = #neworder
+          put("      case '" .. c .. "': ")
+          if len == 1 then
+            put('return ' .. startidx .. ';\n')
+          else
+            put('low = ' .. startidx .. '; ')
+            if bucky then
+              put('high = ' .. endidx .. '; ')
+            end
+            put 'break;\n'
+          end
+        end
+        put '      default: break;\n'
+        put '    }\n    '
+      else
+        local startidx = #neworder
+        table.insert(neworder, posbuck[keys[1]][1])
+        local endidx = #neworder
+        put('low = ' .. startidx .. '; ')
+        if bucky then
+          put('high = ' .. endidx .. '; ')
+        end
+      end
+      put 'break;\n'
+    end
+  end
+  put '    default: break;\n'
+  put '  }\n'
+  return neworder
+end
+
+function M.hashy_hash(name, strings, access)
+  local stats = {}
+  local put = function(str)
+    table.insert(stats, str)
+  end
+  local len_pos_buckets, maxlen, worst_buck_size = M.build_pos_hash(strings)
+  put('int ' .. name .. '_hash(const char *str, size_t len)\n{\n')
+  if maxlen == 1 then
+    put('\n') -- nothing
+  elseif worst_buck_size > 1 then
+    put('  int low = 0, high = 0;\n')
+  else
+    put('  int low = -1;\n')
+  end
+  local neworder = M.switcher(put, len_pos_buckets, maxlen, worst_buck_size)
+  if maxlen == 1 then
+    put([[
+  return -1;
+]])
+  elseif worst_buck_size > 1 then
+    put([[
+  for (int i = low; i < high; i++) {
+    if (!memcmp(str, ]] .. access('i') .. [[, len)) {
+      return i;
+    }
+  }
+  return -1;
+]])
+  else
+    put([[
+  if (low < 0 || memcmp(str, ]] .. access('low') .. [[, len)) {
+    return -1;
+  }
+  return low;
+]])
+  end
+  put '}\n\n'
+  return neworder, table.concat(stats)
+end
+
+return M
diff --git a/src/gen/luacats_grammar.lua b/src/gen/luacats_grammar.lua
new file mode 100644
index 0000000000..b700bcf58f
--- /dev/null
+++ b/src/gen/luacats_grammar.lua
@@ -0,0 +1,207 @@
+--[[!
+LPEG grammar for LuaCATS
+]]
+
+local lpeg = vim.lpeg
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local C, Ct, Cg = lpeg.C, lpeg.Ct, lpeg.Cg
+
+--- @param x vim.lpeg.Pattern
+local function rep(x)
+  return x ^ 0
+end
+
+--- @param x vim.lpeg.Pattern
+local function rep1(x)
+  return x ^ 1
+end
+
+--- @param x vim.lpeg.Pattern
+local function opt(x)
+  return x ^ -1
+end
+
+local ws = rep1(S(' \t'))
+local fill = opt(ws)
+local any = P(1) -- (consume one character)
+local letter = R('az', 'AZ')
+local num = R('09')
+
+--- @param x string | vim.lpeg.Pattern
+local function Pf(x)
+  return fill * P(x) * fill
+end
+
+--- @param x string | vim.lpeg.Pattern
+local function Plf(x)
+  return fill * P(x)
+end
+
+--- @param x string
+local function Sf(x)
+  return fill * S(x) * fill
+end
+
+--- @param x vim.lpeg.Pattern
+local function paren(x)
+  return Pf('(') * x * fill * P(')')
+end
+
+--- @param x vim.lpeg.Pattern
+local function parenOpt(x)
+  return paren(x) + x
+end
+
+--- @param x vim.lpeg.Pattern
+local function comma1(x)
+  return parenOpt(x * rep(Pf(',') * x))
+end
+
+--- @param x vim.lpeg.Pattern
+local function comma(x)
+  return opt(comma1(x))
+end
+
+--- @type table
+local v = setmetatable({}, {
+  __index = function(_, k)
+    return lpeg.V(k)
+  end,
+})
+
+--- @class nvim.luacats.Param
+--- @field kind 'param'
+--- @field name string
+--- @field type string
+--- @field desc? string
+
+--- @class nvim.luacats.Return
+--- @field kind 'return'
+--- @field [integer] { type: string, name?: string}
+--- @field desc? string
+
+--- @class nvim.luacats.Generic
+--- @field kind 'generic'
+--- @field name string
+--- @field type? string
+
+--- @class nvim.luacats.Class
+--- @field kind 'class'
+--- @field name string
+--- @field parent? string
+--- @field access? 'private'|'protected'|'package'
+
+--- @class nvim.luacats.Field
+--- @field kind 'field'
+--- @field name string
+--- @field type string
+--- @field desc? string
+--- @field access? 'private'|'protected'|'package'
+
+--- @class nvim.luacats.Note
+--- @field desc? string
+
+--- @alias nvim.luacats.grammar.result
+--- | nvim.luacats.Param
+--- | nvim.luacats.Return
+--- | nvim.luacats.Generic
+--- | nvim.luacats.Class
+--- | nvim.luacats.Field
+--- | nvim.luacats.Note
+
+--- @class nvim.luacats.grammar
+--- @field match fun(self, input: string): nvim.luacats.grammar.result?
+
+local function annot(nm, pat)
+  if type(nm) == 'string' then
+    nm = P(nm)
+  end
+  if pat then
+    return Ct(Cg(P(nm), 'kind') * fill * pat)
+  end
+  return Ct(Cg(P(nm), 'kind'))
+end
+
+local colon = Pf(':')
+local ellipsis = P('...')
+local ident_first = P('_') + letter
+local ident = ident_first * rep(ident_first + num)
+local opt_ident = ident * opt(P('?'))
+local ty_ident_sep = S('-._')
+local ty_ident = ident * rep(ty_ident_sep * ident)
+local string_single = P "'" * rep(any - P "'") * P "'"
+local string_double = P('"') * rep(any - P('"')) * P('"')
+local generic = P('`') * ty_ident * P('`')
+local literal = string_single + string_double + (opt(P('-')) * rep1(num)) + P('false') + P('true')
+local ty_prims = ty_ident + literal + generic
+
+local array_postfix = rep1(Plf('[]'))
+local opt_postfix = rep1(Plf('?'))
+local rep_array_opt_postfix = rep(array_postfix + opt_postfix)
+
+local typedef = P({
+  'typedef',
+  typedef = C(v.type),
+
+  type = v.ty * rep_array_opt_postfix * rep(Pf('|') * v.ty * rep_array_opt_postfix),
+  ty = v.composite + paren(v.typedef),
+  composite = (v.types * array_postfix) + (v.types * opt_postfix) + v.types,
+  types = v.generics + v.kv_table + v.tuple + v.dict + v.table_literal + v.fun + ty_prims,
+
+  tuple = Pf('[') * comma1(v.type) * Plf(']'),
+  dict = Pf('{') * comma1(Pf('[') * v.type * Pf(']') * colon * v.type) * Plf('}'),
+  kv_table = Pf('table') * Pf('<') * v.type * Pf(',') * v.type * Plf('>'),
+  table_literal = Pf('{') * comma1(opt_ident * Pf(':') * v.type) * Plf('}'),
+  fun_param = (opt_ident + ellipsis) * opt(colon * v.type),
+  fun_ret = v.type + (ellipsis * opt(colon * v.type)),
+  fun = Pf('fun') * paren(comma(v.fun_param)) * opt(Pf(':') * comma1(v.fun_ret)),
+  generics = P(ty_ident) * Pf('<') * comma1(v.type) * Plf('>'),
+}) / function(match)
+  return vim.trim(match):gsub('^%((.*)%)$', '%1'):gsub('%?+', '?')
+end
+
+local access = P('private') + P('protected') + P('package')
+local caccess = Cg(access, 'access')
+local cattr = Cg(comma(access + P('exact')), 'access')
+local desc_delim = Sf '#:' + ws
+local desc = Cg(rep(any), 'desc')
+local opt_desc = opt(desc_delim * desc)
+local ty_name = Cg(ty_ident, 'name')
+local opt_parent = opt(colon * Cg(ty_ident, 'parent'))
+local lname = (ident + ellipsis) * opt(P('?'))
+
+local grammar = P {
+  rep1(P('@') * (v.ats + v.ext_ats)),
+
+  ats = annot('param', Cg(lname, 'name') * ws * v.ctype * opt_desc)
+    + annot('return', comma1(Ct(v.ctype * opt(ws * (ty_name + Cg(ellipsis, 'name'))))) * opt_desc)
+    + annot('type', comma1(Ct(v.ctype)) * opt_desc)
+    + annot('cast', ty_name * ws * opt(Sf('+-')) * v.ctype)
+    + annot('generic', ty_name * opt(colon * v.ctype))
+    + annot('class', opt(paren(cattr)) * fill * ty_name * opt_parent)
+    + annot('field', opt(caccess * ws) * v.field_name * ws * v.ctype * opt_desc)
+    + annot('operator', ty_name * opt(paren(Cg(v.ctype, 'argtype'))) * colon * v.ctype)
+    + annot(access)
+    + annot('deprecated')
+    + annot('alias', ty_name * opt(ws * v.ctype))
+    + annot('enum', ty_name)
+    + annot('overload', v.ctype)
+    + annot('see', opt(desc_delim) * desc)
+    + annot('diagnostic', opt(desc_delim) * desc)
+    + annot('meta'),
+
+  --- Custom extensions
+  ext_ats = (
+    annot('note', desc)
+    + annot('since', desc)
+    + annot('nodoc')
+    + annot('inlinedoc')
+    + annot('brief', desc)
+  ),
+
+  field_name = Cg(lname + (v.ty_index * opt(P('?'))), 'name'),
+  ty_index = C(Pf('[') * typedef * fill * P(']')),
+  ctype = Cg(typedef, 'type'),
+}
+
+return grammar --[[@as nvim.luacats.grammar]]
diff --git a/src/gen/luacats_parser.lua b/src/gen/luacats_parser.lua
new file mode 100644
index 0000000000..36bdc44076
--- /dev/null
+++ b/src/gen/luacats_parser.lua
@@ -0,0 +1,535 @@
+local luacats_grammar = require('gen.luacats_grammar')
+
+--- @class nvim.luacats.parser.param : nvim.luacats.Param
+
+--- @class nvim.luacats.parser.return
+--- @field name string
+--- @field type string
+--- @field desc string
+
+--- @class nvim.luacats.parser.note
+--- @field desc string
+
+--- @class nvim.luacats.parser.brief
+--- @field kind 'brief'
+--- @field desc string
+
+--- @class nvim.luacats.parser.alias
+--- @field kind 'alias'
+--- @field type string[]
+--- @field desc string
+
+--- @class nvim.luacats.parser.fun
+--- @field name string
+--- @field params nvim.luacats.parser.param[]
+--- @field returns nvim.luacats.parser.return[]
+--- @field desc string
+--- @field access? 'private'|'package'|'protected'
+--- @field class? string
+--- @field module? string
+--- @field modvar? string
+--- @field classvar? string
+--- @field deprecated? true
+--- @field since? string
+--- @field attrs? string[]
+--- @field nodoc? true
+--- @field generics? table
+--- @field table? true
+--- @field notes? nvim.luacats.parser.note[]
+--- @field see? nvim.luacats.parser.note[]
+
+--- @class nvim.luacats.parser.field : nvim.luacats.Field
+--- @field classvar? string
+--- @field nodoc? true
+
+--- @class nvim.luacats.parser.class : nvim.luacats.Class
+--- @field desc? string
+--- @field nodoc? true
+--- @field inlinedoc? true
+--- @field fields nvim.luacats.parser.field[]
+--- @field notes? string[]
+
+--- @class nvim.luacats.parser.State
+--- @field doc_lines? string[]
+--- @field cur_obj? nvim.luacats.parser.obj
+--- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note
+--- @field last_doc_item_indent? integer
+
+--- @alias nvim.luacats.parser.obj
+--- | nvim.luacats.parser.class
+--- | nvim.luacats.parser.fun
+--- | nvim.luacats.parser.brief
+--- | nvim.luacats.parser.alias
+
+-- Remove this when we document classes properly
+--- Some doc lines have the form:
+---   param name some.complex.type (table) description
+--- if so then transform the line to remove the complex type:
+---   param name (table) description
+--- @param line string
+local function use_type_alt(line)
+  for _, type in ipairs({ 'table', 'function' }) do
+    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2')
+    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2')
+    line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2')
+
+    line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1')
+    line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1')
+    line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1')
+  end
+  return line
+end
+
+--- If we collected any `---` lines. Add them to the existing (or new) object
+--- Used for function/class descriptions and multiline param descriptions.
+--- @param state nvim.luacats.parser.State
+local function add_doc_lines_to_obj(state)
+  if state.doc_lines then
+    state.cur_obj = state.cur_obj or {}
+    local cur_obj = assert(state.cur_obj)
+    local txt = table.concat(state.doc_lines, '\n')
+    if cur_obj.desc then
+      cur_obj.desc = cur_obj.desc .. '\n' .. txt
+    else
+      cur_obj.desc = txt
+    end
+    state.doc_lines = nil
+  end
+end
+
+--- @param line string
+--- @param state nvim.luacats.parser.State
+local function process_doc_line(line, state)
+  line = line:sub(4):gsub('^%s+@', '@')
+  line = use_type_alt(line)
+
+  local parsed = luacats_grammar:match(line)
+
+  if not parsed then
+    if line:match('^ ') then
+      line = line:sub(2)
+    end
+
+    if state.last_doc_item then
+      if not state.last_doc_item_indent then
+        state.last_doc_item_indent = #line:match('^%s*') + 1
+      end
+      state.last_doc_item.desc = (state.last_doc_item.desc or '')
+        .. '\n'
+        .. line:sub(state.last_doc_item_indent or 1)
+    else
+      state.doc_lines = state.doc_lines or {}
+      table.insert(state.doc_lines, line)
+    end
+    return
+  end
+
+  state.last_doc_item_indent = nil
+  state.last_doc_item = nil
+  state.cur_obj = state.cur_obj or {}
+  local cur_obj = assert(state.cur_obj)
+
+  local kind = parsed.kind
+
+  if kind == 'brief' then
+    state.cur_obj = {
+      kind = 'brief',
+      desc = parsed.desc,
+    }
+  elseif kind == 'class' then
+    --- @cast parsed nvim.luacats.Class
+    cur_obj.kind = 'class'
+    cur_obj.name = parsed.name
+    cur_obj.parent = parsed.parent
+    cur_obj.access = parsed.access
+    cur_obj.desc = state.doc_lines and table.concat(state.doc_lines, '\n') or nil
+    state.doc_lines = nil
+    cur_obj.fields = {}
+  elseif kind == 'field' then
+    --- @cast parsed nvim.luacats.Field
+    parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil
+    if parsed.desc then
+      parsed.desc = vim.trim(parsed.desc)
+    end
+    table.insert(cur_obj.fields, parsed)
+    state.doc_lines = nil
+  elseif kind == 'operator' then
+    parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil
+    if parsed.desc then
+      parsed.desc = vim.trim(parsed.desc)
+    end
+    table.insert(cur_obj.fields, parsed)
+    state.doc_lines = nil
+  elseif kind == 'param' then
+    state.last_doc_item_indent = nil
+    cur_obj.params = cur_obj.params or {}
+    if vim.endswith(parsed.name, '?') then
+      parsed.name = parsed.name:sub(1, -2)
+      parsed.type = parsed.type .. '?'
+    end
+    state.last_doc_item = {
+      name = parsed.name,
+      type = parsed.type,
+      desc = parsed.desc,
+    }
+    table.insert(cur_obj.params, state.last_doc_item)
+  elseif kind == 'return' then
+    cur_obj.returns = cur_obj.returns or {}
+    for _, t in ipairs(parsed) do
+      table.insert(cur_obj.returns, {
+        name = t.name,
+        type = t.type,
+        desc = parsed.desc,
+      })
+    end
+    state.last_doc_item_indent = nil
+    state.last_doc_item = cur_obj.returns[#cur_obj.returns]
+  elseif kind == 'private' then
+    cur_obj.access = 'private'
+  elseif kind == 'package' then
+    cur_obj.access = 'package'
+  elseif kind == 'protected' then
+    cur_obj.access = 'protected'
+  elseif kind == 'deprecated' then
+    cur_obj.deprecated = true
+  elseif kind == 'inlinedoc' then
+    cur_obj.inlinedoc = true
+  elseif kind == 'nodoc' then
+    cur_obj.nodoc = true
+  elseif kind == 'since' then
+    cur_obj.since = parsed.desc
+  elseif kind == 'see' then
+    cur_obj.see = cur_obj.see or {}
+    table.insert(cur_obj.see, { desc = parsed.desc })
+  elseif kind == 'note' then
+    state.last_doc_item_indent = nil
+    state.last_doc_item = {
+      desc = parsed.desc,
+    }
+    cur_obj.notes = cur_obj.notes or {}
+    table.insert(cur_obj.notes, state.last_doc_item)
+  elseif kind == 'type' then
+    cur_obj.desc = parsed.desc
+    parsed.desc = nil
+    parsed.kind = nil
+    cur_obj.type = parsed
+  elseif kind == 'alias' then
+    state.cur_obj = {
+      kind = 'alias',
+      desc = parsed.desc,
+    }
+  elseif kind == 'enum' then
+    -- TODO
+    state.doc_lines = nil
+  elseif
+    vim.tbl_contains({
+      'diagnostic',
+      'cast',
+      'overload',
+      'meta',
+    }, kind)
+  then
+    -- Ignore
+    return
+  elseif kind == 'generic' then
+    cur_obj.generics = cur_obj.generics or {}
+    cur_obj.generics[parsed.name] = parsed.type or 'any'
+  else
+    error('Unhandled' .. vim.inspect(parsed))
+  end
+end
+
+--- @param fun nvim.luacats.parser.fun
+--- @return nvim.luacats.parser.field
+local function fun2field(fun)
+  local parts = { 'fun(' }
+
+  local params = {} ---@type string[]
+  for _, p in ipairs(fun.params or {}) do
+    params[#params + 1] = string.format('%s: %s', p.name, p.type)
+  end
+  parts[#parts + 1] = table.concat(params, ', ')
+  parts[#parts + 1] = ')'
+  if fun.returns then
+    parts[#parts + 1] = ': '
+    local tys = {} --- @type string[]
+    for _, p in ipairs(fun.returns) do
+      tys[#tys + 1] = p.type
+    end
+    parts[#parts + 1] = table.concat(tys, ', ')
+  end
+
+  return {
+    name = fun.name,
+    type = table.concat(parts, ''),
+    access = fun.access,
+    desc = fun.desc,
+    nodoc = fun.nodoc,
+  }
+end
+
+--- Function to normalize known form for declaring functions and normalize into a more standard
+--- form.
+--- @param line string
+--- @return string
+local function filter_decl(line)
+  -- M.fun = vim._memoize(function(...)
+  --   ->
+  -- function M.fun(...)
+  line = line:gsub('^local (.+) = memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)')
+  line = line:gsub('^(.+) = memoize%([^,]+, function%((.*)%)$', 'function %1(%2)')
+  return line
+end
+
+--- @param line string
+--- @param state nvim.luacats.parser.State
+--- @param classes table
+--- @param classvars table
+--- @param has_indent boolean
+local function process_lua_line(line, state, classes, classvars, has_indent)
+  line = filter_decl(line)
+
+  if state.cur_obj and state.cur_obj.kind == 'class' then
+    local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=')
+    if nm then
+      classvars[nm] = state.cur_obj.name
+    end
+    return
+  end
+
+  do
+    local parent_tbl, sep, fun_or_meth_nm =
+      line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(')
+    if parent_tbl then
+      -- Have a decl. Ensure cur_obj
+      state.cur_obj = state.cur_obj or {}
+      local cur_obj = assert(state.cur_obj)
+
+      -- Match `Class:foo` methods for defined classes
+      local class = classvars[parent_tbl]
+      if class then
+        --- @cast cur_obj nvim.luacats.parser.fun
+        cur_obj.name = fun_or_meth_nm
+        cur_obj.class = class
+        cur_obj.classvar = parent_tbl
+        -- Add self param to methods
+        if sep == ':' then
+          cur_obj.params = cur_obj.params or {}
+          table.insert(cur_obj.params, 1, {
+            name = 'self',
+            type = class,
+          })
+        end
+
+        -- Add method as the field to the class
+        local cls = classes[class]
+        local field = fun2field(cur_obj)
+        field.classvar = cur_obj.classvar
+        table.insert(cls.fields, field)
+        return
+      end
+
+      -- Match `M.foo`
+      if cur_obj and parent_tbl == cur_obj.modvar then
+        cur_obj.name = fun_or_meth_nm
+        return
+      end
+    end
+  end
+
+  do
+    -- Handle: `function A.B.C.foo(...)`
+    local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(')
+    if fn_nm then
+      state.cur_obj = state.cur_obj or {}
+      state.cur_obj.name = fn_nm
+      return
+    end
+  end
+
+  do
+    -- Handle: `M.foo = {...}` where `M` is the modvar
+    local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=')
+    if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then
+      state.cur_obj.name = tbl_nm
+      state.cur_obj.table = true
+      return
+    end
+  end
+
+  do
+    -- Handle: `foo = {...}`
+    local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=')
+    if tbl_nm and not has_indent then
+      state.cur_obj = state.cur_obj or {}
+      state.cur_obj.name = tbl_nm
+      state.cur_obj.table = true
+      return
+    end
+  end
+
+  do
+    -- Handle: `vim.foo = {...}`
+    local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=')
+    if state.cur_obj and tbl_nm and not has_indent then
+      state.cur_obj.name = tbl_nm
+      state.cur_obj.table = true
+      return
+    end
+  end
+
+  if state.cur_obj then
+    if line:find('^%s*%-%- luacheck:') then
+      state.cur_obj = nil
+    elseif line:find('^%s*local%s+') then
+      state.cur_obj = nil
+    elseif line:find('^%s*return%s+') then
+      state.cur_obj = nil
+    elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then
+      state.cur_obj = nil
+    end
+  end
+end
+
+--- Determine the table name used to export functions of a module
+--- Usually this is `M`.
+--- @param str string
+--- @return string?
+local function determine_modvar(str)
+  local modvar --- @type string?
+  for line in vim.gsplit(str, '\n') do
+    do
+      --- @type string?
+      local m = line:match('^return%s+([a-zA-Z_]+)')
+      if m then
+        modvar = m
+      end
+    end
+    do
+      --- @type string?
+      local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),')
+      if m then
+        modvar = m
+      end
+    end
+  end
+  return modvar
+end
+
+--- @param obj nvim.luacats.parser.obj
+--- @param funs nvim.luacats.parser.fun[]
+--- @param classes table
+--- @param briefs string[]
+--- @param uncommitted nvim.luacats.parser.obj[]
+local function commit_obj(obj, classes, funs, briefs, uncommitted)
+  local commit = false
+  if obj.kind == 'class' then
+    --- @cast obj nvim.luacats.parser.class
+    if not classes[obj.name] then
+      classes[obj.name] = obj
+      commit = true
+    end
+  elseif obj.kind == 'alias' then
+    -- Just pretend
+    commit = true
+  elseif obj.kind == 'brief' then
+    --- @cast obj nvim.luacats.parser.brief`
+    briefs[#briefs + 1] = obj.desc
+    commit = true
+  else
+    --- @cast obj nvim.luacats.parser.fun`
+    if obj.name then
+      funs[#funs + 1] = obj
+      commit = true
+    end
+  end
+  if not commit then
+    table.insert(uncommitted, obj)
+  end
+  return commit
+end
+
+--- @param filename string
+--- @param uncommitted nvim.luacats.parser.obj[]
+-- luacheck: no unused
+local function dump_uncommitted(filename, uncommitted)
+  local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt'
+  if #uncommitted > 0 then
+    print(string.format('Could not commit %d objects in %s', #uncommitted, filename))
+    vim.fn.mkdir(vim.fs.dirname(out_path), 'p')
+    local f = assert(io.open(out_path, 'w'))
+    for i, x in ipairs(uncommitted) do
+      f:write(i)
+      f:write(': ')
+      f:write(vim.inspect(x))
+      f:write('\n')
+    end
+    f:close()
+  else
+    vim.fn.delete(out_path)
+  end
+end
+
+local M = {}
+
+function M.parse_str(str, filename)
+  local funs = {} --- @type nvim.luacats.parser.fun[]
+  local classes = {} --- @type table
+  local briefs = {} --- @type string[]
+
+  local mod_return = determine_modvar(str)
+
+  --- @type string
+  local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename
+  module = module:gsub('/', '.')
+
+  local classvars = {} --- @type table
+
+  local state = {} --- @type nvim.luacats.parser.State
+
+  -- Keep track of any partial objects we don't commit
+  local uncommitted = {} --- @type nvim.luacats.parser.obj[]
+
+  for line in vim.gsplit(str, '\n') do
+    local has_indent = line:match('^%s+') ~= nil
+    line = vim.trim(line)
+    if vim.startswith(line, '---') then
+      process_doc_line(line, state)
+    else
+      add_doc_lines_to_obj(state)
+
+      if state.cur_obj then
+        state.cur_obj.modvar = mod_return
+        state.cur_obj.module = module
+      end
+
+      process_lua_line(line, state, classes, classvars, has_indent)
+
+      -- Commit the object
+      local cur_obj = state.cur_obj
+      if cur_obj then
+        if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then
+          --- @diagnostic disable-next-line:inject-field
+          cur_obj.line = line
+        end
+      end
+
+      state = {}
+    end
+  end
+
+  -- dump_uncommitted(filename, uncommitted)
+
+  return classes, funs, briefs, uncommitted
+end
+
+--- @param filename string
+function M.parse(filename)
+  local f = assert(io.open(filename, 'r'))
+  local txt = f:read('*all')
+  f:close()
+
+  return M.parse_str(txt, filename)
+end
+
+return M
diff --git a/src/gen/nvim_version.lua.in b/src/gen/nvim_version.lua.in
new file mode 100644
index 0000000000..c29141fc68
--- /dev/null
+++ b/src/gen/nvim_version.lua.in
@@ -0,0 +1,9 @@
+return {
+  {"major", ${NVIM_VERSION_MAJOR}},
+  {"minor", ${NVIM_VERSION_MINOR}},
+  {"patch", ${NVIM_VERSION_PATCH}},
+  {"prerelease", "${NVIM_VERSION_PRERELEASE}" ~= ""},
+  {"api_level", ${NVIM_API_LEVEL}},
+  {"api_compatible", ${NVIM_API_LEVEL_COMPAT}},
+  {"api_prerelease", ${NVIM_API_PRERELEASE}},
+}
diff --git a/src/gen/preload.lua b/src/gen/preload.lua
new file mode 100644
index 0000000000..4856d8d7a1
--- /dev/null
+++ b/src/gen/preload.lua
@@ -0,0 +1,6 @@
+local srcdir = table.remove(arg, 1)
+
+package.path = (srcdir .. '/src/?.lua;') .. (srcdir .. '/runtime/lua/?.lua;') .. package.path
+
+arg[0] = table.remove(arg, 1)
+return loadfile(arg[0])()
diff --git a/src/gen/preload_nlua.lua b/src/gen/preload_nlua.lua
new file mode 100644
index 0000000000..a1d89105bc
--- /dev/null
+++ b/src/gen/preload_nlua.lua
@@ -0,0 +1,17 @@
+local srcdir = table.remove(arg, 1)
+local nlualib = table.remove(arg, 1)
+local gendir = table.remove(arg, 1)
+
+package.path = (srcdir .. '/src/?.lua;')
+  .. (srcdir .. '/runtime/lua/?.lua;')
+  .. (gendir .. '/?.lua;')
+  .. package.path
+
+_G.vim = require 'vim.shared'
+_G.vim.inspect = require 'vim.inspect'
+package.cpath = package.cpath .. ';' .. nlualib
+require 'nlua0'
+vim.NIL = vim.mpack.NIL -- WOW BOB WOW
+
+arg[0] = table.remove(arg, 1)
+return loadfile(arg[0])()
diff --git a/src/gen/util.lua b/src/gen/util.lua
new file mode 100644
index 0000000000..5940221abe
--- /dev/null
+++ b/src/gen/util.lua
@@ -0,0 +1,399 @@
+-- TODO(justinmk): move most of this to `vim.text`.
+
+local fmt = string.format
+
+--- @class nvim.util.MDNode
+--- @field [integer] nvim.util.MDNode
+--- @field type string
+--- @field text? string
+
+local INDENTATION = 4
+
+local NBSP = string.char(160)
+
+local M = {}
+
+local function contains(t, xs)
+  return vim.tbl_contains(xs, t)
+end
+
+-- Map of api_level:version, by inspection of:
+--    :lua= vim.mpack.decode(vim.fn.readfile('test/functional/fixtures/api_level_9.mpack','B')).version
+M.version_level = {
+  [13] = '0.11.0',
+  [12] = '0.10.0',
+  [11] = '0.9.0',
+  [10] = '0.8.0',
+  [9] = '0.7.0',
+  [8] = '0.6.0',
+  [7] = '0.5.0',
+  [6] = '0.4.0',
+  [5] = '0.3.2',
+  [4] = '0.3.0',
+  [3] = '0.2.1',
+  [2] = '0.2.0',
+  [1] = '0.1.0',
+}
+
+--- @param txt string
+--- @param srow integer
+--- @param scol integer
+--- @param erow? integer
+--- @param ecol? integer
+--- @return string
+local function slice_text(txt, srow, scol, erow, ecol)
+  local lines = vim.split(txt, '\n')
+
+  if srow == erow then
+    return lines[srow + 1]:sub(scol + 1, ecol)
+  end
+
+  if erow then
+    -- Trim the end
+    for _ = erow + 2, #lines do
+      table.remove(lines, #lines)
+    end
+  end
+
+  -- Trim the start
+  for _ = 1, srow do
+    table.remove(lines, 1)
+  end
+
+  lines[1] = lines[1]:sub(scol + 1)
+  lines[#lines] = lines[#lines]:sub(1, ecol)
+
+  return table.concat(lines, '\n')
+end
+
+--- @param text string
+--- @return nvim.util.MDNode
+local function parse_md_inline(text)
+  local parser = vim.treesitter.languagetree.new(text, 'markdown_inline')
+  local root = parser:parse(true)[1]:root()
+
+  --- @param node TSNode
+  --- @return nvim.util.MDNode?
+  local function extract(node)
+    local ntype = node:type()
+
+    if ntype:match('^%p$') then
+      return
+    end
+
+    --- @type table
+    local ret = { type = ntype }
+    ret.text = vim.treesitter.get_node_text(node, text)
+
+    local row, col = 0, 0
+
+    for child, child_field in node:iter_children() do
+      local e = extract(child)
+      if e and ntype == 'inline' then
+        local srow, scol = child:start()
+        if (srow == row and scol > col) or srow > row then
+          local t = slice_text(ret.text, row, col, srow, scol)
+          if t and t ~= '' then
+            table.insert(ret, { type = 'text', j = true, text = t })
+          end
+        end
+        row, col = child:end_()
+      end
+
+      if child_field then
+        ret[child_field] = e
+      else
+        table.insert(ret, e)
+      end
+    end
+
+    if ntype == 'inline' and (row > 0 or col > 0) then
+      local t = slice_text(ret.text, row, col)
+      if t and t ~= '' then
+        table.insert(ret, { type = 'text', text = t })
+      end
+    end
+
+    return ret
+  end
+
+  return extract(root) or {}
+end
+
+--- @param text string
+--- @return nvim.util.MDNode
+local function parse_md(text)
+  local parser = vim.treesitter.languagetree.new(text, 'markdown', {
+    injections = { markdown = '' },
+  })
+
+  local root = parser:parse(true)[1]:root()
+
+  local EXCLUDE_TEXT_TYPE = {
+    list = true,
+    list_item = true,
+    section = true,
+    document = true,
+    fenced_code_block = true,
+    fenced_code_block_delimiter = true,
+  }
+
+  --- @param node TSNode
+  --- @return nvim.util.MDNode?
+  local function extract(node)
+    local ntype = node:type()
+
+    if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then
+      return
+    end
+
+    --- @type table
+    local ret = { type = ntype }
+
+    if not EXCLUDE_TEXT_TYPE[ntype] then
+      ret.text = vim.treesitter.get_node_text(node, text)
+    end
+
+    if ntype == 'inline' then
+      ret = parse_md_inline(ret.text)
+    end
+
+    for child, child_field in node:iter_children() do
+      local e = extract(child)
+      if child_field then
+        ret[child_field] = e
+      else
+        table.insert(ret, e)
+      end
+    end
+
+    return ret
+  end
+
+  return extract(root) or {}
+end
+
+--- Prefixes each line in `text`.
+---
+--- Does not wrap, not important for "meta" files? (You probably want md_to_vimdoc instead.)
+---
+--- @param text string
+--- @param prefix_ string
+function M.prefix_lines(prefix_, text)
+  local r = ''
+  for _, l in ipairs(vim.split(text, '\n', { plain = true })) do
+    r = r .. vim.trim(prefix_ .. l) .. '\n'
+  end
+  return r
+end
+
+--- @param x string
+--- @param start_indent integer
+--- @param indent integer
+--- @param text_width integer
+--- @return string
+function M.wrap(x, start_indent, indent, text_width)
+  local words = vim.split(vim.trim(x), '%s+')
+  local parts = { string.rep(' ', start_indent) } --- @type string[]
+  local count = indent
+
+  for i, w in ipairs(words) do
+    if count > indent and count + #w > text_width - 1 then
+      parts[#parts + 1] = '\n'
+      parts[#parts + 1] = string.rep(' ', indent)
+      count = indent
+    elseif i ~= 1 then
+      parts[#parts + 1] = ' '
+      count = count + 1
+    end
+    count = count + #w
+    parts[#parts + 1] = w
+  end
+
+  return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', ''))
+end
+
+--- @param node nvim.util.MDNode
+--- @param start_indent integer
+--- @param indent integer
+--- @param text_width integer
+--- @param level integer
+--- @return string[]
+local function render_md(node, start_indent, indent, text_width, level, is_list)
+  local parts = {} --- @type string[]
+
+  -- For debugging
+  local add_tag = false
+  -- local add_tag = true
+
+  local ntype = node.type
+
+  if add_tag then
+    parts[#parts + 1] = '<' .. ntype .. '>'
+  end
+
+  if ntype == 'text' then
+    parts[#parts + 1] = node.text
+  elseif ntype == 'html_tag' then
+    error('html_tag: ' .. node.text)
+  elseif ntype == 'inline_link' then
+    vim.list_extend(parts, { '*', node[1].text, '*' })
+  elseif ntype == 'shortcut_link' then
+    if node[1].text:find('^<.*>$') then
+      parts[#parts + 1] = node[1].text
+    elseif node[1].text:find('^%d+$') then
+      vim.list_extend(parts, { '[', node[1].text, ']' })
+    else
+      vim.list_extend(parts, { '|', node[1].text, '|' })
+    end
+  elseif ntype == 'backslash_escape' then
+    parts[#parts + 1] = node.text
+  elseif ntype == 'emphasis' then
+    parts[#parts + 1] = node.text:sub(2, -2)
+  elseif ntype == 'code_span' then
+    vim.list_extend(parts, { '`', node.text:sub(2, -2):gsub(' ', NBSP), '`' })
+  elseif ntype == 'inline' then
+    if #node == 0 then
+      local text = assert(node.text)
+      parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width)
+    else
+      for _, child in ipairs(node) do
+        vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1))
+      end
+    end
+  elseif ntype == 'paragraph' then
+    local pparts = {}
+    for _, child in ipairs(node) do
+      vim.list_extend(pparts, render_md(child, start_indent, indent, text_width, level + 1))
+    end
+    parts[#parts + 1] = M.wrap(table.concat(pparts), start_indent, indent, text_width)
+    parts[#parts + 1] = '\n'
+  elseif ntype == 'code_fence_content' then
+    local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n')
+
+    local cindent = indent + INDENTATION
+    if level > 3 then
+      -- The tree-sitter markdown parser doesn't parse the code blocks indents
+      -- correctly in lists. Fudge it!
+      lines[1] = '    ' .. lines[1] -- ¯\_(ツ)_/¯
+      cindent = indent - level
+      local _, initial_indent = lines[1]:find('^%s*')
+      initial_indent = initial_indent + cindent
+      if initial_indent < indent then
+        cindent = indent - INDENTATION
+      end
+    end
+
+    for _, l in ipairs(lines) do
+      if #l > 0 then
+        parts[#parts + 1] = string.rep(' ', cindent)
+        parts[#parts + 1] = l
+      end
+      parts[#parts + 1] = '\n'
+    end
+  elseif ntype == 'fenced_code_block' then
+    parts[#parts + 1] = '>'
+    for _, child in ipairs(node) do
+      if child.type == 'info_string' then
+        parts[#parts + 1] = child.text
+        break
+      end
+    end
+    parts[#parts + 1] = '\n'
+    for _, child in ipairs(node) do
+      if child.type ~= 'info_string' then
+        vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1))
+      end
+    end
+    parts[#parts + 1] = '<\n'
+  elseif ntype == 'html_block' then
+    local text = node.text:gsub('^
help', '')
+    text = text:gsub('
%s*$', '')
+    parts[#parts + 1] = text
+  elseif ntype == 'list_marker_dot' then
+    parts[#parts + 1] = node.text
+  elseif contains(ntype, { 'list_marker_minus', 'list_marker_star' }) then
+    parts[#parts + 1] = '• '
+  elseif ntype == 'list_item' then
+    parts[#parts + 1] = string.rep(' ', indent)
+    local offset = node[1].type == 'list_marker_dot' and 3 or 2
+    for i, child in ipairs(node) do
+      local sindent = i <= 2 and 0 or (indent + offset)
+      vim.list_extend(
+        parts,
+        render_md(child, sindent, indent + offset, text_width, level + 1, true)
+      )
+    end
+  else
+    if node.text then
+      error(fmt('cannot render:\n%s', vim.inspect(node)))
+    end
+    for i, child in ipairs(node) do
+      local start_indent0 = i == 1 and start_indent or indent
+      vim.list_extend(
+        parts,
+        render_md(child, start_indent0, indent, text_width, level + 1, is_list)
+      )
+      if ntype ~= 'list' and i ~= #node then
+        if (node[i + 1] or {}).type ~= 'list' then
+          parts[#parts + 1] = '\n'
+        end
+      end
+    end
+  end
+
+  if add_tag then
+    parts[#parts + 1] = ''
+  end
+
+  return parts
+end
+
+--- @param text_width integer
+local function align_tags(text_width)
+  --- @param line string
+  --- @return string
+  return function(line)
+    local tag_pat = '%s*(%*.+%*)%s*$'
+    local tags = {}
+    for m in line:gmatch(tag_pat) do
+      table.insert(tags, m)
+    end
+
+    if #tags > 0 then
+      line = line:gsub(tag_pat, '')
+      local tags_str = ' ' .. table.concat(tags, ' ')
+      --- @type integer
+      local conceal_offset = select(2, tags_str:gsub('%*', '')) - 2
+      local pad = string.rep(' ', text_width - #line - #tags_str + conceal_offset)
+      return line .. pad .. tags_str
+    end
+
+    return line
+  end
+end
+
+--- @param text string
+--- @param start_indent integer
+--- @param indent integer
+--- @param is_list? boolean
+--- @return string
+function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list)
+  -- Add an extra newline so the parser can properly capture ending ```
+  local parsed = parse_md(text .. '\n')
+  local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list)
+
+  local lines = vim.split(table.concat(ret):gsub(NBSP, ' '), '\n')
+
+  lines = vim.tbl_map(align_tags(text_width), lines)
+
+  local s = table.concat(lines, '\n')
+
+  -- Reduce whitespace in code-blocks
+  s = s:gsub('\n+%s*>([a-z]+)\n', ' >%1\n')
+  s = s:gsub('\n+%s*>\n?\n', ' >\n')
+
+  return s
+end
+
+return M
diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index 111a6567ec..8112045d11 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -292,7 +292,7 @@ set(UI_METADATA ${PROJECT_BINARY_DIR}/ui_metadata.mpack)
 set(BINARY_LIB_DIR ${PROJECT_BINARY_DIR}/lib/nvim)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
 set(GENERATED_INCLUDES_DIR ${PROJECT_BINARY_DIR}/include)
-set(GENERATOR_DIR ${CMAKE_CURRENT_LIST_DIR}/generators)
+set(GENERATOR_DIR ${PROJECT_SOURCE_DIR}/src/gen)
 set(GEN_EVAL_TOUCH ${TOUCHES_DIR}/gen_doc_eval)
 set(LUAJIT_RUNTIME_DIR ${DEPS_PREFIX}/share/luajit-2.1/jit)
 set(NVIM_RUNTIME_DIR ${PROJECT_SOURCE_DIR}/runtime)
@@ -306,7 +306,8 @@ set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua)
 set(FUNCS_GENERATOR ${GENERATOR_DIR}/gen_eval.lua)
 set(GENERATOR_C_GRAMMAR ${GENERATOR_DIR}/c_grammar.lua)
 set(GENERATOR_HASHY ${GENERATOR_DIR}/hashy.lua)
-set(GENERATOR_PRELOAD ${GENERATOR_DIR}/preload.lua)
+set(GENERATOR_PRELOAD ${GENERATOR_DIR}/preload_nlua.lua)
+set(NVIM_LUA_PRELOAD ${GENERATOR_DIR}/preload.lua)
 set(HEADER_GENERATOR ${GENERATOR_DIR}/gen_declarations.lua)
 set(OPTIONS_GENERATOR ${GENERATOR_DIR}/gen_options.lua)
 
@@ -514,6 +515,9 @@ add_custom_command(
 set(LUA_GEN ${LUA_GEN_PRG} ${GENERATOR_PRELOAD} ${PROJECT_SOURCE_DIR} $ ${PROJECT_BINARY_DIR})
 set(LUA_GEN_DEPS ${GENERATOR_PRELOAD} $)
 
+# Like LUA_GEN but includes also vim.fn, vim.api, vim.uv, etc
+set(NVIM_LUA $ -u NONE -l ${NVIM_LUA_PRELOAD} ${PROJECT_SOURCE_DIR})
+
 # NVIM_GENERATED_FOR_HEADERS: generated headers to be included in headers
 # NVIM_GENERATED_FOR_SOURCES: generated headers to be included in sources
 # These lists must be mutually exclusive.
@@ -937,12 +941,12 @@ file(GLOB LUA_SOURCES CONFIGURE_DEPENDS
 )
 
 add_target(doc-vim
-  COMMAND $ -u NONE -l scripts/gen_vimdoc.lua
+  COMMAND ${NVIM_LUA} src/gen/gen_vimdoc.lua
   DEPENDS
     nvim
     ${API_SOURCES}
     ${LUA_SOURCES}
-    ${PROJECT_SOURCE_DIR}/scripts/gen_vimdoc.lua
+    ${PROJECT_SOURCE_DIR}/src/gen/gen_vimdoc.lua
     ${NVIM_RUNTIME_DIR}/doc/api.txt
     ${NVIM_RUNTIME_DIR}/doc/diagnostic.txt
     ${NVIM_RUNTIME_DIR}/doc/lsp.txt
@@ -951,11 +955,11 @@ add_target(doc-vim
   )
 
 add_target(doc-eval
-  COMMAND $ -u NONE -l ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
+  COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua
   DEPENDS
     nvim
     ${FUNCS_METADATA}
-    ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
+    ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua
     ${PROJECT_SOURCE_DIR}/src/nvim/eval.lua
     ${PROJECT_SOURCE_DIR}/src/nvim/options.lua
     ${PROJECT_SOURCE_DIR}/src/nvim/vvars.lua
@@ -966,7 +970,7 @@ add_custom_target(doc)
 add_dependencies(doc doc-vim doc-eval)
 
 add_target(lintdoc
-  COMMAND $ -u NONE -l scripts/lintdoc.lua
+  COMMAND ${NVIM_LUA} scripts/lintdoc.lua
   DEPENDS ${DOCFILES}
   CUSTOM_COMMAND_ARGS USES_TERMINAL)
 add_dependencies(lintdoc nvim)
diff --git a/src/nvim/func_attr.h b/src/nvim/func_attr.h
index 43af880767..e19a0acd5d 100644
--- a/src/nvim/func_attr.h
+++ b/src/nvim/func_attr.h
@@ -1,6 +1,6 @@
 // Undefined DEFINE_FUNC_ATTRIBUTES and undefined DEFINE_EMPTY_ATTRIBUTES
 // leaves file with untouched FUNC_ATTR_* macros. This variant is used for
-// scripts/gen_declarations.lua.
+// src/gen/gen_declarations.lua.
 //
 // Empty macros are used for *.c files.
 // (undefined DEFINE_FUNC_ATTRIBUTES and defined DEFINE_EMPTY_ATTRIBUTES)
diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
deleted file mode 100644
index 890c260843..0000000000
--- a/src/nvim/generators/c_grammar.lua
+++ /dev/null
@@ -1,300 +0,0 @@
--- lpeg grammar for building api metadata from a set of header files. It
--- ignores comments and preprocessor commands and parses a very small subset
--- of C prototypes with a limited set of types
-
---- @class nvim.c_grammar.Proto
---- @field [1] 'proto'
---- @field pos integer
---- @field endpos integer
---- @field fast boolean
---- @field name string
---- @field return_type string
---- @field parameters [string, string][]
---- @field static true?
---- @field inline true?
-
---- @class nvim.c_grammar.Preproc
---- @field [1] 'preproc'
---- @field content string
-
---- @class nvim.c_grammar.Empty
---- @field [1] 'empty'
-
---- @alias nvim.c_grammar.result
---- | nvim.c_grammar.Proto
---- | nvim.c_grammar.Preproc
---- | nvim.c_grammar.Empty
-
---- @class nvim.c_grammar
---- @field match fun(self, input: string): nvim.c_grammar.result[]
-
-local lpeg = vim.lpeg
-
-local P, R, S, V = lpeg.P, lpeg.R, lpeg.S, lpeg.V
-local C, Ct, Cc, Cg, Cp = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cp
-
---- @param pat vim.lpeg.Pattern
-local function rep(pat)
-  return pat ^ 0
-end
-
---- @param pat vim.lpeg.Pattern
-local function rep1(pat)
-  return pat ^ 1
-end
-
---- @param pat vim.lpeg.Pattern
-local function opt(pat)
-  return pat ^ -1
-end
-
-local any = P(1)
-local letter = R('az', 'AZ') + S('_$')
-local num = R('09')
-local alpha = letter + num
-local nl = P('\r\n') + P('\n')
-local space = S(' \t')
-local str = P('"') * rep((P('\\') * any) + (1 - P('"'))) * P('"')
-local char = P("'") * (any - P("'")) * P("'")
-local ws = space + nl
-local wb = #-alpha -- word boundary
-local id = letter * rep(alpha)
-
-local comment_inline = P('/*') * rep(1 - P('*/')) * P('*/')
-local comment = P('//') * rep(1 - nl) * nl
-local preproc = Ct(Cc('preproc') * P('#') * Cg(rep(1 - nl) * nl, 'content'))
-
-local fill = rep(ws + comment_inline + comment + preproc)
-
---- @param s string
---- @return vim.lpeg.Pattern
-local function word(s)
-  return fill * P(s) * wb * fill
-end
-
---- @param x vim.lpeg.Pattern
-local function comma1(x)
-  return x * rep(fill * P(',') * fill * x)
-end
-
---- @param v string
-local function Pf(v)
-  return fill * P(v) * fill
-end
-
---- @param x vim.lpeg.Pattern
-local function paren(x)
-  return P('(') * fill * x * fill * P(')')
-end
-
-local cdoc_comment = P('///') * opt(Ct(Cg(rep(space) * rep(1 - nl), 'comment')))
-
-local braces = P({
-  'S',
-  A = comment_inline + comment + preproc + str + char + (any - S('{}')),
-  S = P('{') * rep(V('A')) * rep(V('S') + V('A')) * P('}'),
-})
-
--- stylua: ignore start
-local typed_container = P({
-  'S',
-  S = (
-    (P('Union') * paren(comma1(V('ID'))))
-    + (P('ArrayOf') * paren(id * opt(P(',') * fill * rep1(num))))
-    + (P('DictOf') * paren(id))
-    + (P('LuaRefOf') * paren(
-      paren(comma1((V('ID') + str) * rep1(ws) * opt(P('*')) * id))
-      * opt(P(',') * fill * opt(P('*')) * V('ID'))
-    ))
-    + (P('Dict') * paren(id))),
-  ID = V('S') + id,
-})
--- stylua: ignore end
-
-local ptr_mod = word('restrict') + word('__restrict') + word('const')
-local opt_ptr = rep(Pf('*') * opt(ptr_mod))
-
---- @param name string
---- @param var string
---- @return vim.lpeg.Pattern
-local function attr(name, var)
-  return Cg((P(name) * Cc(true)), var)
-end
-
---- @param name string
---- @param var string
---- @return vim.lpeg.Pattern
-local function attr_num(name, var)
-  return Cg((P(name) * paren(C(rep1(num)))), var)
-end
-
-local fattr = (
-  attr_num('FUNC_API_SINCE', 'since')
-  + attr_num('FUNC_API_DEPRECATED_SINCE', 'deprecated_since')
-  + attr('FUNC_API_FAST', 'fast')
-  + attr('FUNC_API_RET_ALLOC', 'ret_alloc')
-  + attr('FUNC_API_NOEXPORT', 'noexport')
-  + attr('FUNC_API_REMOTE_ONLY', 'remote_only')
-  + attr('FUNC_API_LUA_ONLY', 'lua_only')
-  + attr('FUNC_API_TEXTLOCK_ALLOW_CMDWIN', 'textlock_allow_cmdwin')
-  + attr('FUNC_API_TEXTLOCK', 'textlock')
-  + attr('FUNC_API_REMOTE_IMPL', 'remote_impl')
-  + attr('FUNC_API_COMPOSITOR_IMPL', 'compositor_impl')
-  + attr('FUNC_API_CLIENT_IMPL', 'client_impl')
-  + attr('FUNC_API_CLIENT_IGNORE', 'client_ignore')
-  + (P('FUNC_') * rep(alpha) * opt(fill * paren(rep(1 - P(')') * any))))
-)
-
-local void = P('void') * wb
-
-local api_param_type = (
-  (word('Error') * opt_ptr * Cc('error'))
-  + (word('Arena') * opt_ptr * Cc('arena'))
-  + (word('lua_State') * opt_ptr * Cc('lstate'))
-)
-
-local ctype = C(
-  opt(word('const'))
-    * (
-      typed_container
-      -- 'unsigned' is a type modifier, and a type itself
-      + (word('unsigned char') + word('unsigned'))
-      + (word('struct') * fill * id)
-      + id
-    )
-    * opt(word('const'))
-    * opt_ptr
-)
-
-local return_type = (C(void) * fill) + ctype
-
--- stylua: ignore start
-local params = Ct(
-  (void * #P(')'))
-  + comma1(Ct(
-      (api_param_type + ctype)
-      * fill
-      * C(id)
-      * rep(Pf('[') * rep(alpha) * Pf(']'))
-      * rep(fill * fattr)
-    ))
-    * opt(Pf(',') * P('...'))
-)
--- stylua: ignore end
-
-local ignore_line = rep1(1 - nl) * nl
-local empty_line = Ct(Cc('empty') * nl * nl)
-
-local proto_name = opt_ptr * fill * id
-
--- __inline is used in MSVC
-local decl_mod = (
-  Cg(word('static') * Cc(true), 'static')
-  + Cg((word('inline') + word('__inline')) * Cc(true), 'inline')
-)
-
-local proto = Ct(
-  Cg(Cp(), 'pos')
-    * Cc('proto')
-    * -#P('typedef')
-    * #alpha
-    * opt(P('DLLEXPORT') * rep1(ws))
-    * rep(decl_mod)
-    * Cg(return_type, 'return_type')
-    * fill
-    * Cg(proto_name, 'name')
-    * fill
-    * paren(Cg(params, 'parameters'))
-    * Cg(Cc(false), 'fast')
-    * rep(fill * fattr)
-    * Cg(Cp(), 'endpos')
-    * (fill * (S(';') + braces))
-)
-
-local keyset_field = Ct(
-  Cg(ctype, 'type')
-    * fill
-    * Cg(id, 'name')
-    * fill
-    * opt(P('DictKey') * paren(Cg(rep1(1 - P(')')), 'dict_key')))
-    * Pf(';')
-)
-
-local keyset = Ct(
-  P('typedef')
-    * word('struct')
-    * Pf('{')
-    * Cg(Ct(rep1(keyset_field)), 'fields')
-    * Pf('}')
-    * P('Dict')
-    * paren(Cg(id, 'keyset_name'))
-    * Pf(';')
-)
-
-local grammar =
-  Ct(rep1(empty_line + proto + cdoc_comment + comment + preproc + ws + keyset + ignore_line))
-
-if arg[1] == '--test' then
-  for i, t in ipairs({
-    'void multiqueue_put_event(MultiQueue *self, Event event) {} ',
-    'void *xmalloc(size_t size) {} ',
-    {
-      'struct tm *os_localtime_r(const time_t *restrict clock,',
-      '                          struct tm *restrict result) FUNC_ATTR_NONNULL_ALL {}',
-    },
-    {
-      '_Bool',
-      '# 163 "src/nvim/event/multiqueue.c"',
-      '    multiqueue_empty(MultiQueue *self)',
-      '{}',
-    },
-    'const char *find_option_end(const char *arg, OptIndex *opt_idxp) {}',
-    'bool semsg(const char *const fmt, ...) {}',
-    'int32_t utf_ptr2CharInfo_impl(uint8_t const *p, uintptr_t const len) {}',
-    'void ex_argdedupe(exarg_T *eap FUNC_ATTR_UNUSED) {}',
-    'static TermKeySym register_c0(TermKey *tk, TermKeySym sym, unsigned char ctrl, const char *name) {}',
-    'unsigned get_bkc_flags(buf_T *buf) {}',
-    'char *xstpcpy(char *restrict dst, const char *restrict src) {}',
-    'bool try_leave(const TryState *const tstate, Error *const err) {}',
-    'void api_set_error(ErrorType errType) {}',
-    {
-      'void nvim_subscribe(uint64_t channel_id, String event)',
-      'FUNC_API_SINCE(1) FUNC_API_DEPRECATED_SINCE(13) FUNC_API_REMOTE_ONLY',
-      '{}',
-    },
-
-    -- Do not consume leading preproc statements
-    {
-      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
-      'static __inline int mark_global_index(const char name)',
-      '  FUNC_ATTR_CONST',
-      '{}',
-    },
-    {
-      '',
-      '#line 1 "D:/a/neovim/neovim/src\\nvim/mark.h"',
-      'static __inline int mark_global_index(const char name)',
-      '{}',
-    },
-    {
-      'size_t xstrlcpy(char *__restrict dst, const char *__restrict src, size_t dsize)',
-      ' FUNC_ATTR_NONNULL_ALL',
-      ' {}',
-    },
-  }) do
-    if type(t) == 'table' then
-      t = table.concat(t, '\n') .. '\n'
-    end
-    t = t:gsub(' +', ' ')
-    local r = grammar:match(t)
-    if not r then
-      print('Test ' .. i .. ' failed')
-      print('    |' .. table.concat(vim.split(t, '\n'), '\n    |'))
-    end
-  end
-end
-
-return {
-  grammar = grammar --[[@as nvim.c_grammar]],
-  typed_container = typed_container,
-}
diff --git a/src/nvim/generators/dump_bin_array.lua b/src/nvim/generators/dump_bin_array.lua
deleted file mode 100644
index c6cda25e73..0000000000
--- a/src/nvim/generators/dump_bin_array.lua
+++ /dev/null
@@ -1,17 +0,0 @@
-local function dump_bin_array(output, name, data)
-  output:write([[
-  static const uint8_t ]] .. name .. [[[] = {
-]])
-
-  for i = 1, #data do
-    output:write(string.byte(data, i) .. ', ')
-    if i % 10 == 0 then
-      output:write('\n  ')
-    end
-  end
-  output:write([[
-};
-]])
-end
-
-return dump_bin_array
diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
deleted file mode 100644
index 378297d86a..0000000000
--- a/src/nvim/generators/gen_api_dispatch.lua
+++ /dev/null
@@ -1,990 +0,0 @@
--- Example (manual) invocation:
---
---    make
---    cp build/nvim_version.lua src/nvim/
---    cd src/nvim
---    nvim -l generators/gen_api_dispatch.lua "../../build/src/nvim/auto/api/private/dispatch_wrappers.generated.h" "../../build/src/nvim/auto/api/private/api_metadata.generated.h" "../../build/funcs_metadata.mpack" "../../build/src/nvim/auto/lua_api_c_bindings.generated.h" "../../build/src/nvim/auto/keysets_defs.generated.h" "../../build/ui_metadata.mpack" "../../build/cmake.config/auto/versiondef_git.h" "./api/autocmd.h" "./api/buffer.h" "./api/command.h" "./api/deprecated.h" "./api/extmark.h" "./api/keysets_defs.h" "./api/options.h" "./api/tabpage.h" "./api/ui.h" "./api/vim.h" "./api/vimscript.h" "./api/win_config.h" "./api/window.h" "../../build/include/api/autocmd.h.generated.h" "../../build/include/api/buffer.h.generated.h" "../../build/include/api/command.h.generated.h" "../../build/include/api/deprecated.h.generated.h" "../../build/include/api/extmark.h.generated.h" "../../build/include/api/options.h.generated.h" "../../build/include/api/tabpage.h.generated.h" "../../build/include/api/ui.h.generated.h" "../../build/include/api/vim.h.generated.h" "../../build/include/api/vimscript.h.generated.h" "../../build/include/api/win_config.h.generated.h" "../../build/include/api/window.h.generated.h"
-
-local mpack = vim.mpack
-
-local hashy = require 'generators.hashy'
-
-local pre_args = 7
-assert(#arg >= pre_args)
--- output h file with generated dispatch functions (dispatch_wrappers.generated.h)
-local dispatch_outputf = arg[1]
--- output h file with packed metadata (api_metadata.generated.h)
-local api_metadata_outputf = arg[2]
--- output metadata mpack file, for use by other build scripts (funcs_metadata.mpack)
-local mpack_outputf = arg[3]
-local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c
-local keysets_outputf = arg[5] -- keysets_defs.generated.h
-local ui_metadata_inputf = arg[6] -- ui events metadata
-local git_version_inputf = arg[7] -- git version header
-
-local functions = {}
-
--- names of all headers relative to the source root (for inclusion in the
--- generated file)
-local headers = {}
-
--- set of function names, used to detect duplicates
-local function_names = {}
-
-local c_grammar = require('generators.c_grammar')
-
-local startswith = vim.startswith
-
-local function add_function(fn)
-  local public = startswith(fn.name, 'nvim_') or fn.deprecated_since
-  if public and not fn.noexport then
-    functions[#functions + 1] = fn
-    function_names[fn.name] = true
-    if
-      #fn.parameters >= 2
-      and fn.parameters[2][1] == 'Array'
-      and fn.parameters[2][2] == 'uidata'
-    then
-      -- function receives the "args" as a parameter
-      fn.receives_array_args = true
-      -- remove the args parameter
-      table.remove(fn.parameters, 2)
-    end
-    if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
-      -- this function should receive the channel id
-      fn.receives_channel_id = true
-      -- remove the parameter since it won't be passed by the api client
-      table.remove(fn.parameters, 1)
-    end
-    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then
-      -- function can fail if the last parameter type is 'Error'
-      fn.can_fail = true
-      -- remove the error parameter, msgpack has it's own special field
-      -- for specifying errors
-      fn.parameters[#fn.parameters] = nil
-    end
-    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'lstate' then
-      fn.has_lua_imp = true
-      fn.parameters[#fn.parameters] = nil
-    end
-    if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'arena' then
-      fn.receives_arena = true
-      fn.parameters[#fn.parameters] = nil
-    end
-  end
-end
-
-local keysets = {}
-
-local function add_keyset(val)
-  local keys = {}
-  local types = {}
-  local c_names = {}
-  local is_set_name = 'is_set__' .. val.keyset_name .. '_'
-  local has_optional = false
-  for i, field in ipairs(val.fields) do
-    local dict_key = field.dict_key or field.name
-    if field.type ~= 'Object' then
-      types[dict_key] = field.type
-    end
-    if field.name ~= is_set_name and field.type ~= 'OptionalKeys' then
-      table.insert(keys, dict_key)
-      if dict_key ~= field.name then
-        c_names[dict_key] = field.name
-      end
-    else
-      if i > 1 then
-        error("'is_set__{type}_' must be first if present")
-      elseif field.name ~= is_set_name then
-        error(val.keyset_name .. ': name of first key should be ' .. is_set_name)
-      elseif field.type ~= 'OptionalKeys' then
-        error("'" .. is_set_name .. "' must have type 'OptionalKeys'")
-      end
-      has_optional = true
-    end
-  end
-  table.insert(keysets, {
-    name = val.keyset_name,
-    keys = keys,
-    c_names = c_names,
-    types = types,
-    has_optional = has_optional,
-  })
-end
-
-local ui_options_text = nil
-
--- read each input file, parse and append to the api metadata
-for i = pre_args + 1, #arg do
-  local full_path = arg[i]
-  local parts = {}
-  for part in string.gmatch(full_path, '[^/]+') do
-    parts[#parts + 1] = part
-  end
-  headers[#headers + 1] = parts[#parts - 1] .. '/' .. parts[#parts]
-
-  local input = assert(io.open(full_path, 'rb'))
-
-  local text = input:read('*all')
-  local tmp = c_grammar.grammar:match(text)
-  for j = 1, #tmp do
-    local val = tmp[j]
-    if val.keyset_name then
-      add_keyset(val)
-    elseif val.name then
-      add_function(val)
-    end
-  end
-
-  ui_options_text = ui_options_text or string.match(text, 'ui_ext_names%[][^{]+{([^}]+)}')
-  input:close()
-end
-
-local function shallowcopy(orig)
-  local copy = {}
-  for orig_key, orig_value in pairs(orig) do
-    copy[orig_key] = orig_value
-  end
-  return copy
-end
-
--- Export functions under older deprecated names.
--- These will be removed eventually.
-local deprecated_aliases = require('api.dispatch_deprecated')
-for _, f in ipairs(shallowcopy(functions)) do
-  local ismethod = false
-  if startswith(f.name, 'nvim_') then
-    if startswith(f.name, 'nvim__') or f.name == 'nvim_error_event' then
-      f.since = -1
-    elseif f.since == nil then
-      print('Function ' .. f.name .. ' lacks since field.\n')
-      os.exit(1)
-    end
-    f.since = tonumber(f.since)
-    if f.deprecated_since ~= nil then
-      f.deprecated_since = tonumber(f.deprecated_since)
-    end
-
-    if startswith(f.name, 'nvim_buf_') then
-      ismethod = true
-    elseif startswith(f.name, 'nvim_win_') then
-      ismethod = true
-    elseif startswith(f.name, 'nvim_tabpage_') then
-      ismethod = true
-    end
-    f.remote = f.remote_only or not f.lua_only
-    f.lua = f.lua_only or not f.remote_only
-    f.eval = (not f.lua_only) and not f.remote_only
-  else
-    f.deprecated_since = tonumber(f.deprecated_since)
-    assert(f.deprecated_since == 1)
-    f.remote = true
-    f.since = 0
-  end
-  f.method = ismethod
-  local newname = deprecated_aliases[f.name]
-  if newname ~= nil then
-    if function_names[newname] then
-      -- duplicate
-      print(
-        'Function '
-          .. f.name
-          .. ' has deprecated alias\n'
-          .. newname
-          .. ' which has a separate implementation.\n'
-          .. 'Please remove it from src/nvim/api/dispatch_deprecated.lua'
-      )
-      os.exit(1)
-    end
-    local newf = shallowcopy(f)
-    newf.name = newname
-    if newname == 'ui_try_resize' then
-      -- The return type was incorrectly set to Object in 0.1.5.
-      -- Keep it that way for clients that rely on this.
-      newf.return_type = 'Object'
-    end
-    newf.impl_name = f.name
-    newf.lua = false
-    newf.eval = false
-    newf.since = 0
-    newf.deprecated_since = 1
-    functions[#functions + 1] = newf
-  end
-end
-
--- don't expose internal attributes like "impl_name" in public metadata
-local exported_attributes = { 'name', 'return_type', 'method', 'since', 'deprecated_since' }
-local exported_functions = {}
-for _, f in ipairs(functions) do
-  if not (startswith(f.name, 'nvim__') or f.name == 'nvim_error_event' or f.name == 'redraw') then
-    local f_exported = {}
-    for _, attr in ipairs(exported_attributes) do
-      f_exported[attr] = f[attr]
-    end
-    f_exported.parameters = {}
-    for i, param in ipairs(f.parameters) do
-      if param[1] == 'DictOf(LuaRef)' then
-        param = { 'Dict', param[2] }
-      elseif startswith(param[1], 'Dict(') then
-        param = { 'Dict', param[2] }
-      end
-      f_exported.parameters[i] = param
-    end
-    if startswith(f.return_type, 'Dict(') then
-      f_exported.return_type = 'Dict'
-    end
-    exported_functions[#exported_functions + 1] = f_exported
-  end
-end
-
-local ui_options = { 'rgb' }
-for x in string.gmatch(ui_options_text, '"([a-z][a-z_]+)"') do
-  table.insert(ui_options, x)
-end
-
-local version = require 'nvim_version' -- `build/nvim_version.lua` file.
-local git_version = io.open(git_version_inputf):read '*a'
-local version_build = string.match(git_version, '#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
-
--- serialize the API metadata using msgpack and embed into the resulting
--- binary for easy querying by clients
-local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb'))
-local pieces = {}
-
--- Naively using mpack.encode({foo=x, bar=y}) will make the build
--- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead
-local function fixdict(num)
-  if num > 15 then
-    error 'implement more dict codes'
-  end
-  table.insert(pieces, string.char(128 + num))
-end
-local function put(item, item2)
-  table.insert(pieces, mpack.encode(item))
-  if item2 ~= nil then
-    table.insert(pieces, mpack.encode(item2))
-  end
-end
-
-fixdict(6)
-
-put('version')
-fixdict(1 + #version)
-for _, item in ipairs(version) do
-  -- NB: all items are mandatory. But any error will be less confusing
-  -- with placeholder vim.NIL (than invalid mpack data)
-  local val = item[2] == nil and vim.NIL or item[2]
-  put(item[1], val)
-end
-put('build', version_build)
-
-put('functions', exported_functions)
-put('ui_events')
-table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all'))
-put('ui_options', ui_options)
-
-put('error_types')
-fixdict(2)
-put('Exception', { id = 0 })
-put('Validation', { id = 1 })
-
-put('types')
-local types =
-  { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } }
-fixdict(#types)
-for i, item in ipairs(types) do
-  put(item[1])
-  fixdict(2)
-  put('id', i - 1)
-  put('prefix', item[2])
-end
-
-local packed = table.concat(pieces)
-local dump_bin_array = require('generators.dump_bin_array')
-dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
-api_metadata_output:close()
-
--- start building the dispatch wrapper output
-local output = assert(io.open(dispatch_outputf, 'wb'))
-
-local keysets_defs = assert(io.open(keysets_outputf, 'wb'))
-
--- ===========================================================================
--- NEW API FILES MUST GO HERE.
---
---  When creating a new API file, you must include it here,
---  so that the dispatcher can find the C functions that you are creating!
--- ===========================================================================
-output:write([[
-#include "nvim/errors.h"
-#include "nvim/ex_docmd.h"
-#include "nvim/ex_getln.h"
-#include "nvim/globals.h"
-#include "nvim/log.h"
-#include "nvim/map_defs.h"
-
-#include "nvim/api/autocmd.h"
-#include "nvim/api/buffer.h"
-#include "nvim/api/command.h"
-#include "nvim/api/deprecated.h"
-#include "nvim/api/extmark.h"
-#include "nvim/api/options.h"
-#include "nvim/api/tabpage.h"
-#include "nvim/api/ui.h"
-#include "nvim/api/vim.h"
-#include "nvim/api/vimscript.h"
-#include "nvim/api/win_config.h"
-#include "nvim/api/window.h"
-#include "nvim/ui_client.h"
-
-]])
-
-keysets_defs:write('// IWYU pragma: private, include "nvim/api/private/dispatch.h"\n\n')
-
-for _, k in ipairs(keysets) do
-  local neworder, hashfun = hashy.hashy_hash(k.name, k.keys, function(idx)
-    return k.name .. '_table[' .. idx .. '].str'
-  end)
-
-  keysets_defs:write('extern KeySetLink ' .. k.name .. '_table[' .. (1 + #neworder) .. '];\n')
-
-  local function typename(type)
-    if type == 'HLGroupID' then
-      return 'kObjectTypeInteger'
-    elseif not type or vim.startswith(type, 'Union') then
-      return 'kObjectTypeNil'
-    elseif vim.startswith(type, 'LuaRefOf') then
-      return 'kObjectTypeLuaRef'
-    elseif type == 'StringArray' then
-      return 'kUnpackTypeStringArray'
-    elseif vim.startswith(type, 'ArrayOf') then
-      return 'kObjectTypeArray'
-    else
-      return 'kObjectType' .. type
-    end
-  end
-
-  output:write('KeySetLink ' .. k.name .. '_table[] = {\n')
-  for i, key in ipairs(neworder) do
-    local ind = -1
-    if k.has_optional then
-      ind = i
-      keysets_defs:write('#define KEYSET_OPTIDX_' .. k.name .. '__' .. key .. ' ' .. ind .. '\n')
-    end
-    output:write(
-      '  {"'
-        .. key
-        .. '", offsetof(KeyDict_'
-        .. k.name
-        .. ', '
-        .. (k.c_names[key] or key)
-        .. '), '
-        .. typename(k.types[key])
-        .. ', '
-        .. ind
-        .. ', '
-        .. (k.types[key] == 'HLGroupID' and 'true' or 'false')
-        .. '},\n'
-    )
-  end
-  output:write('  {NULL, 0, kObjectTypeNil, -1, false},\n')
-  output:write('};\n\n')
-
-  output:write(hashfun)
-
-  output:write([[
-KeySetLink *KeyDict_]] .. k.name .. [[_get_field(const char *str, size_t len)
-{
-  int hash = ]] .. k.name .. [[_hash(str, len);
-  if (hash == -1) {
-    return NULL;
-  }
-  return &]] .. k.name .. [[_table[hash];
-}
-
-]])
-end
-
-local function real_type(type)
-  local rv = type
-  local rmatch = string.match(type, 'Dict%(([_%w]+)%)')
-  if rmatch then
-    return 'KeyDict_' .. rmatch
-  elseif c_grammar.typed_container:match(rv) then
-    if rv:match('Array') then
-      rv = 'Array'
-    else
-      rv = 'Dict'
-    end
-  end
-  return rv
-end
-
-local function attr_name(rt)
-  if rt == 'Float' then
-    return 'floating'
-  else
-    return rt:lower()
-  end
-end
-
--- start the handler functions. Visit each function metadata to build the
--- handler function with code generated for validating arguments and calling to
--- the real API.
-for i = 1, #functions do
-  local fn = functions[i]
-  if fn.impl_name == nil and fn.remote then
-    local args = {}
-
-    output:write(
-      'Object handle_' .. fn.name .. '(uint64_t channel_id, Array args, Arena* arena, Error *error)'
-    )
-    output:write('\n{')
-    output:write('\n#ifdef NVIM_LOG_DEBUG')
-    output:write('\n  DLOG("RPC: ch %" PRIu64 ": invoke ' .. fn.name .. '", channel_id);')
-    output:write('\n#endif')
-    output:write('\n  Object ret = NIL;')
-    -- Declare/initialize variables that will hold converted arguments
-    for j = 1, #fn.parameters do
-      local param = fn.parameters[j]
-      local rt = real_type(param[1])
-      local converted = 'arg_' .. j
-      output:write('\n  ' .. rt .. ' ' .. converted .. ';')
-    end
-    output:write('\n')
-    if not fn.receives_array_args then
-      output:write('\n  if (args.size != ' .. #fn.parameters .. ') {')
-      output:write(
-        '\n    api_set_error(error, kErrorTypeException, \
-        "Wrong number of arguments: expecting '
-          .. #fn.parameters
-          .. ' but got %zu", args.size);'
-      )
-      output:write('\n    goto cleanup;')
-      output:write('\n  }\n')
-    end
-
-    -- Validation/conversion for each argument
-    for j = 1, #fn.parameters do
-      local converted, param
-      param = fn.parameters[j]
-      converted = 'arg_' .. j
-      local rt = real_type(param[1])
-      if rt == 'Object' then
-        output:write('\n  ' .. converted .. ' = args.items[' .. (j - 1) .. '];\n')
-      elseif rt:match('^KeyDict_') then
-        converted = '&' .. converted
-        output:write('\n  if (args.items[' .. (j - 1) .. '].type == kObjectTypeDict) {') --luacheck: ignore 631
-        output:write('\n    memset(' .. converted .. ', 0, sizeof(*' .. converted .. '));') -- TODO: neeeee
-        output:write(
-          '\n    if (!api_dict_to_keydict('
-            .. converted
-            .. ', '
-            .. rt
-            .. '_get_field, args.items['
-            .. (j - 1)
-            .. '].data.dict, error)) {'
-        )
-        output:write('\n      goto cleanup;')
-        output:write('\n    }')
-        output:write(
-          '\n  } else if (args.items['
-            .. (j - 1)
-            .. '].type == kObjectTypeArray && args.items['
-            .. (j - 1)
-            .. '].data.array.size == 0) {'
-        ) --luacheck: ignore 631
-        output:write('\n    memset(' .. converted .. ', 0, sizeof(*' .. converted .. '));')
-
-        output:write('\n  } else {')
-        output:write(
-          '\n    api_set_error(error, kErrorTypeException, \
-          "Wrong type for argument '
-            .. j
-            .. ' when calling '
-            .. fn.name
-            .. ', expecting '
-            .. param[1]
-            .. '");'
-        )
-        output:write('\n    goto cleanup;')
-        output:write('\n  }\n')
-      else
-        if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then
-          -- Buffer, Window, and Tabpage have a specific type, but are stored in integer
-          output:write(
-            '\n  if (args.items['
-              .. (j - 1)
-              .. '].type == kObjectType'
-              .. rt
-              .. ' && args.items['
-              .. (j - 1)
-              .. '].data.integer >= 0) {'
-          )
-          output:write(
-            '\n    ' .. converted .. ' = (handle_T)args.items[' .. (j - 1) .. '].data.integer;'
-          )
-        else
-          output:write('\n  if (args.items[' .. (j - 1) .. '].type == kObjectType' .. rt .. ') {')
-          output:write(
-            '\n    '
-              .. converted
-              .. ' = args.items['
-              .. (j - 1)
-              .. '].data.'
-              .. attr_name(rt)
-              .. ';'
-          )
-        end
-        if
-          rt:match('^Buffer$')
-          or rt:match('^Window$')
-          or rt:match('^Tabpage$')
-          or rt:match('^Boolean$')
-        then
-          -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages
-          output:write(
-            '\n  } else if (args.items['
-              .. (j - 1)
-              .. '].type == kObjectTypeInteger && args.items['
-              .. (j - 1)
-              .. '].data.integer >= 0) {'
-          )
-          output:write(
-            '\n    ' .. converted .. ' = (handle_T)args.items[' .. (j - 1) .. '].data.integer;'
-          )
-        end
-        if rt:match('^Float$') then
-          -- accept integers for Floats
-          output:write('\n  } else if (args.items[' .. (j - 1) .. '].type == kObjectTypeInteger) {')
-          output:write(
-            '\n    ' .. converted .. ' = (Float)args.items[' .. (j - 1) .. '].data.integer;'
-          )
-        end
-        -- accept empty lua tables as empty dictionaries
-        if rt:match('^Dict') then
-          output:write(
-            '\n  } else if (args.items['
-              .. (j - 1)
-              .. '].type == kObjectTypeArray && args.items['
-              .. (j - 1)
-              .. '].data.array.size == 0) {'
-          ) --luacheck: ignore 631
-          output:write('\n    ' .. converted .. ' = (Dict)ARRAY_DICT_INIT;')
-        end
-        output:write('\n  } else {')
-        output:write(
-          '\n    api_set_error(error, kErrorTypeException, \
-          "Wrong type for argument '
-            .. j
-            .. ' when calling '
-            .. fn.name
-            .. ', expecting '
-            .. param[1]
-            .. '");'
-        )
-        output:write('\n    goto cleanup;')
-        output:write('\n  }\n')
-      end
-      args[#args + 1] = converted
-    end
-
-    if fn.textlock then
-      output:write('\n  if (text_locked()) {')
-      output:write('\n    api_set_error(error, kErrorTypeException, "%s", get_text_locked_msg());')
-      output:write('\n    goto cleanup;')
-      output:write('\n  }\n')
-    elseif fn.textlock_allow_cmdwin then
-      output:write('\n  if (textlock != 0 || expr_map_locked()) {')
-      output:write('\n    api_set_error(error, kErrorTypeException, "%s", e_textlock);')
-      output:write('\n    goto cleanup;')
-      output:write('\n  }\n')
-    end
-
-    -- function call
-    output:write('\n  ')
-    if fn.return_type ~= 'void' then
-      -- has a return value, prefix the call with a declaration
-      output:write(fn.return_type .. ' rv = ')
-    end
-
-    -- write the function name and the opening parenthesis
-    output:write(fn.name .. '(')
-
-    local call_args = {}
-    if fn.receives_channel_id then
-      table.insert(call_args, 'channel_id')
-    end
-
-    if fn.receives_array_args then
-      table.insert(call_args, 'args')
-    end
-
-    for _, a in ipairs(args) do
-      table.insert(call_args, a)
-    end
-
-    if fn.receives_arena then
-      table.insert(call_args, 'arena')
-    end
-
-    if fn.has_lua_imp then
-      table.insert(call_args, 'NULL')
-    end
-
-    if fn.can_fail then
-      table.insert(call_args, 'error')
-    end
-
-    output:write(table.concat(call_args, ', '))
-    output:write(');\n')
-
-    if fn.can_fail then
-      -- if the function can fail, also pass a pointer to the local error object
-      -- and check for the error
-      output:write('\n  if (ERROR_SET(error)) {')
-      output:write('\n    goto cleanup;')
-      output:write('\n  }\n')
-    end
-
-    local ret_type = real_type(fn.return_type)
-    if string.match(ret_type, '^KeyDict_') then
-      local table = string.sub(ret_type, 9) .. '_table'
-      output:write(
-        '\n  ret = DICT_OBJ(api_keydict_to_dict(&rv, '
-          .. table
-          .. ', ARRAY_SIZE('
-          .. table
-          .. '), arena));'
-      )
-    elseif ret_type ~= 'void' then
-      output:write('\n  ret = ' .. string.upper(real_type(fn.return_type)) .. '_OBJ(rv);')
-    end
-    output:write('\n\ncleanup:')
-
-    output:write('\n  return ret;\n}\n\n')
-  end
-end
-
-local remote_fns = {}
-for _, fn in ipairs(functions) do
-  if fn.remote then
-    remote_fns[fn.name] = fn
-  end
-end
-remote_fns.redraw = { impl_name = 'ui_client_redraw', fast = true }
-
-local names = vim.tbl_keys(remote_fns)
-table.sort(names)
-local hashorder, hashfun = hashy.hashy_hash('msgpack_rpc_get_handler_for', names, function(idx)
-  return 'method_handlers[' .. idx .. '].name'
-end)
-
-output:write('const MsgpackRpcRequestHandler method_handlers[] = {\n')
-for n, name in ipairs(hashorder) do
-  local fn = remote_fns[name]
-  fn.handler_id = n - 1
-  output:write(
-    '  { .name = "'
-      .. name
-      .. '", .fn = handle_'
-      .. (fn.impl_name or fn.name)
-      .. ', .fast = '
-      .. tostring(fn.fast)
-      .. ', .ret_alloc = '
-      .. tostring(not not fn.ret_alloc)
-      .. '},\n'
-  )
-end
-output:write('};\n\n')
-output:write(hashfun)
-
-output:close()
-
-functions.keysets = keysets
-local mpack_output = assert(io.open(mpack_outputf, 'wb'))
-mpack_output:write(mpack.encode(functions))
-mpack_output:close()
-
-local function include_headers(output_handle, headers_to_include)
-  for i = 1, #headers_to_include do
-    if headers_to_include[i]:sub(-12) ~= '.generated.h' then
-      output_handle:write('\n#include "nvim/' .. headers_to_include[i] .. '"')
-    end
-  end
-end
-
-local function write_shifted_output(str, ...)
-  str = str:gsub('\n  ', '\n')
-  str = str:gsub('^  ', '')
-  str = str:gsub(' +$', '')
-  output:write(string.format(str, ...))
-end
-
--- start building lua output
-output = assert(io.open(lua_c_bindings_outputf, 'wb'))
-
-include_headers(output, headers)
-output:write('\n')
-
-local lua_c_functions = {}
-
-local function process_function(fn)
-  local lua_c_function_name = ('nlua_api_%s'):format(fn.name)
-  write_shifted_output(
-    [[
-
-  static int %s(lua_State *lstate)
-  {
-    Error err = ERROR_INIT;
-    Arena arena = ARENA_EMPTY;
-    char *err_param = 0;
-    if (lua_gettop(lstate) != %i) {
-      api_set_error(&err, kErrorTypeValidation, "Expected %i argument%s");
-      goto exit_0;
-    }
-  ]],
-    lua_c_function_name,
-    #fn.parameters,
-    #fn.parameters,
-    (#fn.parameters == 1) and '' or 's'
-  )
-  lua_c_functions[#lua_c_functions + 1] = {
-    binding = lua_c_function_name,
-    api = fn.name,
-  }
-
-  if not fn.fast then
-    write_shifted_output(
-      [[
-    if (!nlua_is_deferred_safe()) {
-      return luaL_error(lstate, e_fast_api_disabled, "%s");
-    }
-    ]],
-      fn.name
-    )
-  end
-
-  if fn.textlock then
-    write_shifted_output([[
-    if (text_locked()) {
-      api_set_error(&err, kErrorTypeException, "%%s", get_text_locked_msg());
-      goto exit_0;
-    }
-    ]])
-  elseif fn.textlock_allow_cmdwin then
-    write_shifted_output([[
-    if (textlock != 0 || expr_map_locked()) {
-      api_set_error(&err, kErrorTypeException, "%%s", e_textlock);
-      goto exit_0;
-    }
-    ]])
-  end
-
-  local cparams = ''
-  local free_code = {}
-  for j = #fn.parameters, 1, -1 do
-    local param = fn.parameters[j]
-    local cparam = string.format('arg%u', j)
-    local param_type = real_type(param[1])
-    local extra = param_type == 'Dict' and 'false, ' or ''
-    local arg_free_code = ''
-    if param[1] == 'Object' then
-      extra = 'true, '
-      arg_free_code = '  api_luarefs_free_object(' .. cparam .. ');'
-    elseif param[1] == 'DictOf(LuaRef)' then
-      extra = 'true, '
-      arg_free_code = '  api_luarefs_free_dict(' .. cparam .. ');'
-    elseif param[1] == 'LuaRef' then
-      arg_free_code = '  api_free_luaref(' .. cparam .. ');'
-    end
-    local errshift = 0
-    local seterr = ''
-    if string.match(param_type, '^KeyDict_') then
-      write_shifted_output(
-        [[
-    %s %s = KEYDICT_INIT;
-    nlua_pop_keydict(lstate, &%s, %s_get_field, &err_param, &arena, &err);
-    ]],
-        param_type,
-        cparam,
-        cparam,
-        param_type
-      )
-      cparam = '&' .. cparam
-      errshift = 1 -- free incomplete dict on error
-      arg_free_code = '  api_luarefs_free_keydict('
-        .. cparam
-        .. ', '
-        .. string.sub(param_type, 9)
-        .. '_table);'
-    else
-      write_shifted_output(
-        [[
-    const %s %s = nlua_pop_%s(lstate, %s&arena, &err);]],
-        param[1],
-        cparam,
-        param_type,
-        extra
-      )
-      seterr = '\n      err_param = "' .. param[2] .. '";'
-    end
-
-    write_shifted_output([[
-
-    if (ERROR_SET(&err)) {]] .. seterr .. [[
-
-      goto exit_%u;
-    }
-
-    ]], #fn.parameters - j + errshift)
-    free_code[#free_code + 1] = arg_free_code
-    cparams = cparam .. ', ' .. cparams
-  end
-  if fn.receives_channel_id then
-    cparams = 'LUA_INTERNAL_CALL, ' .. cparams
-  end
-  if fn.receives_arena then
-    cparams = cparams .. '&arena, '
-  end
-
-  if fn.has_lua_imp then
-    cparams = cparams .. 'lstate, '
-  end
-
-  if fn.can_fail then
-    cparams = cparams .. '&err'
-  else
-    cparams = cparams:gsub(', $', '')
-  end
-  local free_at_exit_code = ''
-  for i = 1, #free_code do
-    local rev_i = #free_code - i + 1
-    local code = free_code[rev_i]
-    if i == 1 and not string.match(real_type(fn.parameters[1][1]), '^KeyDict_') then
-      free_at_exit_code = free_at_exit_code .. ('\n%s'):format(code)
-    else
-      free_at_exit_code = free_at_exit_code .. ('\nexit_%u:\n%s'):format(rev_i, code)
-    end
-  end
-  local err_throw_code = [[
-
-exit_0:
-  arena_mem_free(arena_finish(&arena));
-  if (ERROR_SET(&err)) {
-    luaL_where(lstate, 1);
-    if (err_param) {
-      lua_pushstring(lstate, "Invalid '");
-      lua_pushstring(lstate, err_param);
-      lua_pushstring(lstate, "': ");
-    }
-    lua_pushstring(lstate, err.msg);
-    api_clear_error(&err);
-    lua_concat(lstate, err_param ? 5 : 2);
-    return lua_error(lstate);
-  }
-]]
-  local return_type
-  if fn.return_type ~= 'void' then
-    if fn.return_type:match('^ArrayOf') then
-      return_type = 'Array'
-    else
-      return_type = fn.return_type
-    end
-    local free_retval = ''
-    if fn.ret_alloc then
-      free_retval = '  api_free_' .. return_type:lower() .. '(ret);'
-    end
-    write_shifted_output('    %s ret = %s(%s);\n', fn.return_type, fn.name, cparams)
-
-    local ret_type = real_type(fn.return_type)
-    local ret_mode = (ret_type == 'Object') and '&' or ''
-    if fn.has_lua_imp then
-      -- only push onto the Lua stack if we haven't already
-      write_shifted_output(
-        [[
-    if (lua_gettop(lstate) == 0) {
-      nlua_push_%s(lstate, %sret, kNluaPushSpecial | kNluaPushFreeRefs);
-    }
-      ]],
-        return_type,
-        ret_mode
-      )
-    elseif string.match(ret_type, '^KeyDict_') then
-      write_shifted_output(
-        '    nlua_push_keydict(lstate, &ret, %s_table);\n',
-        string.sub(ret_type, 9)
-      )
-    else
-      local special = (fn.since ~= nil and fn.since < 11)
-      write_shifted_output(
-        '    nlua_push_%s(lstate, %sret, %s | kNluaPushFreeRefs);\n',
-        return_type,
-        ret_mode,
-        special and 'kNluaPushSpecial' or '0'
-      )
-    end
-
-    -- NOTE: we currently assume err_throw needs nothing from arena
-    write_shifted_output(
-      [[
-  %s
-  %s
-  %s
-    return 1;
-    ]],
-      free_retval,
-      free_at_exit_code,
-      err_throw_code
-    )
-  else
-    write_shifted_output(
-      [[
-    %s(%s);
-  %s
-  %s
-    return 0;
-    ]],
-      fn.name,
-      cparams,
-      free_at_exit_code,
-      err_throw_code
-    )
-  end
-  write_shifted_output([[
-  }
-  ]])
-end
-
-for _, fn in ipairs(functions) do
-  if fn.lua or fn.name:sub(1, 4) == '_vim' then
-    process_function(fn)
-  end
-end
-
-output:write(string.format(
-  [[
-void nlua_add_api_functions(lua_State *lstate)
-{
-  lua_createtable(lstate, 0, %u);
-]],
-  #lua_c_functions
-))
-for _, func in ipairs(lua_c_functions) do
-  output:write(string.format(
-    [[
-
-  lua_pushcfunction(lstate, &%s);
-  lua_setfield(lstate, -2, "%s");]],
-    func.binding,
-    func.api
-  ))
-end
-output:write([[
-
-  lua_setfield(lstate, -2, "api");
-}
-]])
-
-output:close()
-keysets_defs:close()
diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/nvim/generators/gen_api_ui_events.lua
deleted file mode 100644
index a3bb76cb91..0000000000
--- a/src/nvim/generators/gen_api_ui_events.lua
+++ /dev/null
@@ -1,219 +0,0 @@
-local mpack = vim.mpack
-
-assert(#arg == 5)
-local input = io.open(arg[1], 'rb')
-local call_output = io.open(arg[2], 'wb')
-local remote_output = io.open(arg[3], 'wb')
-local metadata_output = io.open(arg[4], 'wb')
-local client_output = io.open(arg[5], 'wb')
-
-local c_grammar = require('generators.c_grammar')
-local events = c_grammar.grammar:match(input:read('*all'))
-
-local hashy = require 'generators.hashy'
-
-local function write_signature(output, ev, prefix, notype)
-  output:write('(' .. prefix)
-  if prefix == '' and #ev.parameters == 0 then
-    output:write('void')
-  end
-  for j = 1, #ev.parameters do
-    if j > 1 or prefix ~= '' then
-      output:write(', ')
-    end
-    local param = ev.parameters[j]
-    if not notype then
-      output:write(param[1] .. ' ')
-    end
-    output:write(param[2])
-  end
-  output:write(')')
-end
-
-local function write_arglist(output, ev)
-  if #ev.parameters == 0 then
-    return
-  end
-  output:write('  MAXSIZE_TEMP_ARRAY(args, ' .. #ev.parameters .. ');\n')
-  for j = 1, #ev.parameters do
-    local param = ev.parameters[j]
-    local kind = string.upper(param[1])
-    output:write('  ADD_C(args, ')
-    output:write(kind .. '_OBJ(' .. param[2] .. ')')
-    output:write(');\n')
-  end
-end
-
-local function call_ui_event_method(output, ev)
-  output:write('void ui_client_event_' .. ev.name .. '(Array args)\n{\n')
-
-  local hlattrs_args_count = 0
-  if #ev.parameters > 0 then
-    output:write('  if (args.size < ' .. #ev.parameters)
-    for j = 1, #ev.parameters do
-      local kind = ev.parameters[j][1]
-      if kind ~= 'Object' then
-        if kind == 'HlAttrs' then
-          kind = 'Dict'
-        end
-        output:write('\n      || args.items[' .. (j - 1) .. '].type != kObjectType' .. kind .. '')
-      end
-    end
-    output:write(') {\n')
-    output:write('    ELOG("Error handling ui event \'' .. ev.name .. '\'");\n')
-    output:write('    return;\n')
-    output:write('  }\n')
-  end
-
-  for j = 1, #ev.parameters do
-    local param = ev.parameters[j]
-    local kind = param[1]
-    output:write('  ' .. kind .. ' arg_' .. j .. ' = ')
-    if kind == 'HlAttrs' then
-      -- The first HlAttrs argument is rgb_attrs and second is cterm_attrs
-      output:write(
-        'ui_client_dict2hlattrs(args.items['
-          .. (j - 1)
-          .. '].data.dict, '
-          .. (hlattrs_args_count == 0 and 'true' or 'false')
-          .. ');\n'
-      )
-      hlattrs_args_count = hlattrs_args_count + 1
-    elseif kind == 'Object' then
-      output:write('args.items[' .. (j - 1) .. '];\n')
-    elseif kind == 'Window' then
-      output:write('(Window)args.items[' .. (j - 1) .. '].data.integer;\n')
-    else
-      output:write('args.items[' .. (j - 1) .. '].data.' .. string.lower(kind) .. ';\n')
-    end
-  end
-
-  output:write('  tui_' .. ev.name .. '(tui')
-  for j = 1, #ev.parameters do
-    output:write(', arg_' .. j)
-  end
-  output:write(');\n')
-
-  output:write('}\n\n')
-end
-
-events = vim.tbl_filter(function(ev)
-  return ev[1] ~= 'empty' and ev[1] ~= 'preproc'
-end, events)
-
-for i = 1, #events do
-  local ev = events[i]
-  assert(ev.return_type == 'void')
-
-  if ev.since == nil and not ev.noexport then
-    print('Ui event ' .. ev.name .. ' lacks since field.\n')
-    os.exit(1)
-  end
-  ev.since = tonumber(ev.since)
-
-  local args = #ev.parameters > 0 and 'args' or 'noargs'
-  if not ev.remote_only then
-    if not ev.remote_impl and not ev.noexport then
-      remote_output:write('void remote_ui_' .. ev.name)
-      write_signature(remote_output, ev, 'RemoteUI *ui')
-      remote_output:write('\n{\n')
-      write_arglist(remote_output, ev)
-      remote_output:write('  push_call(ui, "' .. ev.name .. '", ' .. args .. ');\n')
-      remote_output:write('}\n\n')
-    end
-  end
-
-  if not (ev.remote_only and ev.remote_impl) then
-    call_output:write('void ui_call_' .. ev.name)
-    write_signature(call_output, ev, '')
-    call_output:write('\n{\n')
-    if ev.remote_only then
-      -- Lua callbacks may emit other events or the same event again. Avoid the latter
-      -- by adding a recursion guard to each generated function that may call a Lua callback.
-      call_output:write('  static bool entered = false;\n')
-      call_output:write('  if (entered) {\n')
-      call_output:write('    return;\n')
-      call_output:write('  }\n')
-      call_output:write('  entered = true;\n')
-      write_arglist(call_output, ev)
-      call_output:write(('  ui_call_event("%s", %s, %s)'):format(ev.name, tostring(ev.fast), args))
-      call_output:write(';\n  entered = false;\n')
-    elseif ev.compositor_impl then
-      call_output:write('  ui_comp_' .. ev.name)
-      write_signature(call_output, ev, '', true)
-      call_output:write(';\n')
-      call_output:write('  UI_CALL')
-      write_signature(call_output, ev, '!ui->composed, ' .. ev.name .. ', ui', true)
-      call_output:write(';\n')
-    else
-      call_output:write('  UI_CALL')
-      write_signature(call_output, ev, 'true, ' .. ev.name .. ', ui', true)
-      call_output:write(';\n')
-    end
-    call_output:write('}\n\n')
-  end
-
-  if ev.compositor_impl then
-    call_output:write('void ui_composed_call_' .. ev.name)
-    write_signature(call_output, ev, '')
-    call_output:write('\n{\n')
-    call_output:write('  UI_CALL')
-    write_signature(call_output, ev, 'ui->composed, ' .. ev.name .. ', ui', true)
-    call_output:write(';\n')
-    call_output:write('}\n\n')
-  end
-
-  if (not ev.remote_only) and not ev.noexport and not ev.client_impl and not ev.client_ignore then
-    call_ui_event_method(client_output, ev)
-  end
-end
-
-local client_events = {}
-for _, ev in ipairs(events) do
-  if (not ev.noexport) and ((not ev.remote_only) or ev.client_impl) and not ev.client_ignore then
-    client_events[ev.name] = ev
-  end
-end
-
-local hashorder, hashfun = hashy.hashy_hash(
-  'ui_client_handler',
-  vim.tbl_keys(client_events),
-  function(idx)
-    return 'event_handlers[' .. idx .. '].name'
-  end
-)
-
-client_output:write('static const UIClientHandler event_handlers[] = {\n')
-
-for _, name in ipairs(hashorder) do
-  client_output:write('  { .name = "' .. name .. '", .fn = ui_client_event_' .. name .. '},\n')
-end
-
-client_output:write('\n};\n\n')
-client_output:write(hashfun)
-
-call_output:close()
-remote_output:close()
-client_output:close()
-
--- don't expose internal attributes like "impl_name" in public metadata
-local exported_attributes = { 'name', 'parameters', 'since', 'deprecated_since' }
-local exported_events = {}
-for _, ev in ipairs(events) do
-  local ev_exported = {}
-  for _, attr in ipairs(exported_attributes) do
-    ev_exported[attr] = ev[attr]
-  end
-  for _, p in ipairs(ev_exported.parameters) do
-    if p[1] == 'HlAttrs' or p[1] == 'Dict' then
-      -- TODO(justinmk): for back-compat, but do clients actually look at this?
-      p[1] = 'Dictionary'
-    end
-  end
-  if not ev.noexport then
-    exported_events[#exported_events + 1] = ev_exported
-  end
-end
-
-metadata_output:write(mpack.encode(exported_events))
-metadata_output:close()
diff --git a/src/nvim/generators/gen_char_blob.lua b/src/nvim/generators/gen_char_blob.lua
deleted file mode 100644
index c40e0d6e82..0000000000
--- a/src/nvim/generators/gen_char_blob.lua
+++ /dev/null
@@ -1,96 +0,0 @@
-if arg[1] == '--help' then
-  print('Usage:')
-  print('  ' .. arg[0] .. ' [-c] target source varname [source varname]...')
-  print('')
-  print('Generates C file with big uint8_t blob.')
-  print('Blob will be stored in a static const array named varname.')
-  os.exit()
-end
-
--- Recognized options:
---   -c   compile Lua bytecode
-local options = {}
-
-while true do
-  local opt = string.match(arg[1], '^-(%w)')
-  if not opt then
-    break
-  end
-
-  options[opt] = true
-  table.remove(arg, 1)
-end
-
-assert(#arg >= 3 and (#arg - 1) % 2 == 0)
-
-local target_file = arg[1] or error('Need a target file')
-local target = io.open(target_file, 'w')
-
-target:write('#include \n\n')
-
-local index_items = {}
-
-local warn_on_missing_compiler = true
-local modnames = {}
-for argi = 2, #arg, 2 do
-  local source_file = arg[argi]
-  local modname = arg[argi + 1]
-  if modnames[modname] then
-    error(string.format('modname %q is already specified for file %q', modname, modnames[modname]))
-  end
-  modnames[modname] = source_file
-
-  local varname = string.gsub(modname, '%.', '_dot_') .. '_module'
-  target:write(('static const uint8_t %s[] = {\n'):format(varname))
-
-  local output
-  if options.c then
-    local luac = os.getenv('LUAC_PRG')
-    if luac and luac ~= '' then
-      output = io.popen(luac:format(source_file), 'r'):read('*a')
-    elseif warn_on_missing_compiler then
-      print('LUAC_PRG is missing, embedding raw source')
-      warn_on_missing_compiler = false
-    end
-  end
-
-  if not output then
-    local source = io.open(source_file, 'r')
-      or error(string.format("source_file %q doesn't exist", source_file))
-    output = source:read('*a')
-    source:close()
-  end
-
-  local num_bytes = 0
-  local MAX_NUM_BYTES = 15 -- 78 / 5: maximum number of bytes on one line
-  target:write(' ')
-
-  local increase_num_bytes
-  increase_num_bytes = function()
-    num_bytes = num_bytes + 1
-    if num_bytes == MAX_NUM_BYTES then
-      num_bytes = 0
-      target:write('\n ')
-    end
-  end
-
-  for i = 1, string.len(output) do
-    local byte = output:byte(i)
-    target:write(string.format(' %3u,', byte))
-    increase_num_bytes()
-  end
-
-  target:write('  0};\n')
-  if modname ~= '_' then
-    table.insert(
-      index_items,
-      '  { "' .. modname .. '", ' .. varname .. ', sizeof ' .. varname .. ' },\n\n'
-    )
-  end
-end
-
-target:write('static ModuleDef builtin_modules[] = {\n')
-target:write(table.concat(index_items))
-target:write('};\n')
-
-target:close()
diff --git a/src/nvim/generators/gen_declarations.lua b/src/nvim/generators/gen_declarations.lua
deleted file mode 100644
index 6e1ea92572..0000000000
--- a/src/nvim/generators/gen_declarations.lua
+++ /dev/null
@@ -1,186 +0,0 @@
-local grammar = require('generators.c_grammar').grammar
-
---- @param fname string
---- @return string?
-local function read_file(fname)
-  local f = io.open(fname, 'r')
-  if not f then
-    return
-  end
-  local contents = f:read('*a')
-  f:close()
-  return contents
-end
-
---- @param fname string
---- @param contents string[]
-local function write_file(fname, contents)
-  local contents_s = table.concat(contents, '\n') .. '\n'
-  local fcontents = read_file(fname)
-  if fcontents == contents_s then
-    return
-  end
-  local f = assert(io.open(fname, 'w'))
-  f:write(contents_s)
-  f:close()
-end
-
---- @param fname string
---- @param non_static_fname string
---- @return string? non_static
-local function add_iwyu_non_static(fname, non_static_fname)
-  if fname:find('.*/src/nvim/.*%.c$') then
-    -- Add an IWYU pragma comment if the corresponding .h file exists.
-    local header_fname = fname:sub(1, -3) .. '.h'
-    local header_f = io.open(header_fname, 'r')
-    if header_f then
-      header_f:close()
-      return (header_fname:gsub('.*/src/nvim/', 'nvim/'))
-    end
-  elseif non_static_fname:find('/include/api/private/dispatch_wrappers%.h%.generated%.h$') then
-    return 'nvim/api/private/dispatch.h'
-  elseif non_static_fname:find('/include/ui_events_call%.h%.generated%.h$') then
-    return 'nvim/ui.h'
-  elseif non_static_fname:find('/include/ui_events_client%.h%.generated%.h$') then
-    return 'nvim/ui_client.h'
-  elseif non_static_fname:find('/include/ui_events_remote%.h%.generated%.h$') then
-    return 'nvim/api/ui.h'
-  end
-end
-
---- @param d string
-local function process_decl(d)
-  -- Comments are really handled by preprocessor, so the following is not
-  -- needed
-  d = d:gsub('/%*.-%*/', '')
-  d = d:gsub('//.-\n', '\n')
-  d = d:gsub('# .-\n', '')
-  d = d:gsub('\n', ' ')
-  d = d:gsub('%s+', ' ')
-  d = d:gsub(' ?%( ?', '(')
-  d = d:gsub(' ?, ?', ', ')
-  d = d:gsub(' ?(%*+) ?', ' %1')
-  d = d:gsub(' ?(FUNC_ATTR_)', ' %1')
-  d = d:gsub(' $', '')
-  d = d:gsub('^ ', '')
-  return d .. ';'
-end
-
---- @param fname string
---- @param text string
---- @return string[] static
---- @return string[] non_static
---- @return boolean any_static
-local function gen_declarations(fname, text)
-  local non_static = {} --- @type string[]
-  local static = {} --- @type string[]
-
-  local neededfile = fname:match('[^/]+$')
-  local curfile = nil
-  local any_static = false
-  for _, node in ipairs(grammar:match(text)) do
-    if node[1] == 'preproc' then
-      curfile = node.content:match('^%a* %d+ "[^"]-/?([^"/]+)"') or curfile
-    elseif node[1] == 'proto' and curfile == neededfile then
-      local node_text = text:sub(node.pos, node.endpos - 1)
-      local declaration = process_decl(node_text)
-
-      if node.static then
-        if not any_static and declaration:find('FUNC_ATTR_') then
-          any_static = true
-        end
-        static[#static + 1] = declaration
-      else
-        non_static[#non_static + 1] = 'DLLEXPORT ' .. declaration
-      end
-    end
-  end
-
-  return static, non_static, any_static
-end
-
-local usage = [[
-Usage:
-
-    gen_declarations.lua definitions.c static.h non-static.h definitions.i
-
-Generates declarations for a C file definitions.c, putting declarations for
-static functions into static.h and declarations for non-static functions into
-non-static.h. File `definitions.i' should contain an already preprocessed
-version of definitions.c and it is the only one which is actually parsed,
-definitions.c is needed only to determine functions from which file out of all
-functions found in definitions.i are needed and to generate an IWYU comment.
-]]
-
-local function main()
-  local fname = arg[1]
-  local static_fname = arg[2]
-  local non_static_fname = arg[3]
-  local preproc_fname = arg[4]
-  local static_basename = arg[5]
-
-  if fname == '--help' or #arg < 5 then
-    print(usage)
-    os.exit()
-  end
-
-  local text = assert(read_file(preproc_fname))
-
-  local static_decls, non_static_decls, any_static = gen_declarations(fname, text)
-
-  local static = {} --- @type string[]
-  if fname:find('.*/src/nvim/.*%.h$') then
-    static[#static + 1] = ('// IWYU pragma: private, include "%s"'):format(
-      fname:gsub('.*/src/nvim/', 'nvim/')
-    )
-  end
-  vim.list_extend(static, {
-    '#define DEFINE_FUNC_ATTRIBUTES',
-    '#include "nvim/func_attr.h"',
-    '#undef DEFINE_FUNC_ATTRIBUTES',
-  })
-  vim.list_extend(static, static_decls)
-  vim.list_extend(static, {
-    '#define DEFINE_EMPTY_ATTRIBUTES',
-    '#include "nvim/func_attr.h"  // IWYU pragma: export',
-    '',
-  })
-
-  write_file(static_fname, static)
-
-  if any_static then
-    local orig_text = assert(read_file(fname))
-    local pat = '\n#%s?include%s+"' .. static_basename .. '"\n'
-    local pat_comment = '\n#%s?include%s+"' .. static_basename .. '"%s*//'
-    if not orig_text:find(pat) and not orig_text:find(pat_comment) then
-      error(('fail: missing include for %s in %s'):format(static_basename, fname))
-    end
-  end
-
-  if non_static_fname ~= 'SKIP' then
-    local non_static = {} --- @type string[]
-    local iwyu_non_static = add_iwyu_non_static(fname, non_static_fname)
-    if iwyu_non_static then
-      non_static[#non_static + 1] = ('// IWYU pragma: private, include "%s"'):format(
-        iwyu_non_static
-      )
-    end
-    vim.list_extend(non_static, {
-      '#define DEFINE_FUNC_ATTRIBUTES',
-      '#include "nvim/func_attr.h"',
-      '#undef DEFINE_FUNC_ATTRIBUTES',
-      '#ifndef DLLEXPORT',
-      '#  ifdef MSWIN',
-      '#    define DLLEXPORT __declspec(dllexport)',
-      '#  else',
-      '#    define DLLEXPORT',
-      '#  endif',
-      '#endif',
-    })
-    vim.list_extend(non_static, non_static_decls)
-    non_static[#non_static + 1] = '#include "nvim/func_attr.h"'
-    write_file(non_static_fname, non_static)
-  end
-end
-
-return main()
diff --git a/src/nvim/generators/gen_eval.lua b/src/nvim/generators/gen_eval.lua
deleted file mode 100644
index 0b6ee6cb24..0000000000
--- a/src/nvim/generators/gen_eval.lua
+++ /dev/null
@@ -1,112 +0,0 @@
-local mpack = vim.mpack
-
-local autodir = arg[1]
-local metadata_file = arg[2]
-local funcs_file = arg[3]
-
-local funcsfname = autodir .. '/funcs.generated.h'
-
---Will generate funcs.generated.h with definition of functions static const array.
-
-local hashy = require 'generators.hashy'
-
-local hashpipe = assert(io.open(funcsfname, 'wb'))
-
-hashpipe:write([[
-#include "nvim/arglist.h"
-#include "nvim/cmdexpand.h"
-#include "nvim/cmdhist.h"
-#include "nvim/digraph.h"
-#include "nvim/eval.h"
-#include "nvim/eval/buffer.h"
-#include "nvim/eval/deprecated.h"
-#include "nvim/eval/fs.h"
-#include "nvim/eval/funcs.h"
-#include "nvim/eval/typval.h"
-#include "nvim/eval/vars.h"
-#include "nvim/eval/window.h"
-#include "nvim/ex_docmd.h"
-#include "nvim/ex_getln.h"
-#include "nvim/fold.h"
-#include "nvim/getchar.h"
-#include "nvim/insexpand.h"
-#include "nvim/mapping.h"
-#include "nvim/match.h"
-#include "nvim/mbyte.h"
-#include "nvim/menu.h"
-#include "nvim/mouse.h"
-#include "nvim/move.h"
-#include "nvim/quickfix.h"
-#include "nvim/runtime.h"
-#include "nvim/search.h"
-#include "nvim/state.h"
-#include "nvim/strings.h"
-#include "nvim/sign.h"
-#include "nvim/testing.h"
-#include "nvim/undo.h"
-
-]])
-
-local funcs = require('eval').funcs
-for _, func in pairs(funcs) do
-  if func.float_func then
-    func.func = 'float_op_wrapper'
-    func.data = '{ .float_func = &' .. func.float_func .. ' }'
-  end
-end
-
-local metadata = mpack.decode(io.open(metadata_file, 'rb'):read('*all'))
-for _, fun in ipairs(metadata) do
-  if fun.eval then
-    funcs[fun.name] = {
-      args = #fun.parameters,
-      func = 'api_wrapper',
-      data = '{ .api_handler = &method_handlers[' .. fun.handler_id .. '] }',
-    }
-  end
-end
-
-local func_names = vim.tbl_filter(function(name)
-  return name:match('__%d*$') == nil
-end, vim.tbl_keys(funcs))
-
-table.sort(func_names)
-
-local funcsdata = assert(io.open(funcs_file, 'w'))
-funcsdata:write(mpack.encode(func_names))
-funcsdata:close()
-
-local neworder, hashfun = hashy.hashy_hash('find_internal_func', func_names, function(idx)
-  return 'functions[' .. idx .. '].name'
-end)
-
-hashpipe:write('static const EvalFuncDef functions[] = {\n')
-
-for _, name in ipairs(neworder) do
-  local def = funcs[name]
-  local args = def.args or 0
-  if type(args) == 'number' then
-    args = { args, args }
-  elseif #args == 1 then
-    args[2] = 'MAX_FUNC_ARGS'
-  end
-  local base = def.base or 'BASE_NONE'
-  local func = def.func or ('f_' .. name)
-  local data = def.data or '{ .null = NULL }'
-  local fast = def.fast and 'true' or 'false'
-  hashpipe:write(
-    ('  { "%s", %s, %s, %s, %s, &%s, %s },\n'):format(
-      name,
-      args[1],
-      args[2],
-      base,
-      fast,
-      func,
-      data
-    )
-  )
-end
-hashpipe:write('  { NULL, 0, 0, BASE_NONE, false, NULL, { .null = NULL } },\n')
-hashpipe:write('};\n\n')
-hashpipe:write(hashfun)
-hashpipe:close()
diff --git a/src/nvim/generators/gen_events.lua b/src/nvim/generators/gen_events.lua
deleted file mode 100644
index 8c87815a74..0000000000
--- a/src/nvim/generators/gen_events.lua
+++ /dev/null
@@ -1,42 +0,0 @@
-local fileio_enum_file = arg[1]
-local names_file = arg[2]
-
-local auevents = require('auevents')
-local events = auevents.events
-
-local enum_tgt = io.open(fileio_enum_file, 'w')
-local names_tgt = io.open(names_file, 'w')
-
-enum_tgt:write([[
-// IWYU pragma: private, include "nvim/autocmd_defs.h"
-
-typedef enum auto_event {]])
-names_tgt:write([[
-static const struct event_name {
-  size_t len;
-  char *name;
-  int event;
-} event_names[] = {]])
-
-local aliases = 0
-for i, event in ipairs(events) do
-  enum_tgt:write(('\n  EVENT_%s = %u,'):format(event[1]:upper(), i + aliases - 1))
-  -- Events with positive keys aren't allowed in 'eventignorewin'.
-  local event_int = ('%sEVENT_%s'):format(event[3] and '-' or '', event[1]:upper())
-  names_tgt:write(('\n  {%u, "%s", %s},'):format(#event[1], event[1], event_int))
-  for _, alias in ipairs(event[2]) do
-    aliases = aliases + 1
-    names_tgt:write(('\n  {%u, "%s", %s},'):format(#alias, alias, event_int))
-    enum_tgt:write(('\n  EVENT_%s = %u,'):format(alias:upper(), i + aliases - 1))
-  end
-  if i == #events then -- Last item.
-    enum_tgt:write(('\n  NUM_EVENTS = %u,'):format(i + aliases))
-  end
-end
-
-names_tgt:write('\n  {0, NULL, (event_T)0},\n};\n')
-names_tgt:write('\nstatic AutoCmdVec autocmds[NUM_EVENTS] = { 0 };\n')
-names_tgt:close()
-
-enum_tgt:write('\n} event_T;\n')
-enum_tgt:close()
diff --git a/src/nvim/generators/gen_ex_cmds.lua b/src/nvim/generators/gen_ex_cmds.lua
deleted file mode 100644
index e8d1aac182..0000000000
--- a/src/nvim/generators/gen_ex_cmds.lua
+++ /dev/null
@@ -1,194 +0,0 @@
-local includedir = arg[1]
-local autodir = arg[2]
-
--- Will generate files ex_cmds_enum.generated.h with cmdidx_T enum
--- and ex_cmds_defs.generated.h with main Ex commands definitions.
-
-local enumfname = includedir .. '/ex_cmds_enum.generated.h'
-local defsfname = autodir .. '/ex_cmds_defs.generated.h'
-
-local enumfile = io.open(enumfname, 'w')
-local defsfile = io.open(defsfname, 'w')
-
-local bit = require 'bit'
-local ex_cmds = require('ex_cmds')
-local defs = ex_cmds.cmds
-local flags = ex_cmds.flags
-
-local byte_a = string.byte('a')
-local byte_z = string.byte('z')
-local a_to_z = byte_z - byte_a + 1
-
--- Table giving the index of the first command in cmdnames[] to lookup
--- based on the first letter of a command.
-local cmdidxs1_out = string.format(
-  [[
-static const uint16_t cmdidxs1[%u] = {
-]],
-  a_to_z
-)
--- Table giving the index of the first command in cmdnames[] to lookup
--- based on the first 2 letters of a command.
--- Values in cmdidxs2[c1][c2] are relative to cmdidxs1[c1] so that they
--- fit in a byte.
-local cmdidxs2_out = string.format(
-  [[
-static const uint8_t cmdidxs2[%u][%u] = {
-  /*           a   b   c   d   e   f   g   h   i   j   k   l   m   n   o   p   q   r   s   t   u   v   w   x   y   z */
-]],
-  a_to_z,
-  a_to_z
-)
-
-enumfile:write([[
-// IWYU pragma: private, include "nvim/ex_cmds_defs.h"
-
-typedef enum CMD_index {
-]])
-defsfile:write(string.format(
-  [[
-#include "nvim/arglist.h"
-#include "nvim/autocmd.h"
-#include "nvim/buffer.h"
-#include "nvim/cmdhist.h"
-#include "nvim/debugger.h"
-#include "nvim/diff.h"
-#include "nvim/digraph.h"
-#include "nvim/eval.h"
-#include "nvim/eval/userfunc.h"
-#include "nvim/eval/vars.h"
-#include "nvim/ex_cmds.h"
-#include "nvim/ex_cmds2.h"
-#include "nvim/ex_docmd.h"
-#include "nvim/ex_eval.h"
-#include "nvim/ex_session.h"
-#include "nvim/help.h"
-#include "nvim/indent.h"
-#include "nvim/lua/executor.h"
-#include "nvim/lua/secure.h"
-#include "nvim/mapping.h"
-#include "nvim/mark.h"
-#include "nvim/match.h"
-#include "nvim/menu.h"
-#include "nvim/message.h"
-#include "nvim/ops.h"
-#include "nvim/option.h"
-#include "nvim/os/lang.h"
-#include "nvim/profile.h"
-#include "nvim/quickfix.h"
-#include "nvim/runtime.h"
-#include "nvim/sign.h"
-#include "nvim/spell.h"
-#include "nvim/spellfile.h"
-#include "nvim/syntax.h"
-#include "nvim/undo.h"
-#include "nvim/usercmd.h"
-#include "nvim/version.h"
-
-static const int command_count = %u;
-static CommandDefinition cmdnames[%u] = {
-]],
-  #defs,
-  #defs
-))
-local cmds, cmdidxs1, cmdidxs2 = {}, {}, {}
-for _, cmd in ipairs(defs) do
-  if bit.band(cmd.flags, flags.RANGE) == flags.RANGE then
-    assert(
-      cmd.addr_type ~= 'ADDR_NONE',
-      string.format('ex_cmds.lua:%s: Using RANGE with ADDR_NONE\n', cmd.command)
-    )
-  else
-    assert(
-      cmd.addr_type == 'ADDR_NONE',
-      string.format('ex_cmds.lua:%s: Missing ADDR_NONE\n', cmd.command)
-    )
-  end
-  if bit.band(cmd.flags, flags.DFLALL) == flags.DFLALL then
-    assert(
-      cmd.addr_type ~= 'ADDR_OTHER' and cmd.addr_type ~= 'ADDR_NONE',
-      string.format('ex_cmds.lua:%s: Missing misplaced DFLALL\n', cmd.command)
-    )
-  end
-  if bit.band(cmd.flags, flags.PREVIEW) == flags.PREVIEW then
-    assert(
-      cmd.preview_func ~= nil,
-      string.format('ex_cmds.lua:%s: Missing preview_func\n', cmd.command)
-    )
-  end
-  local enumname = cmd.enum or ('CMD_' .. cmd.command)
-  local byte_cmd = cmd.command:sub(1, 1):byte()
-  if byte_a <= byte_cmd and byte_cmd <= byte_z then
-    table.insert(cmds, cmd.command)
-  end
-  local preview_func
-  if cmd.preview_func then
-    preview_func = string.format('&%s', cmd.preview_func)
-  else
-    preview_func = 'NULL'
-  end
-  enumfile:write('  ' .. enumname .. ',\n')
-  defsfile:write(string.format(
-    [[
-  [%s] = {
-    .cmd_name = "%s",
-    .cmd_func = (ex_func_T)&%s,
-    .cmd_preview_func = %s,
-    .cmd_argt = %uL,
-    .cmd_addr_type = %s
-  },
-]],
-    enumname,
-    cmd.command,
-    cmd.func,
-    preview_func,
-    cmd.flags,
-    cmd.addr_type
-  ))
-end
-for i = #cmds, 1, -1 do
-  local cmd = cmds[i]
-  -- First and second characters of the command
-  local c1 = cmd:sub(1, 1)
-  cmdidxs1[c1] = i - 1
-  if cmd:len() >= 2 then
-    local c2 = cmd:sub(2, 2)
-    local byte_c2 = string.byte(c2)
-    if byte_a <= byte_c2 and byte_c2 <= byte_z then
-      if not cmdidxs2[c1] then
-        cmdidxs2[c1] = {}
-      end
-      cmdidxs2[c1][c2] = i - 1
-    end
-  end
-end
-for i = byte_a, byte_z do
-  local c1 = string.char(i)
-  cmdidxs1_out = cmdidxs1_out .. '  /*  ' .. c1 .. '  */ ' .. cmdidxs1[c1] .. ',\n'
-  cmdidxs2_out = cmdidxs2_out .. '  /*  ' .. c1 .. '  */ {'
-  for j = byte_a, byte_z do
-    local c2 = string.char(j)
-    cmdidxs2_out = cmdidxs2_out
-      .. ((cmdidxs2[c1] and cmdidxs2[c1][c2]) and string.format(
-        '%3d',
-        cmdidxs2[c1][c2] - cmdidxs1[c1]
-      ) or '  0')
-      .. ','
-  end
-  cmdidxs2_out = cmdidxs2_out .. ' },\n'
-end
-enumfile:write([[
-  CMD_SIZE,
-  CMD_USER = -1,
-  CMD_USER_BUF = -2
-} cmdidx_T;
-]])
-defsfile:write(string.format(
-  [[
-};
-%s};
-%s};
-]],
-  cmdidxs1_out,
-  cmdidxs2_out
-))
diff --git a/src/nvim/generators/gen_options.lua b/src/nvim/generators/gen_options.lua
deleted file mode 100644
index e5dba90925..0000000000
--- a/src/nvim/generators/gen_options.lua
+++ /dev/null
@@ -1,535 +0,0 @@
---- @module 'nvim.options'
-local options = require('options')
-local options_meta = options.options
-local cstr = options.cstr
-local valid_scopes = options.valid_scopes
-
---- @param o vim.option_meta
---- @return string
-local function get_values_var(o)
-  return ('opt_%s_values'):format(o.abbreviation or o.full_name)
-end
-
---- @param s string
---- @return string
-local function lowercase_to_titlecase(s)
-  return table.concat(vim.tbl_map(function(word) --- @param word string
-    return word:sub(1, 1):upper() .. word:sub(2)
-  end, vim.split(s, '[-_]')))
-end
-
---- @param scope string
---- @param option_name string
---- @return string
-local function get_scope_option(scope, option_name)
-  return ('k%sOpt%s'):format(lowercase_to_titlecase(scope), lowercase_to_titlecase(option_name))
-end
-
-local redraw_flags = {
-  ui_option = 'kOptFlagUIOption',
-  tabline = 'kOptFlagRedrTabl',
-  statuslines = 'kOptFlagRedrStat',
-  current_window = 'kOptFlagRedrWin',
-  current_buffer = 'kOptFlagRedrBuf',
-  all_windows = 'kOptFlagRedrAll',
-  curswant = 'kOptFlagCurswant',
-  highlight_only = 'kOptFlagHLOnly',
-}
-
-local list_flags = {
-  comma = 'kOptFlagComma',
-  onecomma = 'kOptFlagOneComma',
-  commacolon = 'kOptFlagComma|kOptFlagColon',
-  onecommacolon = 'kOptFlagOneComma|kOptFlagColon',
-  flags = 'kOptFlagFlagList',
-  flagscomma = 'kOptFlagComma|kOptFlagFlagList',
-}
-
---- @param o vim.option_meta
---- @return string
-local function get_flags(o)
-  --- @type string[]
-  local flags = { '0' }
-
-  --- @param f string
-  local function add_flag(f)
-    table.insert(flags, f)
-  end
-
-  if o.list then
-    add_flag(list_flags[o.list])
-  end
-
-  for _, r_flag in ipairs(o.redraw or {}) do
-    add_flag(redraw_flags[r_flag])
-  end
-
-  if o.expand then
-    add_flag('kOptFlagExpand')
-    if o.expand == 'nodefault' then
-      add_flag('kOptFlagNoDefExp')
-    end
-  end
-
-  for _, flag_desc in ipairs({
-    { 'nodefault', 'NoDefault' },
-    { 'no_mkrc', 'NoMkrc' },
-    { 'secure' },
-    { 'gettext' },
-    { 'noglob', 'NoGlob' },
-    { 'normal_fname_chars', 'NFname' },
-    { 'normal_dname_chars', 'NDname' },
-    { 'pri_mkrc', 'PriMkrc' },
-    { 'deny_in_modelines', 'NoML' },
-    { 'deny_duplicates', 'NoDup' },
-    { 'modelineexpr', 'MLE' },
-    { 'func' },
-  }) do
-    local key_name, flag_suffix = flag_desc[1], flag_desc[2]
-    if o[key_name] then
-      local def_name = 'kOptFlag' .. (flag_suffix or lowercase_to_titlecase(key_name))
-      add_flag(def_name)
-    end
-  end
-
-  return table.concat(flags, '|')
-end
-
---- @param opt_type vim.option_type
---- @return string
-local function opt_type_enum(opt_type)
-  return ('kOptValType%s'):format(lowercase_to_titlecase(opt_type))
-end
-
---- @param scope vim.option_scope
---- @return string
-local function opt_scope_enum(scope)
-  return ('kOptScope%s'):format(lowercase_to_titlecase(scope))
-end
-
---- @param o vim.option_meta
---- @return string
-local function get_scope_flags(o)
-  local scope_flags = '0'
-
-  for _, scope in ipairs(o.scope) do
-    scope_flags = ('%s | (1 << %s)'):format(scope_flags, opt_scope_enum(scope))
-  end
-
-  return scope_flags
-end
-
---- @param o vim.option_meta
---- @return string
-local function get_scope_idx(o)
-  --- @type string[]
-  local strs = {}
-
-  for _, scope in pairs(valid_scopes) do
-    local has_scope = vim.tbl_contains(o.scope, scope)
-    strs[#strs + 1] = ('      [%s] = %s'):format(
-      opt_scope_enum(scope),
-      get_scope_option(scope, has_scope and o.full_name or 'Invalid')
-    )
-  end
-
-  return ('{\n%s\n    }'):format(table.concat(strs, ',\n'))
-end
-
---- @param s string
---- @return string
-local function static_cstr_as_string(s)
-  return ('{ .data = %s, .size = sizeof(%s) - 1 }'):format(s, s)
-end
-
---- @param v vim.option_value|function
---- @return string
-local function get_opt_val(v)
-  --- @type vim.option_type
-  local v_type
-
-  if type(v) == 'function' then
-    v, v_type = v() --[[ @as string, vim.option_type ]]
-
-    if v_type == 'string' then
-      v = static_cstr_as_string(v)
-    end
-  else
-    v_type = type(v) --[[ @as vim.option_type ]]
-
-    if v_type == 'boolean' then
-      v = v and 'true' or 'false'
-    elseif v_type == 'number' then
-      v = ('%iL'):format(v)
-    elseif v_type == 'string' then
-      --- @cast v string
-      v = static_cstr_as_string(cstr(v))
-    end
-  end
-
-  return ('{ .type = %s, .data.%s = %s }'):format(opt_type_enum(v_type), v_type, v)
-end
-
---- @param d vim.option_value|function
---- @param n string
---- @return string
-local function get_defaults(d, n)
-  if d == nil then
-    error("option '" .. n .. "' should have a default value")
-  end
-  return get_opt_val(d)
-end
-
---- @param i integer
---- @param o vim.option_meta
---- @param write fun(...: string)
-local function dump_option(i, o, write)
-  write('  [', ('%u'):format(i - 1) .. ']={')
-  write('    .fullname=', cstr(o.full_name))
-  if o.abbreviation then
-    write('    .shortname=', cstr(o.abbreviation))
-  end
-  write('    .type=', opt_type_enum(o.type))
-  write('    .flags=', get_flags(o))
-  write('    .scope_flags=', get_scope_flags(o))
-  write('    .scope_idx=', get_scope_idx(o))
-  write('    .values=', (o.values and get_values_var(o) or 'NULL'))
-  write('    .values_len=', (o.values and #o.values or '0'))
-  write('    .flags_var=', (o.flags_varname and ('&%s'):format(o.flags_varname) or 'NULL'))
-  if o.enable_if then
-    write(('#if defined(%s)'):format(o.enable_if))
-  end
-
-  local is_window_local = #o.scope == 1 and o.scope[1] == 'win'
-
-  if is_window_local then
-    write('    .var=NULL')
-  elseif o.varname then
-    write('    .var=&', o.varname)
-  elseif o.immutable then
-    -- Immutable options can directly point to the default value.
-    write(('    .var=&options[%u].def_val.data'):format(i - 1))
-  else
-    error('Option must be immutable or have a variable.')
-  end
-
-  write('    .immutable=', (o.immutable and 'true' or 'false'))
-  write('    .opt_did_set_cb=', o.cb or 'NULL')
-  write('    .opt_expand_cb=', o.expand_cb or 'NULL')
-
-  if o.enable_if then
-    write('#else')
-    -- Hidden option directly points to default value.
-    write(('    .var=&options[%u].def_val.data'):format(i - 1))
-    -- Option is always immutable on the false branch of `enable_if`.
-    write('    .immutable=true')
-    write('#endif')
-  end
-
-  if not o.defaults then
-    write('    .def_val=NIL_OPTVAL')
-  elseif o.defaults.condition then
-    write(('#if defined(%s)'):format(o.defaults.condition))
-    write('    .def_val=', get_defaults(o.defaults.if_true, o.full_name))
-    if o.defaults.if_false then
-      write('#else')
-      write('    .def_val=', get_defaults(o.defaults.if_false, o.full_name))
-    end
-    write('#endif')
-  else
-    write('    .def_val=', get_defaults(o.defaults.if_true, o.full_name))
-  end
-
-  write('  },')
-end
-
---- @param prefix string
---- @param values vim.option_valid_values
-local function preorder_traversal(prefix, values)
-  local out = {} --- @type string[]
-
-  local function add(s)
-    table.insert(out, s)
-  end
-
-  add('')
-  add(('EXTERN const char *(%s_values[%s]) INIT( = {'):format(prefix, #vim.tbl_keys(values) + 1))
-
-  --- @type [string,vim.option_valid_values][]
-  local children = {}
-
-  for _, value in ipairs(values) do
-    if type(value) == 'string' then
-      add(('  "%s",'):format(value))
-    else
-      assert(type(value) == 'table' and type(value[1]) == 'string' and type(value[2]) == 'table')
-      add(('  "%s",'):format(value[1]))
-      table.insert(children, value)
-    end
-  end
-
-  add('  NULL')
-  add('});')
-
-  for _, value in pairs(children) do
-    -- Remove trailing colon from the added prefix to prevent syntax errors.
-    add(preorder_traversal(prefix .. '_' .. value[1]:gsub(':$', ''), value[2]))
-  end
-
-  return table.concat(out, '\n')
-end
-
---- @param o vim.option_meta
---- @return string
-local function gen_opt_enum(o)
-  local out = {} --- @type string[]
-
-  local function add(s)
-    table.insert(out, s)
-  end
-
-  add('')
-  add('typedef enum {')
-
-  local opt_name = lowercase_to_titlecase(o.abbreviation or o.full_name)
-  --- @type table
-  local enum_values
-
-  if type(o.flags) == 'table' then
-    enum_values = o.flags --[[ @as table ]]
-  else
-    enum_values = {}
-    for i, flag_name in ipairs(o.values) do
-      assert(type(flag_name) == 'string')
-      enum_values[flag_name] = math.pow(2, i - 1)
-    end
-  end
-
-  -- Sort the keys by the flag value so that the enum can be generated in order.
-  --- @type string[]
-  local flag_names = vim.tbl_keys(enum_values)
-  table.sort(flag_names, function(a, b)
-    return enum_values[a] < enum_values[b]
-  end)
-
-  for _, flag_name in pairs(flag_names) do
-    add(
-      ('  kOpt%sFlag%s = 0x%02x,'):format(
-        opt_name,
-        lowercase_to_titlecase(flag_name:gsub(':$', '')),
-        enum_values[flag_name]
-      )
-    )
-  end
-
-  add(('} Opt%sFlags;'):format(opt_name))
-
-  return table.concat(out, '\n')
-end
-
---- @param output_file string
---- @return table options_index Map of option name to option index
-local function gen_enums(output_file)
-  --- Options for each scope.
-  --- @type table
-  local scope_options = {}
-  for _, scope in ipairs(valid_scopes) do
-    scope_options[scope] = {}
-  end
-
-  local fd = assert(io.open(output_file, 'w'))
-
-  --- @param s string
-  local function write(s)
-    fd:write(s)
-    fd:write('\n')
-  end
-
-  -- Generate options enum file
-  write('// IWYU pragma: private, include "nvim/option_defs.h"')
-  write('')
-
-  --- Map of option name to option index
-  --- @type table
-  local option_index = {}
-
-  -- Generate option index enum and populate the `option_index` and `scope_option` dicts.
-  write('typedef enum {')
-  write('  kOptInvalid = -1,')
-
-  for i, o in ipairs(options_meta) do
-    local enum_val_name = 'kOpt' .. lowercase_to_titlecase(o.full_name)
-    write(('  %s = %u,'):format(enum_val_name, i - 1))
-
-    option_index[o.full_name] = enum_val_name
-
-    if o.abbreviation then
-      option_index[o.abbreviation] = enum_val_name
-    end
-
-    local alias = o.alias or {} --[[@as string[] ]]
-    for _, v in ipairs(alias) do
-      option_index[v] = enum_val_name
-    end
-
-    for _, scope in ipairs(o.scope) do
-      table.insert(scope_options[scope], o)
-    end
-  end
-
-  write('  // Option count')
-  write('#define kOptCount ' .. tostring(#options_meta))
-  write('} OptIndex;')
-
-  -- Generate option index enum for each scope
-  for _, scope in ipairs(valid_scopes) do
-    write('')
-
-    local scope_name = lowercase_to_titlecase(scope)
-    write('typedef enum {')
-    write(('  %s = -1,'):format(get_scope_option(scope, 'Invalid')))
-
-    for idx, option in ipairs(scope_options[scope]) do
-      write(('  %s = %u,'):format(get_scope_option(scope, option.full_name), idx - 1))
-    end
-
-    write(('  // %s option count'):format(scope_name))
-    write(('#define %s %d'):format(get_scope_option(scope, 'Count'), #scope_options[scope]))
-    write(('} %sOptIndex;'):format(scope_name))
-  end
-
-  -- Generate reverse lookup from option scope index to option index for each scope.
-  for _, scope in ipairs(valid_scopes) do
-    write('')
-    write(('EXTERN const OptIndex %s_opt_idx[] INIT( = {'):format(scope))
-    for _, option in ipairs(scope_options[scope]) do
-      local idx = option_index[option.full_name]
-      write(('  [%s] = %s,'):format(get_scope_option(scope, option.full_name), idx))
-    end
-    write('});')
-  end
-
-  fd:close()
-
-  return option_index
-end
-
---- @param output_file string
---- @param option_index table
-local function gen_map(output_file, option_index)
-  -- Generate option index map.
-  local hashy = require('generators.hashy')
-
-  local neworder, hashfun = hashy.hashy_hash(
-    'find_option',
-    vim.tbl_keys(option_index),
-    function(idx)
-      return ('option_hash_elems[%s].name'):format(idx)
-    end
-  )
-
-  local fd = assert(io.open(output_file, 'w'))
-
-  --- @param s string
-  local function write(s)
-    fd:write(s)
-    fd:write('\n')
-  end
-
-  write('static const struct { const char *name; OptIndex opt_idx; } option_hash_elems[] = {')
-
-  for _, name in ipairs(neworder) do
-    assert(option_index[name] ~= nil)
-    write(('  { .name = "%s", .opt_idx = %s },'):format(name, option_index[name]))
-  end
-
-  write('};')
-  write('')
-  write('static ' .. hashfun)
-
-  fd:close()
-end
-
---- @param output_file string
-local function gen_vars(output_file)
-  local fd = assert(io.open(output_file, 'w'))
-
-  --- @param s string
-  local function write(s)
-    fd:write(s)
-    fd:write('\n')
-  end
-
-  write('// IWYU pragma: private, include "nvim/option_vars.h"')
-
-  -- Generate enums for option flags.
-  for _, o in ipairs(options_meta) do
-    if o.flags and (type(o.flags) == 'table' or o.values) then
-      write(gen_opt_enum(o))
-    end
-  end
-
-  -- Generate valid values for each option.
-  for _, option in ipairs(options_meta) do
-    -- Since option values can be nested, we need to do preorder traversal to generate the values.
-    if option.values then
-      local values_var = ('opt_%s'):format(option.abbreviation or option.full_name)
-      write(preorder_traversal(values_var, option.values))
-    end
-  end
-
-  fd:close()
-end
-
---- @param output_file string
-local function gen_options(output_file)
-  local fd = assert(io.open(output_file, 'w'))
-
-  --- @param ... string
-  local function write(...)
-    local s = table.concat({ ... }, '')
-    fd:write(s)
-    if s:match('^    %.') then
-      fd:write(',')
-    end
-    fd:write('\n')
-  end
-
-  -- Generate options[] array.
-  write([[
-  #include "nvim/ex_docmd.h"
-  #include "nvim/ex_getln.h"
-  #include "nvim/insexpand.h"
-  #include "nvim/mapping.h"
-  #include "nvim/ops.h"
-  #include "nvim/option.h"
-  #include "nvim/optionstr.h"
-  #include "nvim/quickfix.h"
-  #include "nvim/runtime.h"
-  #include "nvim/tag.h"
-  #include "nvim/window.h"
-
-  static vimoption_T options[] = {]])
-
-  for i, o in ipairs(options_meta) do
-    dump_option(i, o, write)
-  end
-
-  write('};')
-
-  fd:close()
-end
-
-local function main()
-  local options_file = arg[1]
-  local options_enum_file = arg[2]
-  local options_map_file = arg[3]
-  local option_vars_file = arg[4]
-
-  local option_index = gen_enums(options_enum_file)
-  gen_map(options_map_file, option_index)
-  gen_vars(option_vars_file)
-  gen_options(options_file)
-end
-
-main()
diff --git a/src/nvim/generators/gen_vimvim.lua b/src/nvim/generators/gen_vimvim.lua
deleted file mode 100644
index 3817735a55..0000000000
--- a/src/nvim/generators/gen_vimvim.lua
+++ /dev/null
@@ -1,156 +0,0 @@
-local mpack = vim.mpack
-
-local syntax_file = arg[1]
-local funcs_file = arg[2]
-
-local lld = {}
-local syn_fd = io.open(syntax_file, 'w')
-lld.line_length = 0
-local function w(s)
-  syn_fd:write(s)
-  if s:find('\n') then
-    lld.line_length = #(s:gsub('.*\n', ''))
-  else
-    lld.line_length = lld.line_length + #s
-  end
-end
-
-local options = require('options')
-local auevents = require('auevents')
-local ex_cmds = require('ex_cmds')
-
-local function cmd_kw(prev_cmd, cmd)
-  if not prev_cmd then
-    return cmd:sub(1, 1) .. '[' .. cmd:sub(2) .. ']'
-  else
-    local shift = 1
-    while cmd:sub(shift, shift) == prev_cmd:sub(shift, shift) do
-      shift = shift + 1
-    end
-    if cmd:sub(1, shift) == 'def' then
-      shift = shift + 1
-    end
-    if shift >= #cmd then
-      return cmd
-    else
-      return cmd:sub(1, shift) .. '[' .. cmd:sub(shift + 1) .. ']'
-    end
-  end
-end
-
--- Exclude these from the vimCommand keyword list, they are handled specially
--- in syntax/vim.vim (vimAugroupKey, vimAutoCmd, vimGlobal, vimSubst). #9327
-local function is_special_cased_cmd(cmd)
-  return (
-    cmd == 'augroup'
-    or cmd == 'autocmd'
-    or cmd == 'doautocmd'
-    or cmd == 'doautoall'
-    or cmd == 'global'
-    or cmd == 'substitute'
-  )
-end
-
-local vimcmd_start = 'syn keyword vimCommand contained '
-local vimcmd_end = ' nextgroup=vimBang'
-w(vimcmd_start)
-
-local prev_cmd = nil
-for _, cmd_desc in ipairs(ex_cmds.cmds) do
-  if lld.line_length > 850 then
-    w(vimcmd_end .. '\n' .. vimcmd_start)
-  end
-  local cmd = cmd_desc.command
-  if cmd:match('%w') and cmd ~= 'z' and not is_special_cased_cmd(cmd) then
-    w(' ' .. cmd_kw(prev_cmd, cmd))
-  end
-  if cmd == 'delete' then
-    -- Add special abbreviations of :delete
-    w(' ' .. cmd_kw('d', 'dl'))
-    w(' ' .. cmd_kw('del', 'dell'))
-    w(' ' .. cmd_kw('dele', 'delel'))
-    w(' ' .. cmd_kw('delet', 'deletl'))
-    w(' ' .. cmd_kw('delete', 'deletel'))
-    w(' ' .. cmd_kw('d', 'dp'))
-    w(' ' .. cmd_kw('de', 'dep'))
-    w(' ' .. cmd_kw('del', 'delp'))
-    w(' ' .. cmd_kw('dele', 'delep'))
-    w(' ' .. cmd_kw('delet', 'deletp'))
-    w(' ' .. cmd_kw('delete', 'deletep'))
-  end
-  prev_cmd = cmd
-end
-
-w(vimcmd_end .. '\n')
-
-local vimopt_start = 'syn keyword vimOption contained '
-local vimopt_end = ' skipwhite nextgroup=vimSetEqual,vimSetMod'
-w('\n' .. vimopt_start)
-
-for _, opt_desc in ipairs(options.options) do
-  if not opt_desc.immutable then
-    if lld.line_length > 850 then
-      w(vimopt_end .. '\n' .. vimopt_start)
-    end
-    w(' ' .. opt_desc.full_name)
-    if opt_desc.abbreviation then
-      w(' ' .. opt_desc.abbreviation)
-    end
-    if opt_desc.type == 'boolean' then
-      w(' inv' .. opt_desc.full_name)
-      w(' no' .. opt_desc.full_name)
-      if opt_desc.abbreviation then
-        w(' inv' .. opt_desc.abbreviation)
-        w(' no' .. opt_desc.abbreviation)
-      end
-    end
-  end
-end
-
-w(vimopt_end .. '\n')
-
-w('\nsyn case ignore')
-local vimau_start = 'syn keyword vimAutoEvent contained '
-w('\n\n' .. vimau_start)
-
-for _, au in ipairs(auevents.events) do
-  if not auevents.nvim_specific[au[1]] then
-    if lld.line_length > 850 then
-      w('\n' .. vimau_start)
-    end
-    w(' ' .. au[1])
-    for _, alias in ipairs(au[2]) do
-      if lld.line_length > 850 then
-        w('\n' .. vimau_start)
-      end
-      -- au[1] is aliased to alias
-      w(' ' .. alias)
-    end
-  end
-end
-
-local nvimau_start = 'syn keyword nvimAutoEvent contained '
-w('\n\n' .. nvimau_start)
-
-for au, _ in vim.spairs(auevents.nvim_specific) do
-  if lld.line_length > 850 then
-    w('\n' .. nvimau_start)
-  end
-  w(' ' .. au)
-end
-
-w('\n\nsyn case match')
-local vimfun_start = 'syn keyword vimFuncName contained '
-w('\n\n' .. vimfun_start)
-local funcs = mpack.decode(io.open(funcs_file, 'rb'):read('*all'))
-for _, name in ipairs(funcs) do
-  if name then
-    if lld.line_length > 850 then
-      w('\n' .. vimfun_start)
-    end
-    w(' ' .. name)
-  end
-end
-
-w('\n')
-syn_fd:close()
diff --git a/src/nvim/generators/hashy.lua b/src/nvim/generators/hashy.lua
deleted file mode 100644
index 74b7655324..0000000000
--- a/src/nvim/generators/hashy.lua
+++ /dev/null
@@ -1,145 +0,0 @@
--- HASHY McHASHFACE
-
-local M = {}
-_G.d = M
-
-local function setdefault(table, key)
-  local val = table[key]
-  if val == nil then
-    val = {}
-    table[key] = val
-  end
-  return val
-end
-
-function M.build_pos_hash(strings)
-  local len_buckets = {}
-  local maxlen = 0
-  for _, s in ipairs(strings) do
-    table.insert(setdefault(len_buckets, #s), s)
-    if #s > maxlen then
-      maxlen = #s
-    end
-  end
-
-  local len_pos_buckets = {}
-  local worst_buck_size = 0
-
-  for len = 1, maxlen do
-    local strs = len_buckets[len]
-    if strs then
-      -- the best position so far generates `best_bucket`
-      -- with `minsize` worst case collisions
-      local bestpos, minsize, best_bucket = nil, #strs * 2, nil
-      for pos = 1, len do
-        local try_bucket = {}
-        for _, str in ipairs(strs) do
-          local poschar = string.sub(str, pos, pos)
-          table.insert(setdefault(try_bucket, poschar), str)
-        end
-        local maxsize = 1
-        for _, pos_strs in pairs(try_bucket) do
-          maxsize = math.max(maxsize, #pos_strs)
-        end
-        if maxsize < minsize then
-          bestpos = pos
-          minsize = maxsize
-          best_bucket = try_bucket
-        end
-      end
-      len_pos_buckets[len] = { bestpos, best_bucket }
-      worst_buck_size = math.max(worst_buck_size, minsize)
-    end
-  end
-  return len_pos_buckets, maxlen, worst_buck_size
-end
-
-function M.switcher(put, tab, maxlen, worst_buck_size)
-  local neworder = {} --- @type string[]
-  put '  switch (len) {\n'
-  local bucky = worst_buck_size > 1
-  for len = 1, maxlen do
-    local vals = tab[len]
-    if vals then
-      put('    case ' .. len .. ': ')
-      local pos, posbuck = unpack(vals)
-      local keys = vim.tbl_keys(posbuck)
-      if #keys > 1 then
-        table.sort(keys)
-        put('switch (str[' .. (pos - 1) .. ']) {\n')
-        for _, c in ipairs(keys) do
-          local buck = posbuck[c]
-          local startidx = #neworder
-          vim.list_extend(neworder, buck)
-          local endidx = #neworder
-          put("      case '" .. c .. "': ")
-          if len == 1 then
-            put('return ' .. startidx .. ';\n')
-          else
-            put('low = ' .. startidx .. '; ')
-            if bucky then
-              put('high = ' .. endidx .. '; ')
-            end
-            put 'break;\n'
-          end
-        end
-        put '      default: break;\n'
-        put '    }\n    '
-      else
-        local startidx = #neworder
-        table.insert(neworder, posbuck[keys[1]][1])
-        local endidx = #neworder
-        put('low = ' .. startidx .. '; ')
-        if bucky then
-          put('high = ' .. endidx .. '; ')
-        end
-      end
-      put 'break;\n'
-    end
-  end
-  put '    default: break;\n'
-  put '  }\n'
-  return neworder
-end
-
-function M.hashy_hash(name, strings, access)
-  local stats = {}
-  local put = function(str)
-    table.insert(stats, str)
-  end
-  local len_pos_buckets, maxlen, worst_buck_size = M.build_pos_hash(strings)
-  put('int ' .. name .. '_hash(const char *str, size_t len)\n{\n')
-  if maxlen == 1 then
-    put('\n') -- nothing
-  elseif worst_buck_size > 1 then
-    put('  int low = 0, high = 0;\n')
-  else
-    put('  int low = -1;\n')
-  end
-  local neworder = M.switcher(put, len_pos_buckets, maxlen, worst_buck_size)
-  if maxlen == 1 then
-    put([[
-  return -1;
-]])
-  elseif worst_buck_size > 1 then
-    put([[
-  for (int i = low; i < high; i++) {
-    if (!memcmp(str, ]] .. access('i') .. [[, len)) {
-      return i;
-    }
-  }
-  return -1;
-]])
-  else
-    put([[
-  if (low < 0 || memcmp(str, ]] .. access('low') .. [[, len)) {
-    return -1;
-  }
-  return low;
-]])
-  end
-  put '}\n\n'
-  return neworder, table.concat(stats)
-end
-
-return M
diff --git a/src/nvim/generators/nvim_version.lua.in b/src/nvim/generators/nvim_version.lua.in
deleted file mode 100644
index c29141fc68..0000000000
--- a/src/nvim/generators/nvim_version.lua.in
+++ /dev/null
@@ -1,9 +0,0 @@
-return {
-  {"major", ${NVIM_VERSION_MAJOR}},
-  {"minor", ${NVIM_VERSION_MINOR}},
-  {"patch", ${NVIM_VERSION_PATCH}},
-  {"prerelease", "${NVIM_VERSION_PRERELEASE}" ~= ""},
-  {"api_level", ${NVIM_API_LEVEL}},
-  {"api_compatible", ${NVIM_API_LEVEL_COMPAT}},
-  {"api_prerelease", ${NVIM_API_PRERELEASE}},
-}
diff --git a/src/nvim/generators/preload.lua b/src/nvim/generators/preload.lua
deleted file mode 100644
index e14671074c..0000000000
--- a/src/nvim/generators/preload.lua
+++ /dev/null
@@ -1,13 +0,0 @@
-local srcdir = table.remove(arg, 1)
-local nlualib = table.remove(arg, 1)
-local gendir = table.remove(arg, 1)
-package.path = srcdir .. '/src/nvim/?.lua;' .. srcdir .. '/runtime/lua/?.lua;' .. package.path
-package.path = gendir .. '/?.lua;' .. package.path
-_G.vim = require 'vim.shared'
-_G.vim.inspect = require 'vim.inspect'
-package.cpath = package.cpath .. ';' .. nlualib
-require 'nlua0'
-vim.NIL = vim.mpack.NIL -- WOW BOB WOW
-
-arg[0] = table.remove(arg, 1)
-return loadfile(arg[0])()
-- 
cgit