diff options
author | Lewis Russell <lewis6991@gmail.com> | 2025-02-26 11:38:07 +0000 |
---|---|---|
committer | Lewis Russell <me@lewisr.dev> | 2025-02-26 16:54:37 +0000 |
commit | 0f24b0826a27b7868a3aacc25199787e7453d4cc (patch) | |
tree | 49585aac252581a735577f2e5711201a85ab8a7e /src | |
parent | 85caaa70d44b7b18c633aa0b140de5f3f6d3eee7 (diff) | |
download | rneovim-0f24b0826a27b7868a3aacc25199787e7453d4cc.tar.gz rneovim-0f24b0826a27b7868a3aacc25199787e7453d4cc.tar.bz2 rneovim-0f24b0826a27b7868a3aacc25199787e7453d4cc.zip |
build: move all generator scripts to `src/gen/`
- Move all generator Lua scripts to the `src/gen/`
- Add a `.luarc.json` to `src/gen/`
- Add a `preload.lua` to `src/gen/`
- Add `src` to `package.path` so it aligns with `.luarc.json'
- Fix all `require` statements in `src/gen/` so they are consistent:
- `require('scripts.foo')` -> `require('gen.foo')`
- `require('src.nvim.options')` -> `require('nvim.options')`
- `require('api.dispatch_deprecated')` -> `require('nvim.api.dispatch_deprecated')`
Diffstat (limited to 'src')
-rw-r--r-- | src/.luarc.json | 24 | ||||
-rw-r--r-- | src/gen/c_grammar.lua (renamed from src/nvim/generators/c_grammar.lua) | 0 | ||||
-rw-r--r-- | src/gen/cdoc_grammar.lua | 87 | ||||
-rw-r--r-- | src/gen/cdoc_parser.lua | 223 | ||||
-rw-r--r-- | src/gen/dump_bin_array.lua (renamed from src/nvim/generators/dump_bin_array.lua) | 0 | ||||
-rw-r--r-- | src/gen/gen_api_dispatch.lua (renamed from src/nvim/generators/gen_api_dispatch.lua) | 8 | ||||
-rw-r--r-- | src/gen/gen_api_ui_events.lua (renamed from src/nvim/generators/gen_api_ui_events.lua) | 4 | ||||
-rw-r--r-- | src/gen/gen_char_blob.lua (renamed from src/nvim/generators/gen_char_blob.lua) | 0 | ||||
-rw-r--r-- | src/gen/gen_declarations.lua (renamed from src/nvim/generators/gen_declarations.lua) | 2 | ||||
-rw-r--r-- | src/gen/gen_eval.lua (renamed from src/nvim/generators/gen_eval.lua) | 4 | ||||
-rwxr-xr-x | src/gen/gen_eval_files.lua | 1090 | ||||
-rw-r--r-- | src/gen/gen_events.lua (renamed from src/nvim/generators/gen_events.lua) | 2 | ||||
-rw-r--r-- | src/gen/gen_ex_cmds.lua (renamed from src/nvim/generators/gen_ex_cmds.lua) | 2 | ||||
-rw-r--r-- | src/gen/gen_filetype.lua | 209 | ||||
-rw-r--r-- | src/gen/gen_help_html.lua | 1491 | ||||
-rw-r--r-- | src/gen/gen_lsp.lua | 514 | ||||
-rw-r--r-- | src/gen/gen_options.lua (renamed from src/nvim/generators/gen_options.lua) | 4 | ||||
-rwxr-xr-x | src/gen/gen_vimdoc.lua | 1041 | ||||
-rw-r--r-- | src/gen/gen_vimvim.lua (renamed from src/nvim/generators/gen_vimvim.lua) | 6 | ||||
-rw-r--r-- | src/gen/hashy.lua (renamed from src/nvim/generators/hashy.lua) | 0 | ||||
-rw-r--r-- | src/gen/luacats_grammar.lua | 207 | ||||
-rw-r--r-- | src/gen/luacats_parser.lua | 535 | ||||
-rw-r--r-- | src/gen/nvim_version.lua.in (renamed from src/nvim/generators/nvim_version.lua.in) | 0 | ||||
-rw-r--r-- | src/gen/preload.lua | 6 | ||||
-rw-r--r-- | src/gen/preload_nlua.lua (renamed from src/nvim/generators/preload.lua) | 8 | ||||
-rw-r--r-- | src/gen/util.lua | 399 | ||||
-rw-r--r-- | src/nvim/CMakeLists.txt | 18 | ||||
-rw-r--r-- | src/nvim/func_attr.h | 2 |
28 files changed, 5860 insertions, 26 deletions
diff --git a/src/.luarc.json b/src/.luarc.json new file mode 100644 index 0000000000..06f49f65d0 --- /dev/null +++ b/src/.luarc.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://raw.githubusercontent.com/LuaLS/vscode-lua/master/setting/schema.json", + "runtime": { + "version": "LuaJIT" + }, + "workspace": { + "library": [ + "../runtime/lua", + "${3rd}/luv/library" + ], + "checkThirdParty": "Disable" + }, + "diagnostics": { + "groupFileStatus": { + "strict": "Opened", + "strong": "Opened" + }, + "groupSeverity": { + "strong": "Warning", + "strict": "Warning" + }, + "unusedLocalExclude": [ "_*" ] + } +} diff --git a/src/nvim/generators/c_grammar.lua b/src/gen/c_grammar.lua index 890c260843..890c260843 100644 --- a/src/nvim/generators/c_grammar.lua +++ b/src/gen/c_grammar.lua diff --git a/src/gen/cdoc_grammar.lua b/src/gen/cdoc_grammar.lua new file mode 100644 index 0000000000..6a7610883b --- /dev/null +++ b/src/gen/cdoc_grammar.lua @@ -0,0 +1,87 @@ +--[[! +LPEG grammar for C doc comments +]] + +--- @class nvim.cdoc.Param +--- @field kind 'param' +--- @field name string +--- @field desc? string + +--- @class nvim.cdoc.Return +--- @field kind 'return' +--- @field desc string + +--- @class nvim.cdoc.Note +--- @field desc? string + +--- @alias nvim.cdoc.grammar.result +--- | nvim.cdoc.Param +--- | nvim.cdoc.Return +--- | nvim.cdoc.Note + +--- @class nvim.cdoc.grammar +--- @field match fun(self, input: string): nvim.cdoc.grammar.result? + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local Ct, Cg = lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local nl = P('\r\n') + P('\n') +local ws = rep1(S(' \t') + nl) + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local ident = letter * rep(letter + R('09')) + +local io = P('[') * (P('in') + P('out') + P('inout')) * P(']') + +--- @param x string +local function Pf(x) + return opt(ws) * P(x) * opt(ws) +end + +--- @type table<string,vim.lpeg.Pattern> +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +local grammar = P { + rep1(P('@') * v.ats), + + ats = v.at_param + v.at_return + v.at_deprecated + v.at_see + v.at_brief + v.at_note + v.at_nodoc, + + at_param = Ct( + Cg(P('param'), 'kind') * opt(io) * ws * Cg(ident, 'name') * opt(ws * Cg(rep(any), 'desc')) + ), + + at_return = Ct(Cg(P('return'), 'kind') * opt(S('s')) * opt(ws * Cg(rep(any), 'desc'))), + + at_deprecated = Ct(Cg(P('deprecated'), 'kind')), + + at_see = Ct(Cg(P('see'), 'kind') * ws * opt(Pf('#')) * Cg(rep(any), 'desc')), + + at_brief = Ct(Cg(P('brief'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_note = Ct(Cg(P('note'), 'kind') * ws * Cg(rep(any), 'desc')), + + at_nodoc = Ct(Cg(P('nodoc'), 'kind')), +} + +return grammar --[[@as nvim.cdoc.grammar]] diff --git a/src/gen/cdoc_parser.lua b/src/gen/cdoc_parser.lua new file mode 100644 index 0000000000..38314c0efd --- /dev/null +++ b/src/gen/cdoc_parser.lua @@ -0,0 +1,223 @@ +local cdoc_grammar = require('gen.cdoc_grammar') +local c_grammar = require('gen.c_grammar') + +--- @class nvim.cdoc.parser.param +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.cdoc.parser.note +--- @field desc string + +--- @class nvim.cdoc.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.cdoc.parser.fun +--- @field name string +--- @field params nvim.cdoc.parser.param[] +--- @field returns nvim.cdoc.parser.return[] +--- @field desc string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field notes? nvim.cdoc.parser.note[] +--- @field see? nvim.cdoc.parser.note[] + +--- @class nvim.cdoc.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.cdoc.parser.obj +--- @field last_doc_item? nvim.cdoc.parser.param|nvim.cdoc.parser.return|nvim.cdoc.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.cdoc.parser.obj +--- | nvim.cdoc.parser.fun +--- | nvim.cdoc.parser.brief + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.cdoc.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.cdoc.parser.State +local function process_doc_line(line, state) + line = line:gsub('^%s+@', '@') + + local parsed = cdoc_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + + local kind = parsed.kind + + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + state.last_doc_item = { + name = parsed.name, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = { { + desc = parsed.desc, + } } + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[1] + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param item table +--- @param state nvim.cdoc.parser.State +local function process_proto(item, state) + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + cur_obj.name = item.name + cur_obj.params = cur_obj.params or {} + + for _, p in ipairs(item.parameters) do + local param = { name = p[2], type = p[1] } + local added = false + for _, cp in ipairs(cur_obj.params) do + if cp.name == param.name then + cp.type = param.type + added = true + break + end + end + + if not added then + table.insert(cur_obj.params, param) + end + end + + cur_obj.returns = cur_obj.returns or { {} } + cur_obj.returns[1].type = item.return_type + + for _, a in ipairs({ + 'fast', + 'remote_only', + 'lua_only', + 'textlock', + 'textlock_allow_cmdwin', + }) do + if item[a] then + cur_obj.attrs = cur_obj.attrs or {} + table.insert(cur_obj.attrs, a) + end + end + + cur_obj.deprecated_since = item.deprecated_since + + -- Remove some arguments + for i = #cur_obj.params, 1, -1 do + local p = cur_obj.params[i] + if p.name == 'channel_id' or vim.tbl_contains({ 'lstate', 'arena', 'error' }, p.type) then + table.remove(cur_obj.params, i) + end + end +end + +local M = {} + +--- @param filename string +--- @return {} classes +--- @return nvim.cdoc.parser.fun[] funs +--- @return string[] briefs +function M.parse(filename) + local funs = {} --- @type nvim.cdoc.parser.fun[] + local briefs = {} --- @type string[] + local state = {} --- @type nvim.cdoc.parser.State + + local txt = assert(io.open(filename, 'r')):read('*all') + + local parsed = c_grammar.grammar:match(txt) + for _, item in ipairs(parsed) do + if item.comment then + process_doc_line(item.comment, state) + else + add_doc_lines_to_obj(state) + if item[1] == 'proto' then + process_proto(item, state) + table.insert(funs, state.cur_obj) + end + local cur_obj = state.cur_obj + if cur_obj and not item.static then + if cur_obj.kind == 'brief' then + table.insert(briefs, cur_obj.desc) + end + end + state = {} + end + end + + return {}, funs, briefs +end + +-- M.parse('src/nvim/api/vim.c') + +return M diff --git a/src/nvim/generators/dump_bin_array.lua b/src/gen/dump_bin_array.lua index c6cda25e73..c6cda25e73 100644 --- a/src/nvim/generators/dump_bin_array.lua +++ b/src/gen/dump_bin_array.lua diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/gen/gen_api_dispatch.lua index 378297d86a..a5d0890c2f 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/gen/gen_api_dispatch.lua @@ -7,7 +7,7 @@ local mpack = vim.mpack -local hashy = require 'generators.hashy' +local hashy = require 'gen.hashy' local pre_args = 7 assert(#arg >= pre_args) @@ -31,7 +31,7 @@ local headers = {} -- set of function names, used to detect duplicates local function_names = {} -local c_grammar = require('generators.c_grammar') +local c_grammar = require('gen.c_grammar') local startswith = vim.startswith @@ -150,7 +150,7 @@ end -- Export functions under older deprecated names. -- These will be removed eventually. -local deprecated_aliases = require('api.dispatch_deprecated') +local deprecated_aliases = require('nvim.api.dispatch_deprecated') for _, f in ipairs(shallowcopy(functions)) do local ismethod = false if startswith(f.name, 'nvim_') then @@ -300,7 +300,7 @@ for i, item in ipairs(types) do end local packed = table.concat(pieces) -local dump_bin_array = require('generators.dump_bin_array') +local dump_bin_array = require('gen.dump_bin_array') dump_bin_array(api_metadata_output, 'packed_api_metadata', packed) api_metadata_output:close() diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/gen/gen_api_ui_events.lua index a3bb76cb91..8ba67dafff 100644 --- a/src/nvim/generators/gen_api_ui_events.lua +++ b/src/gen/gen_api_ui_events.lua @@ -7,10 +7,10 @@ local remote_output = io.open(arg[3], 'wb') local metadata_output = io.open(arg[4], 'wb') local client_output = io.open(arg[5], 'wb') -local c_grammar = require('generators.c_grammar') +local c_grammar = require('gen.c_grammar') local events = c_grammar.grammar:match(input:read('*all')) -local hashy = require 'generators.hashy' +local hashy = require 'gen.hashy' local function write_signature(output, ev, prefix, notype) output:write('(' .. prefix) diff --git a/src/nvim/generators/gen_char_blob.lua b/src/gen/gen_char_blob.lua index c40e0d6e82..c40e0d6e82 100644 --- a/src/nvim/generators/gen_char_blob.lua +++ b/src/gen/gen_char_blob.lua diff --git a/src/nvim/generators/gen_declarations.lua b/src/gen/gen_declarations.lua index 6e1ea92572..582ac756b4 100644 --- a/src/nvim/generators/gen_declarations.lua +++ b/src/gen/gen_declarations.lua @@ -1,4 +1,4 @@ -local grammar = require('generators.c_grammar').grammar +local grammar = require('gen.c_grammar').grammar --- @param fname string --- @return string? diff --git a/src/nvim/generators/gen_eval.lua b/src/gen/gen_eval.lua index 0b6ee6cb24..9d2f2f7523 100644 --- a/src/nvim/generators/gen_eval.lua +++ b/src/gen/gen_eval.lua @@ -8,7 +8,7 @@ local funcsfname = autodir .. '/funcs.generated.h' --Will generate funcs.generated.h with definition of functions static const array. -local hashy = require 'generators.hashy' +local hashy = require 'gen.hashy' local hashpipe = assert(io.open(funcsfname, 'wb')) @@ -47,7 +47,7 @@ hashpipe:write([[ ]]) -local funcs = require('eval').funcs +local funcs = require('nvim.eval').funcs for _, func in pairs(funcs) do if func.float_func then func.func = 'float_op_wrapper' diff --git a/src/gen/gen_eval_files.lua b/src/gen/gen_eval_files.lua new file mode 100755 index 0000000000..74e45507e5 --- /dev/null +++ b/src/gen/gen_eval_files.lua @@ -0,0 +1,1090 @@ +#!/usr/bin/env -S nvim -l + +-- Generator for various vimdoc and Lua type files + +local util = require('gen.util') +local fmt = string.format + +local DEP_API_METADATA = 'build/funcs_metadata.mpack' +local TEXT_WIDTH = 78 + +--- @class vim.api.metadata +--- @field name string +--- @field parameters [string,string][] +--- @field return_type string +--- @field deprecated_since integer +--- @field eval boolean +--- @field fast boolean +--- @field handler_id integer +--- @field impl_name string +--- @field lua boolean +--- @field method boolean +--- @field remote boolean +--- @field since integer + +local LUA_API_RETURN_OVERRIDES = { + nvim_buf_get_command = 'table<string,vim.api.keyset.command_info>', + nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item_by_id', + nvim_buf_get_extmarks = 'vim.api.keyset.get_extmark_item[]', + nvim_buf_get_keymap = 'vim.api.keyset.get_keymap[]', + nvim_get_autocmds = 'vim.api.keyset.get_autocmds.ret[]', + nvim_get_color_map = 'table<string,integer>', + nvim_get_command = 'table<string,vim.api.keyset.command_info>', + nvim_get_keymap = 'vim.api.keyset.get_keymap[]', + nvim_get_mark = 'vim.api.keyset.get_mark', + + -- Can also return table<string,vim.api.keyset.get_hl_info>, however we need to + -- pick one to get some benefit. + -- REVISIT lewrus01 (26/01/24): we can maybe add + -- @overload fun(ns: integer, {}): table<string,vim.api.keyset.get_hl_info> + nvim_get_hl = 'vim.api.keyset.get_hl_info', + + nvim_get_mode = 'vim.api.keyset.get_mode', + nvim_get_namespaces = 'table<string,integer>', + nvim_get_option_info = 'vim.api.keyset.get_option_info', + nvim_get_option_info2 = 'vim.api.keyset.get_option_info', + nvim_parse_cmd = 'vim.api.keyset.parse_cmd', + nvim_win_get_config = 'vim.api.keyset.win_config', +} + +local LUA_API_KEYSET_OVERRIDES = { + create_autocmd = { + callback = 'string|(fun(args: vim.api.keyset.create_autocmd.callback_args): boolean?)', + }, +} + +local LUA_API_PARAM_OVERRIDES = { + nvim_create_user_command = { + command = 'string|fun(args: vim.api.keyset.create_user_command.command_args)', + }, +} + +local LUA_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", +} + +local LUA_API_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", + '', + '--- This file embeds vimdoc as the function descriptions', + '--- so ignore any doc related errors.', + '--- @diagnostic disable: undefined-doc-name,luadoc-miss-symbol', + '', + 'vim.api = {}', +} + +local LUA_OPTION_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", + '', + '---@class vim.bo', + '---@field [integer] vim.bo', + 'vim.bo = vim.bo', + '', + '---@class vim.wo', + '---@field [integer] vim.wo', + 'vim.wo = vim.wo', +} + +local LUA_VVAR_META_HEADER = { + '--- @meta _', + '-- THIS FILE IS GENERATED', + '-- DO NOT EDIT', + "error('Cannot require a meta file')", + '', + '--- @class vim.v', + 'vim.v = ...', +} + +local LUA_KEYWORDS = { + ['and'] = true, + ['end'] = true, + ['function'] = true, + ['or'] = true, + ['if'] = true, + ['while'] = true, + ['repeat'] = true, + ['true'] = true, + ['false'] = true, +} + +local OPTION_TYPES = { + boolean = 'boolean', + number = 'integer', + string = 'string', +} + +local API_TYPES = { + Window = 'integer', + Tabpage = 'integer', + Buffer = 'integer', + Boolean = 'boolean', + Object = 'any', + Integer = 'integer', + String = 'string', + Array = 'any[]', + LuaRef = 'function', + Dict = 'table<string,any>', + Float = 'number', + HLGroupID = 'integer|string', + void = '', +} + +--- @param s string +--- @return string +local function luaescape(s) + if LUA_KEYWORDS[s] then + return s .. '_' + end + return s +end + +--- @param x string +--- @param sep? string +--- @return string[] +local function split(x, sep) + return vim.split(x, sep or '\n', { plain = true }) +end + +--- Convert an API type to Lua +--- @param t string +--- @return string +local function api_type(t) + if vim.startswith(t, '*') then + return api_type(t:sub(2)) .. '?' + end + + local as0 = t:match('^ArrayOf%((.*)%)') + if as0 then + local as = split(as0, ', ') + return api_type(as[1]) .. '[]' + end + + local d = t:match('^Dict%((.*)%)') + if d then + return 'vim.api.keyset.' .. d + end + + local d0 = t:match('^DictOf%((.*)%)') + if d0 then + return 'table<string,' .. api_type(d0) .. '>' + end + + local u = t:match('^Union%((.*)%)') + if u then + local us = vim.split(u, ',%s*') + return table.concat(vim.tbl_map(api_type, us), '|') + end + + local l = t:match('^LuaRefOf%((.*)%)') + if l then + --- @type string + l = l:gsub('%s+', ' ') + --- @type string?, string? + local as, r = l:match('%((.*)%),%s*(.*)') + if not as then + --- @type string + as = assert(l:match('%((.*)%)')) + end + + local as1 = {} --- @type string[] + for a in vim.gsplit(as, ',%s') do + local a1 = vim.split(a, '%s+', { trimempty = true }) + local nm = a1[2]:gsub('%*(.*)$', '%1?') + as1[#as1 + 1] = nm .. ': ' .. api_type(a1[1]) + end + + return ('fun(%s)%s'):format(table.concat(as1, ', '), r and ': ' .. api_type(r) or '') + end + + return API_TYPES[t] or t +end + +--- @param f string +--- @param params [string,string][]|true +--- @return string +local function render_fun_sig(f, params) + local param_str --- @type string + if params == true then + param_str = '...' + else + param_str = table.concat( + vim.tbl_map( + --- @param v [string,string] + --- @return string + function(v) + return luaescape(v[1]) + end, + params + ), + ', ' + ) + end + + if LUA_KEYWORDS[f] then + return fmt("vim.fn['%s'] = function(%s) end", f, param_str) + else + return fmt('function vim.fn.%s(%s) end', f, param_str) + end +end + +--- Uniquify names +--- @param params [string,string,string][] +--- @return [string,string,string][] +local function process_params(params) + local seen = {} --- @type table<string,true> + local sfx = 1 + + for _, p in ipairs(params) do + if seen[p[1]] then + p[1] = p[1] .. sfx + sfx = sfx + 1 + else + seen[p[1]] = true + end + end + + return params +end + +--- @return table<string, vim.EvalFn> +local function get_api_meta() + local ret = {} --- @type table<string, vim.EvalFn> + + local cdoc_parser = require('gen.cdoc_parser') + + local f = 'src/nvim/api' + + local function include(fun) + if not vim.startswith(fun.name, 'nvim_') then + return false + end + if vim.tbl_contains(fun.attrs or {}, 'lua_only') then + return true + end + if vim.tbl_contains(fun.attrs or {}, 'remote_only') then + return false + end + return true + end + + --- @type table<string,nvim.cdoc.parser.fun> + local functions = {} + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + local filename = vim.fs.joinpath(f, path) + local _, funs = cdoc_parser.parse(filename) + for _, fn in ipairs(funs) do + if include(fn) then + functions[fn.name] = fn + end + end + end + end + + for _, fun in pairs(functions) do + local deprecated = fun.deprecated_since ~= nil + + local notes = {} --- @type string[] + for _, note in ipairs(fun.notes or {}) do + notes[#notes + 1] = note.desc + end + + local sees = {} --- @type string[] + for _, see in ipairs(fun.see or {}) do + sees[#sees + 1] = see.desc + end + + local pty_overrides = LUA_API_PARAM_OVERRIDES[fun.name] or {} + + local params = {} --- @type [string,string][] + for _, p in ipairs(fun.params) do + params[#params + 1] = { + p.name, + api_type(pty_overrides[p.name] or p.type), + not deprecated and p.desc or nil, + } + end + + local r = { + signature = 'NA', + name = fun.name, + params = params, + notes = notes, + see = sees, + returns = api_type(fun.returns[1].type), + deprecated = deprecated, + } + + if not deprecated then + r.desc = fun.desc + r.returns_desc = fun.returns[1].desc + end + + ret[fun.name] = r + end + return ret +end + +--- Convert vimdoc references to markdown literals +--- Convert vimdoc codeblocks to markdown codeblocks +--- +--- Ensure code blocks have one empty line before the start fence and after the closing fence. +--- +--- @param x string +--- @param special string? +--- | 'see-api-meta' Normalize `@see` for API meta docstrings. +--- @return string +local function norm_text(x, special) + if special == 'see-api-meta' then + -- Try to guess a symbol that actually works in @see. + -- "nvim_xx()" => "vim.api.nvim_xx" + x = x:gsub([=[%|?(nvim_[^.()| ]+)%(?%)?%|?]=], 'vim.api.%1') + -- TODO: Remove backticks when LuaLS resolves: https://github.com/LuaLS/lua-language-server/issues/2889 + -- "|foo|" => "`:help foo`" + x = x:gsub([=[|([^ ]+)|]=], '`:help %1`') + end + + return ( + x:gsub('|([^ ]+)|', '`%1`') + :gsub('\n*>lua', '\n\n```lua') + :gsub('\n*>vim', '\n\n```vim') + :gsub('\n+<$', '\n```') + :gsub('\n+<\n+', '\n```\n\n') + :gsub('%s+>\n+', '\n```\n') + :gsub('\n+<%s+\n?', '\n```\n') + ) +end + +--- Generates LuaLS docstring for an API function. +--- @param _f string +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_api_meta(_f, fun, write) + write('') + + if vim.startswith(fun.name, 'nvim__') then + write('--- @private') + end + + if fun.deprecated then + write('--- @deprecated') + end + + local desc = fun.desc + if desc then + write(util.prefix_lines('--- ', norm_text(desc))) + end + + -- LuaLS doesn't support @note. Render @note items as a markdown list. + if fun.notes and #fun.notes > 0 then + write('--- Note:') + write(util.prefix_lines('--- ', table.concat(fun.notes, '\n'))) + write('---') + end + + for _, see in ipairs(fun.see or {}) do + write(util.prefix_lines('--- @see ', norm_text(see, 'see-api-meta'))) + end + + local param_names = {} --- @type string[] + local params = process_params(fun.params) + for _, p in ipairs(params) do + local pname, ptype, pdesc = luaescape(p[1]), p[2], p[3] + param_names[#param_names + 1] = pname + if pdesc then + local s = '--- @param ' .. pname .. ' ' .. ptype .. ' ' + local pdesc_a = split(vim.trim(norm_text(pdesc))) + write(s .. pdesc_a[1]) + for i = 2, #pdesc_a do + if not pdesc_a[i] then + break + end + write('--- ' .. pdesc_a[i]) + end + else + write('--- @param ' .. pname .. ' ' .. ptype) + end + end + + if fun.returns ~= '' then + local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or '' + local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns + write(util.prefix_lines('--- ', '@return ' .. ret .. ret_desc)) + end + local param_str = table.concat(param_names, ', ') + + write(fmt('function vim.api.%s(%s) end', fun.name, param_str)) +end + +--- @return table<string, vim.EvalFn> +local function get_api_keysets_meta() + local mpack_f = assert(io.open(DEP_API_METADATA, 'rb')) + local metadata = assert(vim.mpack.decode(mpack_f:read('*all'))) + + local ret = {} --- @type table<string, vim.EvalFn> + + --- @type {name: string, keys: string[], types: table<string,string>}[] + local keysets = metadata.keysets + + for _, k in ipairs(keysets) do + local pty_overrides = LUA_API_KEYSET_OVERRIDES[k.name] or {} + local params = {} + for _, key in ipairs(k.keys) do + local pty = pty_overrides[key] or k.types[key] or 'any' + table.insert(params, { key .. '?', api_type(pty) }) + end + ret[k.name] = { + signature = 'NA', + name = k.name, + params = params, + } + end + + return ret +end + +--- Generates LuaLS docstring for an API keyset. +--- @param _f string +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_api_keyset_meta(_f, fun, write) + if string.sub(fun.name, 1, 1) == '_' then + return -- not exported + end + write('') + write('--- @class vim.api.keyset.' .. fun.name) + for _, p in ipairs(fun.params) do + write('--- @field ' .. p[1] .. ' ' .. p[2]) + end +end + +--- @return table<string, vim.EvalFn> +local function get_eval_meta() + return require('nvim.eval').funcs +end + +--- Generates LuaLS docstring for a Vimscript "eval" function. +--- @param f string +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_eval_meta(f, fun, write) + if fun.lua == false then + return + end + + local funname = fun.name or f + local params = process_params(fun.params) + + write('') + if fun.deprecated then + write('--- @deprecated') + end + + local desc = fun.desc + + if desc then + --- @type string + desc = desc:gsub('\n%s*\n%s*$', '\n') + for _, l in ipairs(split(desc)) do + l = l:gsub('^ ', ''):gsub('\t', ' '):gsub('@', '\\@') + write('--- ' .. l) + end + end + + for _, text in ipairs(vim.fn.reverse(fun.generics or {})) do + write(fmt('--- @generic %s', text)) + end + + local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 + + for i, param in ipairs(params) do + local pname, ptype = luaescape(param[1]), param[2] + local optional = (pname ~= '...' and i > req_args) and '?' or '' + write(fmt('--- @param %s%s %s', pname, optional, ptype)) + end + + if fun.returns ~= false then + local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or '' + write('--- @return ' .. (fun.returns or 'any') .. ret_desc) + end + + write(render_fun_sig(funname, params)) +end + +--- Generates vimdoc heading for a Vimscript "eval" function signature. +--- @param name string +--- @param name_tag boolean +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_sig_and_tag(name, name_tag, fun, write) + if not fun.signature then + return + end + + local tags = name_tag and { '*' .. name .. '()*' } or {} + + if fun.tags then + for _, t in ipairs(fun.tags) do + tags[#tags + 1] = '*' .. t .. '*' + end + end + + if #tags == 0 then + write(fun.signature) + return + end + + local tag = table.concat(tags, ' ') + local siglen = #fun.signature + local conceal_offset = 2 * (#tags - 1) + local tag_pad_len = math.max(1, 80 - #tag + conceal_offset) + + if siglen + #tag > 80 then + write(string.rep(' ', tag_pad_len) .. tag) + write(fun.signature) + else + write(fmt('%s%s%s', fun.signature, string.rep(' ', tag_pad_len - siglen), tag)) + end +end + +--- Generates vimdoc for a Vimscript "eval" function. +--- @param f string +--- @param fun vim.EvalFn +--- @param write fun(line: string) +local function render_eval_doc(f, fun, write) + if fun.deprecated or not fun.signature then + return + end + + render_sig_and_tag(fun.name or f, not f:find('__%d+$'), fun, write) + + if not fun.desc then + return + end + + local params = process_params(fun.params) + local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0 + + local desc_l = split(vim.trim(fun.desc)) + for _, l in ipairs(desc_l) do + l = l:gsub('^ ', '') + if vim.startswith(l, '<') and not l:match('^<[^ \t]+>') then + write('<\t\t' .. l:sub(2)) + elseif l:match('^>[a-z0-9]*$') then + write(l) + else + write('\t\t' .. l) + end + end + + if #desc_l > 0 and not desc_l[#desc_l]:match('^<?$') then + write('') + end + + if #params > 0 then + write(util.md_to_vimdoc('Parameters: ~', 16, 16, TEXT_WIDTH)) + for i, param in ipairs(params) do + local pname, ptype = param[1], param[2] + local optional = (pname ~= '...' and i > req_args) and '?' or '' + local s = fmt('- %-14s (`%s%s`)', fmt('{%s}', pname), ptype, optional) + write(util.md_to_vimdoc(s, 16, 18, TEXT_WIDTH)) + end + write('') + end + + if fun.returns ~= false then + write(util.md_to_vimdoc('Return: ~', 16, 16, TEXT_WIDTH)) + local ret = ('(`%s`)'):format((fun.returns or 'any')) + ret = ret .. (fun.returns_desc and ' ' .. fun.returns_desc or '') + ret = util.md_to_vimdoc(ret, 18, 18, TEXT_WIDTH) + write(ret) + write('') + end +end + +--- @param d vim.option_defaults +--- @param vimdoc? boolean +--- @return string +local function render_option_default(d, vimdoc) + local dt --- @type integer|boolean|string|fun(): string + if d.if_false ~= nil then + dt = d.if_false + else + dt = d.if_true + end + + if vimdoc then + if d.doc then + return d.doc + end + if type(dt) == 'boolean' then + return dt and 'on' or 'off' + end + end + + if dt == '' or dt == nil or type(dt) == 'function' then + dt = d.meta + end + + local v --- @type string + if not vimdoc then + v = vim.inspect(dt) --[[@as string]] + else + v = type(dt) == 'string' and '"' .. dt .. '"' or tostring(dt) + end + + --- @type table<string, string|false> + local envvars = { + TMPDIR = false, + VIMRUNTIME = false, + XDG_CONFIG_HOME = vim.env.HOME .. '/.local/config', + XDG_DATA_HOME = vim.env.HOME .. '/.local/share', + XDG_STATE_HOME = vim.env.HOME .. '/.local/state', + } + + for name, default in pairs(envvars) do + local value = vim.env[name] or default + if value then + v = v:gsub(vim.pesc(value), '$' .. name) + end + end + + return v +end + +--- @param _f string +--- @param opt vim.option_meta +--- @param write fun(line: string) +local function render_option_meta(_f, opt, write) + write('') + for _, l in ipairs(split(norm_text(opt.desc))) do + write('--- ' .. l) + end + + if opt.type == 'string' and not opt.list and opt.values then + local values = {} --- @type string[] + for _, e in ipairs(opt.values) do + values[#values + 1] = fmt("'%s'", e) + end + write('--- @type ' .. table.concat(values, '|')) + else + write('--- @type ' .. OPTION_TYPES[opt.type]) + end + + write('vim.o.' .. opt.full_name .. ' = ' .. render_option_default(opt.defaults)) + if opt.abbreviation then + write('vim.o.' .. opt.abbreviation .. ' = vim.o.' .. opt.full_name) + end + + for _, s in pairs { + { 'wo', 'win' }, + { 'bo', 'buf' }, + { 'go', 'global' }, + } do + local id, scope = s[1], s[2] + if vim.list_contains(opt.scope, scope) or (id == 'go' and #opt.scope > 1) then + local pfx = 'vim.' .. id .. '.' + write(pfx .. opt.full_name .. ' = vim.o.' .. opt.full_name) + if opt.abbreviation then + write(pfx .. opt.abbreviation .. ' = ' .. pfx .. opt.full_name) + end + end + end +end + +--- @param _f string +--- @param opt vim.option_meta +--- @param write fun(line: string) +local function render_vvar_meta(_f, opt, write) + write('') + + local desc = split(norm_text(opt.desc)) + while desc[#desc]:match('^%s*$') do + desc[#desc] = nil + end + + for _, l in ipairs(desc) do + write('--- ' .. l) + end + + write('--- @type ' .. (opt.type or 'any')) + + if LUA_KEYWORDS[opt.full_name] then + write("vim.v['" .. opt.full_name .. "'] = ...") + else + write('vim.v.' .. opt.full_name .. ' = ...') + end +end + +--- @param s string[] +--- @return string +local function scope_to_doc(s) + local m = { + global = 'global', + buf = 'local to buffer', + win = 'local to window', + tab = 'local to tab page', + } + + if #s == 1 then + return m[s[1]] + end + assert(s[1] == 'global') + return 'global or ' .. m[s[2]] .. (s[2] ~= 'tab' and ' |global-local|' or '') +end + +-- @param o vim.option_meta +-- @return string +local function scope_more_doc(o) + if + vim.list_contains({ + 'bufhidden', + 'buftype', + 'filetype', + 'modified', + 'previewwindow', + 'readonly', + 'scroll', + 'syntax', + 'winfixheight', + 'winfixwidth', + }, o.full_name) + then + return ' |local-noglobal|' + end + + return '' +end + +--- @param x string +--- @return string +local function dedent(x) + local xs = split(x) + local leading_ws = xs[1]:match('^%s*') --[[@as string]] + local leading_ws_pat = '^' .. leading_ws + + for i in ipairs(xs) do + local strip_pat = xs[i]:match(leading_ws_pat) and leading_ws_pat or '^%s*' + xs[i] = xs[i]:gsub(strip_pat, '') + end + + return table.concat(xs, '\n') +end + +--- @return table<string,vim.option_meta> +local function get_option_meta() + local opts = require('nvim.options').options + local optinfo = vim.api.nvim_get_all_options_info() + local ret = {} --- @type table<string,vim.option_meta> + for _, o in ipairs(opts) do + local is_window_option = #o.scope == 1 and o.scope[1] == 'win' + local is_option_hidden = o.immutable and not o.varname and not is_window_option + if not is_option_hidden and o.desc then + if o.full_name == 'cmdheight' then + table.insert(o.scope, 'tab') + end + local r = vim.deepcopy(o) --[[@as vim.option_meta]] + r.desc = o.desc:gsub('^ ', ''):gsub('\n ', '\n') + r.defaults = r.defaults or {} + if r.defaults.meta == nil then + r.defaults.meta = optinfo[o.full_name].default + end + ret[o.full_name] = r + end + end + return ret +end + +--- @return table<string,vim.option_meta> +local function get_vvar_meta() + local info = require('nvim.vvars').vars + local ret = {} --- @type table<string,vim.option_meta> + for name, o in pairs(info) do + o.desc = dedent(o.desc) + o.full_name = name + ret[name] = o + end + return ret +end + +--- @param opt vim.option_meta +--- @return string[] +local function build_option_tags(opt) + --- @type string[] + local tags = { opt.full_name } + + tags[#tags + 1] = opt.abbreviation + if opt.type == 'boolean' then + for i = 1, #tags do + tags[#tags + 1] = 'no' .. tags[i] + end + end + + for i, t in ipairs(tags) do + tags[i] = "'" .. t .. "'" + end + + for _, t in ipairs(opt.tags or {}) do + tags[#tags + 1] = t + end + + for i, t in ipairs(tags) do + tags[i] = '*' .. t .. '*' + end + + return tags +end + +--- @param _f string +--- @param opt vim.option_meta +--- @param write fun(line: string) +local function render_option_doc(_f, opt, write) + local tags = build_option_tags(opt) + local tag_str = table.concat(tags, ' ') + local conceal_offset = 2 * (#tags - 1) + local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) + -- local pad = string.rep(' ', 80 - #tag_str + conceal_offset) + write(tag_pad .. tag_str) + + local name_str --- @type string + if opt.abbreviation then + name_str = fmt("'%s' '%s'", opt.full_name, opt.abbreviation) + else + name_str = fmt("'%s'", opt.full_name) + end + + local otype = opt.type == 'boolean' and 'boolean' or opt.type + if opt.defaults.doc or opt.defaults.if_true ~= nil or opt.defaults.meta ~= nil then + local v = render_option_default(opt.defaults, true) + local pad = string.rep('\t', math.max(1, math.ceil((24 - #name_str) / 8))) + if opt.defaults.doc then + local deflen = #fmt('%s%s%s (', name_str, pad, otype) + --- @type string + v = v:gsub('\n', '\n' .. string.rep(' ', deflen - 2)) + end + write(fmt('%s%s%s\t(default %s)', name_str, pad, otype, v)) + else + write(fmt('%s\t%s', name_str, otype)) + end + + write('\t\t\t' .. scope_to_doc(opt.scope) .. scope_more_doc(opt)) + for _, l in ipairs(split(opt.desc)) do + if l == '<' or l:match('^<%s') then + write(l) + else + write('\t' .. l:gsub('\\<', '<')) + end + end +end + +--- @param _f string +--- @param vvar vim.option_meta +--- @param write fun(line: string) +local function render_vvar_doc(_f, vvar, write) + local name = vvar.full_name + + local tags = { 'v:' .. name, name .. '-variable' } + if vvar.tags then + vim.list_extend(tags, vvar.tags) + end + + for i, t in ipairs(tags) do + tags[i] = '*' .. t .. '*' + end + + local tag_str = table.concat(tags, ' ') + local conceal_offset = 2 * (#tags - 1) + + local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8)) + write(tag_pad .. tag_str) + + local desc = split(vvar.desc) + + if (#desc == 1 or #desc == 2 and desc[2]:match('^%s*$')) and #name < 10 then + -- single line + write('v:' .. name .. '\t' .. desc[1]:gsub('^%s*', '')) + write('') + else + write('v:' .. name) + for _, l in ipairs(desc) do + if l == '<' or l:match('^<%s') then + write(l) + else + write('\t\t' .. l:gsub('\\<', '<')) + end + end + end +end + +--- @class nvim.gen_eval_files.elem +--- @field path string +--- @field from? string Skip lines in path until this pattern is reached. +--- @field funcs fun(): table<string, table> +--- @field render fun(f:string,obj:table,write:fun(line:string)) +--- @field header? string[] +--- @field footer? string[] + +--- @type nvim.gen_eval_files.elem[] +local CONFIG = { + { + path = 'runtime/lua/vim/_meta/vimfn.lua', + header = LUA_META_HEADER, + funcs = get_eval_meta, + render = render_eval_meta, + }, + { + path = 'runtime/lua/vim/_meta/api.lua', + header = LUA_API_META_HEADER, + funcs = get_api_meta, + render = render_api_meta, + }, + { + path = 'runtime/lua/vim/_meta/api_keysets.lua', + header = LUA_META_HEADER, + funcs = get_api_keysets_meta, + render = render_api_keyset_meta, + }, + { + path = 'runtime/doc/builtin.txt', + funcs = get_eval_meta, + render = render_eval_doc, + header = { + '*builtin.txt* Nvim', + '', + '', + '\t\t NVIM REFERENCE MANUAL', + '', + '', + 'Builtin functions\t\t*vimscript-functions* *builtin-functions*', + '', + 'For functions grouped by what they are used for see |function-list|.', + '', + '\t\t\t\t Type |gO| to see the table of contents.', + '==============================================================================', + '1. Details *builtin-function-details*', + '', + }, + footer = { + '==============================================================================', + '2. Matching a pattern in a String *string-match*', + '', + 'This is common between several functions. A regexp pattern as explained at', + '|pattern| is normally used to find a match in the buffer lines. When a', + 'pattern is used to find a match in a String, almost everything works in the', + 'same way. The difference is that a String is handled like it is one line.', + 'When it contains a "\\n" character, this is not seen as a line break for the', + 'pattern. It can be matched with a "\\n" in the pattern, or with ".". Example:', + '>vim', + '\tlet a = "aaaa\\nxxxx"', + '\techo matchstr(a, "..\\n..")', + '\t" aa', + '\t" xx', + '\techo matchstr(a, "a.x")', + '\t" a', + '\t" x', + '', + 'Don\'t forget that "^" will only match at the first character of the String and', + '"$" at the last character of the string. They don\'t match after or before a', + '"\\n".', + '', + ' vim:tw=78:ts=8:noet:ft=help:norl:', + }, + }, + { + path = 'runtime/lua/vim/_meta/options.lua', + header = LUA_OPTION_META_HEADER, + funcs = get_option_meta, + render = render_option_meta, + }, + { + path = 'runtime/doc/options.txt', + header = { '' }, + from = 'A jump table for the options with a short description can be found at |Q_op|.', + footer = { + ' vim:tw=78:ts=8:noet:ft=help:norl:', + }, + funcs = get_option_meta, + render = render_option_doc, + }, + { + path = 'runtime/lua/vim/_meta/vvars.lua', + header = LUA_VVAR_META_HEADER, + funcs = get_vvar_meta, + render = render_vvar_meta, + }, + { + path = 'runtime/doc/vvars.txt', + header = { '' }, + from = 'Type |gO| to see the table of contents.', + footer = { + ' vim:tw=78:ts=8:noet:ft=help:norl:', + }, + funcs = get_vvar_meta, + render = render_vvar_doc, + }, +} + +--- @param elem nvim.gen_eval_files.elem +local function render(elem) + print('Rendering ' .. elem.path) + local from_lines = {} --- @type string[] + local from = elem.from + if from then + for line in io.lines(elem.path) do + from_lines[#from_lines + 1] = line + if line:match(from) then + break + end + end + end + + local o = assert(io.open(elem.path, 'w')) + + --- @param l string + local function write(l) + local l1 = l:gsub('%s+$', '') + o:write(l1) + o:write('\n') + end + + for _, l in ipairs(from_lines) do + write(l) + end + + for _, l in ipairs(elem.header or {}) do + write(l) + end + + local funcs = elem.funcs() + + --- @type string[] + local fnames = vim.tbl_keys(funcs) + table.sort(fnames) + + for _, f in ipairs(fnames) do + elem.render(f, funcs[f], write) + end + + for _, l in ipairs(elem.footer or {}) do + write(l) + end + + o:close() +end + +local function main() + for _, c in ipairs(CONFIG) do + render(c) + end +end + +main() diff --git a/src/nvim/generators/gen_events.lua b/src/gen/gen_events.lua index 8c87815a74..77f766bb28 100644 --- a/src/nvim/generators/gen_events.lua +++ b/src/gen/gen_events.lua @@ -1,7 +1,7 @@ local fileio_enum_file = arg[1] local names_file = arg[2] -local auevents = require('auevents') +local auevents = require('nvim.auevents') local events = auevents.events local enum_tgt = io.open(fileio_enum_file, 'w') diff --git a/src/nvim/generators/gen_ex_cmds.lua b/src/gen/gen_ex_cmds.lua index e8d1aac182..6c03e8fc4d 100644 --- a/src/nvim/generators/gen_ex_cmds.lua +++ b/src/gen/gen_ex_cmds.lua @@ -11,7 +11,7 @@ local enumfile = io.open(enumfname, 'w') local defsfile = io.open(defsfname, 'w') local bit = require 'bit' -local ex_cmds = require('ex_cmds') +local ex_cmds = require('nvim.ex_cmds') local defs = ex_cmds.cmds local flags = ex_cmds.flags diff --git a/src/gen/gen_filetype.lua b/src/gen/gen_filetype.lua new file mode 100644 index 0000000000..18b53f1ea4 --- /dev/null +++ b/src/gen/gen_filetype.lua @@ -0,0 +1,209 @@ +local do_not_run = true +if do_not_run then + print([[ + This script was used to bootstrap the filetype patterns in runtime/lua/vim/filetype.lua. It + should no longer be used except for testing purposes. New filetypes, or changes to existing + filetypes, should be ported manually as part of the vim-patch process. + ]]) + return +end + +local filetype_vim = 'runtime/filetype.vim' +local filetype_lua = 'runtime/lua/vim/filetype.lua' + +local keywords = { + ['for'] = true, + ['or'] = true, + ['and'] = true, + ['end'] = true, + ['do'] = true, + ['if'] = true, + ['while'] = true, + ['repeat'] = true, +} + +local sections = { + extension = { str = {}, func = {} }, + filename = { str = {}, func = {} }, + pattern = { str = {}, func = {} }, +} + +local specialchars = '%*%?\\%$%[%]%{%}' + +local function add_pattern(pat, ft) + local ok = true + + -- Patterns that start or end with { or } confuse splitting on commas and make parsing harder, so just skip those + if not string.find(pat, '^%{') and not string.find(pat, '%}$') then + for part in string.gmatch(pat, '[^,]+') do + if not string.find(part, '[' .. specialchars .. ']') then + if type(ft) == 'string' then + sections.filename.str[part] = ft + else + sections.filename.func[part] = ft + end + elseif string.match(part, '^%*%.[^%./' .. specialchars .. ']+$') then + if type(ft) == 'string' then + sections.extension.str[part:sub(3)] = ft + else + sections.extension.func[part:sub(3)] = ft + end + else + if string.match(part, '^%*/[^' .. specialchars .. ']+$') then + -- For patterns matching */some/pattern we want to easily match files + -- with path /some/pattern, so include those in filename detection + if type(ft) == 'string' then + sections.filename.str[part:sub(2)] = ft + else + sections.filename.func[part:sub(2)] = ft + end + end + + if string.find(part, '^[%w-_.*?%[%]/]+$') then + local p = part:gsub('%.', '%%.'):gsub('%*', '.*'):gsub('%?', '.') + -- Insert into array to maintain order rather than setting + -- key-value directly + if type(ft) == 'string' then + sections.pattern.str[p] = ft + else + sections.pattern.func[p] = ft + end + else + ok = false + end + end + end + end + + return ok +end + +local function parse_line(line) + local pat, ft + pat, ft = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)') + if pat then + return add_pattern(pat, ft) + else + local func + pat, func = line:match('^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)') + if pat then + return add_pattern(pat, function() + return func + end) + end + end +end + +local unparsed = {} +local full_line +for line in io.lines(filetype_vim) do + local cont = string.match(line, '^%s*\\%s*(.*)$') + if cont then + full_line = full_line .. ' ' .. cont + else + if full_line then + if not parse_line(full_line) and string.find(full_line, '^%s*au%a* Buf') then + table.insert(unparsed, full_line) + end + end + full_line = line + end +end + +if #unparsed > 0 then + print('Failed to parse the following patterns:') + for _, v in ipairs(unparsed) do + print(v) + end +end + +local function add_item(indent, key, ft) + if type(ft) == 'string' then + if string.find(key, '%A') or keywords[key] then + key = string.format('["%s"]', key) + end + return string.format([[%s%s = "%s",]], indent, key, ft) + elseif type(ft) == 'function' then + local func = ft() + if string.find(key, '%A') or keywords[key] then + key = string.format('["%s"]', key) + end + -- Right now only a single argument is supported, which covers + -- everything in filetype.vim as of this writing + local arg = string.match(func, '%((.*)%)$') + func = string.gsub(func, '%(.*$', '') + if arg == '' then + -- Function with no arguments, call the function directly + return string.format([[%s%s = function() vim.fn["%s"]() end,]], indent, key, func) + elseif string.match(arg, [[^(["']).*%1$]]) then + -- String argument + if func == 's:StarSetf' then + return string.format([[%s%s = starsetf(%s),]], indent, key, arg) + else + return string.format([[%s%s = function() vim.fn["%s"](%s) end,]], indent, key, func, arg) + end + elseif string.find(arg, '%(') then + -- Function argument + return string.format( + [[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]], + indent, + key, + func, + arg + ) + else + assert(false, arg) + end + end +end + +do + local lines = {} + local start = false + for line in io.lines(filetype_lua) do + if line:match('^%s+-- END [A-Z]+$') then + start = false + end + + if not start then + table.insert(lines, line) + end + + local indent, section = line:match('^(%s+)-- BEGIN ([A-Z]+)$') + if section then + start = true + local t = sections[string.lower(section)] + + local sorted = {} + for k, v in pairs(t.str) do + table.insert(sorted, { [k] = v }) + end + + table.sort(sorted, function(a, b) + return a[next(a)] < b[next(b)] + end) + + for _, v in ipairs(sorted) do + local k = next(v) + table.insert(lines, add_item(indent, k, v[k])) + end + + sorted = {} + for k, v in pairs(t.func) do + table.insert(sorted, { [k] = v }) + end + + table.sort(sorted, function(a, b) + return next(a) < next(b) + end) + + for _, v in ipairs(sorted) do + local k = next(v) + table.insert(lines, add_item(indent, k, v[k])) + end + end + end + local f = io.open(filetype_lua, 'w') + f:write(table.concat(lines, '\n') .. '\n') + f:close() +end diff --git a/src/gen/gen_help_html.lua b/src/gen/gen_help_html.lua new file mode 100644 index 0000000000..53a65fd65f --- /dev/null +++ b/src/gen/gen_help_html.lua @@ -0,0 +1,1491 @@ +--- Converts Nvim :help files to HTML. Validates |tag| links and document syntax (parser errors). +-- +-- USAGE (For CI/local testing purposes): Simply `make lintdoc` or `scripts/lintdoc.lua`, which +-- basically does the following: +-- 1. :helptags ALL +-- 2. nvim -V1 -es +"lua require('scripts.gen_help_html').run_validate()" +q +-- 3. nvim -V1 -es +"lua require('scripts.gen_help_html').test_gen()" +q +-- +-- USAGE (GENERATE HTML): +-- 1. `:helptags ALL` first; this script depends on vim.fn.taglist(). +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./runtime/doc', 'target/dir/')" +q +-- - Read the docstring at gen(). +-- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 +-- 4. Visit http://localhost:4000/…/help.txt.html +-- +-- USAGE (VALIDATE): +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate('./runtime/doc')" +q +-- - validate() is 10x faster than gen(), so it is used in CI. +-- +-- SELF-TEST MODE: +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +q +-- +-- NOTES: +-- * This script is used by the automation repo: https://github.com/neovim/doc +-- * :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). +-- * gen() and validate() are the primary (programmatic) entrypoints. validate() only exists +-- because gen() is too slow (~1 min) to run in per-commit CI. +-- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. +-- * visit_validate() is the core function used by validate(). +-- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. + +local tagmap = nil ---@type table<string, string> +local helpfiles = nil ---@type string[] +local invalid_links = {} ---@type table<string, any> +local invalid_urls = {} ---@type table<string, any> +local invalid_spelling = {} ---@type table<string, table<string, string>> +local spell_dict = { + Neovim = 'Nvim', + NeoVim = 'Nvim', + neovim = 'Nvim', + lua = 'Lua', + VimL = 'Vimscript', + vimL = 'Vimscript', + viml = 'Vimscript', + ['tree-sitter'] = 'treesitter', + ['Tree-sitter'] = 'Treesitter', +} +--- specify the list of keywords to ignore (i.e. allow), or true to disable spell check completely. +--- @type table<string, true|string[]> +local spell_ignore_files = { + ['credits.txt'] = { 'Neovim' }, + ['news.txt'] = { 'tree-sitter' }, -- in news, may refer to the upstream "tree-sitter" library + ['news-0.10.txt'] = { 'tree-sitter' }, +} +local language = nil + +local M = {} + +-- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). +-- All other files are "legacy" files which require fixed-width layout. +local new_layout = { + ['api.txt'] = true, + ['lsp.txt'] = true, + ['channel.txt'] = true, + ['deprecated.txt'] = true, + ['develop.txt'] = true, + ['dev_style.txt'] = true, + ['dev_theme.txt'] = true, + ['dev_tools.txt'] = true, + ['dev_vimpatch.txt'] = true, + ['editorconfig.txt'] = true, + ['faq.txt'] = true, + ['gui.txt'] = true, + ['intro.txt'] = true, + ['lua.txt'] = true, + ['luaref.txt'] = true, + ['news.txt'] = true, + ['news-0.9.txt'] = true, + ['news-0.10.txt'] = true, + ['nvim.txt'] = true, + ['provider.txt'] = true, + ['tui.txt'] = true, + ['ui.txt'] = true, + ['vim_diff.txt'] = true, +} + +-- Map of new:old pages, to redirect renamed pages. +local redirects = { + ['credits'] = 'backers', + ['tui'] = 'term', + ['terminal'] = 'nvim_terminal_emulator', +} + +-- TODO: These known invalid |links| require an update to the relevant docs. +local exclude_invalid = { + ["'string'"] = 'eval.txt', + Query = 'treesitter.txt', + matchit = 'vim_diff.txt', + ['set!'] = 'treesitter.txt', +} + +-- False-positive "invalid URLs". +local exclude_invalid_urls = { + ['http://'] = 'usr_23.txt', + ['http://.'] = 'usr_23.txt', + ['http://aspell.net/man-html/Affix-Compression.html'] = 'spell.txt', + ['http://aspell.net/man-html/Phonetic-Code.html'] = 'spell.txt', + ['http://canna.sourceforge.jp/'] = 'mbyte.txt', + ['http://gnuada.sourceforge.net'] = 'ft_ada.txt', + ['http://lua-users.org/wiki/StringLibraryTutorial'] = 'lua.txt', + ['http://michael.toren.net/code/'] = 'pi_tar.txt', + ['http://papp.plan9.de'] = 'syntax.txt', + ['http://wiki.services.openoffice.org/wiki/Dictionaries'] = 'spell.txt', + ['http://www.adapower.com'] = 'ft_ada.txt', + ['http://www.jclark.com/'] = 'quickfix.txt', + ['http://oldblog.antirez.com/post/redis-and-scripting.html'] = 'faq.txt', +} + +-- Deprecated, brain-damaged files that I don't care about. +local ignore_errors = { + ['pi_netrw.txt'] = true, + ['credits.txt'] = true, +} + +local function tofile(fname, text) + local f = io.open(fname, 'w') + if not f then + error(('failed to write: %s'):format(f)) + else + f:write(text) + f:close() + end +end + +---@type fun(s: string): string +local function html_esc(s) + return (s:gsub('&', '&'):gsub('<', '<'):gsub('>', '>')) +end + +local function url_encode(s) + -- Credit: tpope / vim-unimpaired + -- NOTE: these chars intentionally *not* escaped: ' ( ) + return vim.fn.substitute( + vim.fn.iconv(s, 'latin1', 'utf-8'), + [=[[^A-Za-z0-9()'_.~-]]=], + [=[\="%".printf("%02X",char2nr(submatch(0)))]=], + 'g' + ) +end + +local function expandtabs(s) + return s:gsub('\t', (' '):rep(8)) --[[ @as string ]] +end + +local function to_titlecase(s) + local text = '' + for w in vim.gsplit(s, '[ \t]+') do + text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2)) + end + return text +end + +local function to_heading_tag(text) + -- Prepend "_" to avoid conflicts with actual :help tags. + return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown' +end + +local function basename_noext(f) + return vim.fs.basename(f:gsub('%.txt', '')) +end + +local function is_blank(s) + return not not s:find([[^[\t ]*$]]) +end + +---@type fun(s: string, dir?:0|1|2): string +local function trim(s, dir) + return vim.fn.trim(s, '\r\t\n ', dir or 0) +end + +--- Removes common punctuation from URLs. +--- +--- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc +--- +--- @param url string +--- @return string, string (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. +local function fix_url(url) + local removed_chars = '' + local fixed_url = url + -- Remove up to one of each char from end of the URL, in this order. + for _, c in ipairs({ '.', ')' }) do + if fixed_url:sub(-1) == c then + removed_chars = c .. removed_chars + fixed_url = fixed_url:sub(1, -2) + end + end + return fixed_url, removed_chars +end + +--- Checks if a given line is a "noise" line that doesn't look good in HTML form. +local function is_noise(line, noise_lines) + if + -- First line is always noise. + (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0) + or line:find('Type .*gO.* to see the table of contents') + -- Title line of traditional :help pages. + -- Example: "NVIM REFERENCE MANUAL by ..." + or line:find([[^%s*N?VIM[ \t]*REFERENCE[ \t]*MANUAL]]) + -- First line of traditional :help pages. + -- Example: "*api.txt* Nvim" + or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') + -- modeline + -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" + or line:find('^%s*vim?%:.*ft=help') + or line:find('^%s*vim?%:.*filetype=help') + or line:find('[*>]local%-additions[*<]') + then + -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + table.insert(noise_lines or {}, line) + return true + end + return false +end + +--- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. +--- @return string +local function get_bug_url_vimdoc(fname, to_fname, sample_text) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ( + 'https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + .. vim.fs.basename(fname) + .. '+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' + .. this_url + .. '%0D%0DContext%3A%0D%0D%60%60%60%0D' + .. url_encode(sample_text) + .. '%0D%60%60%60' + ) + return bug_url +end + +--- Creates a github issue URL at neovim/neovim with prefilled content. +--- @return string +local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ( + 'https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' + .. vim.fs.basename(fname) + .. '+&body=%60gen_help_html.lua%60+problem+at%3A+' + .. this_url + .. '%0D' + .. (token_name and '+unhandled+token%3A+%60' .. token_name .. '%60' or '') + .. '%0DContext%3A%0D%0D%60%60%60%0D' + .. url_encode(sample_text) + .. '%0D%60%60%60' + ) + return bug_url +end + +--- Gets a "foo.html" name from a "foo.txt" helpfile name. +local function get_helppage(f) + if not f then + return nil + end + -- Special case: help.txt is the "main landing page" of :help files, not index.txt. + if f == 'index.txt' then + return 'vimindex.html' + elseif f == 'help.txt' then + return 'index.html' + end + + return (f:gsub('%.txt$', '')) .. '.html' +end + +--- Counts leading spaces (tab=8) to decide the indent size of multiline text. +--- +--- Blank lines (empty or whitespace-only) are ignored. +local function get_indent(s) + local min_indent = nil + for line in vim.gsplit(s, '\n') do + if line and not is_blank(line) then + local ws = expandtabs(line:match('^%s+') or '') + min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent + end + end + return min_indent or 0 +end + +--- Removes the common indent level, after expanding tabs to 8 spaces. +local function trim_indent(s) + local indent_size = get_indent(s) + local trimmed = '' + for line in vim.gsplit(s, '\n') do + line = expandtabs(line) + trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1)) + end + return trimmed:sub(1, -2) +end + +--- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. +---@param node TSNode +---@param bufnr integer +---@param offset integer +local function getbuflinestr(node, bufnr, offset) + local line1, _, line2, _ = node:range() + line1 = line1 - offset + line2 = line2 + offset + local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1) + return table.concat(lines, '\n') +end + +--- Gets the whitespace just before `node` from the raw buffer text. +--- Needed for preformatted `old` lines. +---@param node TSNode +---@param bufnr integer +---@return string +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + ---@type string + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + +local function get_tagname(node, bufnr) + local text = vim.treesitter.get_node_text(node, bufnr) + local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') + and ("'%s'"):format(text) + or text + local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" + return helppage, tag +end + +--- Returns true if the given invalid tagname is a false positive. +local function ignore_invalid(s) + return not not ( + exclude_invalid[s] + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + or s:find('===') + or s:find('%-%-%-') + ) +end + +local function ignore_parse_error(fname, s) + if ignore_errors[vim.fs.basename(fname)] then + return true + end + -- Ignore parse errors for unclosed tag. + -- This is common in vimdocs and is treated as plaintext by :help. + return s:find("^[`'|*]") +end + +---@param node TSNode +local function has_ancestor(node, ancestor_name) + local p = node ---@type TSNode? + while p do + p = p:parent() + if not p or p:type() == 'help_file' then + break + elseif p:type() == ancestor_name then + return true + end + end + return false +end + +--- Gets the first matching child node matching `name`. +---@param node TSNode +local function first(node, name) + for c, _ in node:iter_children() do + if c:named() and c:type() == name then + return c + end + end + return nil +end + +local function validate_link(node, bufnr, fname) + local helppage, tagname = get_tagname(node:child(1), bufnr) + local ignored = false + if not tagmap[tagname] then + ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname) + if not ignored then + invalid_links[tagname] = vim.fs.basename(fname) + end + end + return helppage, tagname, ignored +end + +--- TODO: port the logic from scripts/check_urls.vim +local function validate_url(text, fname) + local ignored = false + if ignore_errors[vim.fs.basename(fname)] then + ignored = true + elseif text:find('http%:') and not exclude_invalid_urls[text] then + invalid_urls[text] = vim.fs.basename(fname) + end + return ignored +end + +--- Traverses the tree at `root` and checks that |tag| links point to valid helptags. +---@param root TSNode +---@param level integer +---@param lang_tree TSTree +---@param opt table +---@param stats table +local function visit_validate(root, level, lang_tree, opt, stats) + level = level or 0 + local node_name = (root.named and root:named()) and root:type() or nil + -- Parent kind (string). + local parent = root:parent() and root:parent():type() or nil + local toplevel = level < 1 + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) + end + local text = trim(node_text()) + + if root:child_count() > 0 then + for node, _ in root:iter_children() do + if node:named() then + visit_validate(node, level + 1, lang_tree, opt, stats) + end + end + end + + if node_name == 'ERROR' then + if ignore_parse_error(opt.fname, text) then + return + end + -- Store the raw text to give context to the error report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 0) or '[top level!]' + -- Flatten the sample text to a single, truncated line. + sample_text = vim.trim(sample_text):gsub('[\t\n]', ' '):sub(1, 80) + table.insert(stats.parse_errors, sample_text) + elseif + (node_name == 'word' or node_name == 'uppercase_name') + and (not vim.tbl_contains({ 'codespan', 'taglink', 'tag' }, parent)) + then + local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. + local fname_basename = assert(vim.fs.basename(opt.fname)) + if spell_dict[text_nopunct] then + local should_ignore = ( + spell_ignore_files[fname_basename] == true + or vim.tbl_contains( + (spell_ignore_files[fname_basename] or {}) --[[ @as string[] ]], + text_nopunct + ) + ) + if not should_ignore then + invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} + invalid_spelling[text_nopunct][fname_basename] = node_text(root:parent()) + end + end + elseif node_name == 'url' then + local fixed_url, _ = fix_url(trim(text)) + validate_url(fixed_url, opt.fname) + elseif node_name == 'taglink' or node_name == 'optionlink' then + local _, _, _ = validate_link(root, opt.buf, opt.fname) + end +end + +-- Fix tab alignment issues caused by concealed characters like |, `, * in tags +-- and code blocks. +---@param text string +---@param next_node_text string +local function fix_tab_after_conceal(text, next_node_text) + -- Vim tabs take into account the two concealed characters even though they + -- are invisible, so we need to add back in the two spaces if this is + -- followed by a tab to make the tab alignment to match Vim's behavior. + if string.sub(next_node_text, 1, 1) == '\t' then + text = text .. ' ' + end + return text +end + +---@class (exact) nvim.gen_help_html.heading +---@field name string +---@field subheadings nvim.gen_help_html.heading[] +---@field tag string + +-- Generates HTML from node `root` recursively. +---@param root TSNode +---@param level integer +---@param lang_tree TSTree +---@param headings nvim.gen_help_html.heading[] +---@param opt table +---@param stats table +local function visit_node(root, level, lang_tree, headings, opt, stats) + level = level or 0 + + local node_name = (root.named and root:named()) and root:type() or nil + -- Previous sibling kind (string). + local prev = root:prev_sibling() + and (root:prev_sibling().named and root:prev_sibling():named()) + and root:prev_sibling():type() + or nil + -- Next sibling kind (string). + local next_ = root:next_sibling() + and (root:next_sibling().named and root:next_sibling():named()) + and root:next_sibling():type() + or nil + -- Parent kind (string). + local parent = root:parent() and root:parent():type() or nil + -- Gets leading whitespace of `node`. + local function ws(node) + node = node or root + local ws_ = getws(node, opt.buf) + -- XXX: first node of a (line) includes whitespace, even after + -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ? + if ws_ == '' then + ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or '' + end + return ws_ + end + local function node_text(node, ws_) + node = node or root + ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or '' + return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) + end + + local text = '' + local trimmed ---@type string + if root:named_child_count() == 0 or node_name == 'ERROR' then + text = node_text() + trimmed = html_esc(trim(text)) + text = html_esc(text) + else + -- Process children and join them with whitespace. + for node, _ in root:iter_children() do + if node:named() then + local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) + text = string.format('%s%s', text, r) + end + end + trimmed = trim(text) + end + + if node_name == 'help_file' then -- root node + return text + elseif node_name == 'url' then + local fixed_url, removed_chars = fix_url(trimmed) + return ('%s<a href="%s">%s</a>%s'):format(ws(), fixed_url, fixed_url, removed_chars) + elseif node_name == 'word' or node_name == 'uppercase_name' then + return text + elseif node_name == 'note' then + return ('<b>%s</b>'):format(text) + elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then + if is_noise(text, stats.noise_lines) then + return '' -- Discard common "noise" lines. + end + -- Remove tags from ToC text. + local heading_node = first(root, 'heading') + local hname = trim(node_text(heading_node):gsub('%*.*%*', '')) + if not heading_node or hname == '' then + return '' -- Spurious "===" or "---" in the help doc. + end + + -- Generate an anchor id from the heading text. + local tagname = to_heading_tag(hname) + if node_name == 'h1' or #headings == 0 then + ---@type nvim.gen_help_html.heading + local heading = { name = hname, subheadings = {}, tag = tagname } + headings[#headings + 1] = heading + else + table.insert( + headings[#headings].subheadings, + { name = hname, subheadings = {}, tag = tagname } + ) + end + local el = node_name == 'h1' and 'h2' or 'h3' + return ('<%s id="%s" class="help-heading">%s</%s>\n'):format(el, tagname, trimmed, el) + elseif node_name == 'heading' then + return trimmed + elseif node_name == 'column_heading' or node_name == 'column_name' then + if root:has_error() then + return text + end + return ('<div class="help-column_heading">%s</div>'):format(text) + elseif node_name == 'block' then + if is_blank(text) then + return '' + end + if opt.old then + -- XXX: Treat "old" docs as preformatted: they use indentation for layout. + -- Trim trailing newlines to avoid too much whitespace between divs. + return ('<div class="old-help-para">%s</div>\n'):format(trim(text, 2)) + end + return string.format('<div class="help-para">\n%s\n</div>\n', text) + elseif node_name == 'line' then + if + (parent ~= 'codeblock' or parent ~= 'code') + and (is_blank(text) or is_noise(text, stats.noise_lines)) + then + return '' -- Discard common "noise" lines. + end + -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. + local div = opt.old + and root:child(0) + and vim.list_contains({ 'column_heading', 'h1', 'h2', 'h3' }, root:child(0):type()) + return string.format('%s%s', div and trim(text) or text, div and '' or '\n') + elseif node_name == 'line_li' then + local sib = root:prev_sibling() + local prev_li = sib and sib:type() == 'line_li' + + if not prev_li then + opt.indent = 1 + else + -- The previous listitem _sibling_ is _logically_ the _parent_ if it is indented less. + local parent_indent = get_indent(node_text(sib)) + local this_indent = get_indent(node_text()) + if this_indent > parent_indent then + opt.indent = opt.indent + 1 + elseif this_indent < parent_indent then + opt.indent = math.max(1, opt.indent - 1) + end + end + local margin = opt.indent == 1 and '' or ('margin-left: %drem;'):format((1.5 * opt.indent)) + + return string.format('<div class="help-li" style="%s">%s</div>', margin, text) + elseif node_name == 'taglink' or node_name == 'optionlink' then + local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname) + if ignored then + return text + end + local s = ('%s<a href="%s#%s">%s</a>'):format( + ws(), + helppage, + url_encode(tagname), + html_esc(tagname) + ) + if opt.old and node_name == 'taglink' then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + return s + elseif vim.list_contains({ 'codespan', 'keycode' }, node_name) then + if root:has_error() then + return text + end + local s = ('%s<code>%s</code>'):format(ws(), trimmed) + if opt.old and node_name == 'codespan' then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + return s + elseif node_name == 'argument' then + return ('%s<code>{%s}</code>'):format(ws(), text) + elseif node_name == 'codeblock' then + return text + elseif node_name == 'language' then + language = node_text(root) + return '' + elseif node_name == 'code' then -- Highlighted codeblock (child). + if is_blank(text) then + return '' + end + local code ---@type string + if language then + code = ('<pre><code class="language-%s">%s</code></pre>'):format( + language, + trim(trim_indent(text), 2) + ) + language = nil + else + code = ('<pre>%s</pre>'):format(trim(trim_indent(text), 2)) + end + return code + elseif node_name == 'tag' then -- anchor, h4 pseudo-heading + if root:has_error() then + return text + end + local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent) + local h4 = not in_heading and not next_ and get_indent(node_text()) > 8 -- h4 pseudo-heading + local cssclass = h4 and 'help-tag-right' or 'help-tag' + local tagname = node_text(root:child(1), false) + if vim.tbl_count(stats.first_tags) < 2 then + -- Force the first 2 tags in the doc to be anchored at the main heading. + table.insert(stats.first_tags, tagname) + return '' + end + local el = 'span' + local encoded_tagname = url_encode(tagname) + local s = ('%s<%s id="%s" class="%s"><a href="#%s">%s</a></%s>'):format( + ws(), + el, + encoded_tagname, + cssclass, + encoded_tagname, + trimmed, + el + ) + if opt.old then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + + if in_heading and prev ~= 'tag' then + -- Start the <span> container for tags in a heading. + -- This makes "justify-content:space-between" right-align the tags. + -- <h2>foo bar<span>tag1 tag2</span></h2> + return string.format('<span class="help-heading-tags">%s', s) + elseif in_heading and next_ == nil then + -- End the <span> container for tags in a heading. + return string.format('%s</span>', s) + end + return s .. (h4 and '<br>' or '') -- HACK: <br> avoids h4 pseudo-heading mushing with text. + elseif node_name == 'delimiter' or node_name == 'modeline' then + return '' + elseif node_name == 'ERROR' then + if ignore_parse_error(opt.fname, trimmed) then + return text + end + + -- Store the raw text to give context to the bug report. + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + return ('<a class="parse-error" target="_blank" title="Report bug... (parse error)" href="%s">%s</a>'):format( + get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), + trimmed + ) + else -- Unknown token. + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' + return ('<a class="unknown-token" target="_blank" title="Report bug... (unhandled token "%s")" href="%s">%s</a>'):format( + node_name, + get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), + trimmed + ), + ('unknown-token:"%s"'):format(node_name) + end +end + +--- @param dir string e.g. '$VIMRUNTIME/doc' +--- @param include string[]|nil +--- @return string[] +local function get_helpfiles(dir, include) + local rv = {} + for f, type in vim.fs.dir(dir) do + if + vim.endswith(f, '.txt') + and type == 'file' + and (not include or vim.list_contains(include, f)) + then + local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') + table.insert(rv, fullpath) + end + end + return rv +end + +--- Populates the helptags map. +local function get_helptags(help_dir) + local m = {} + -- Load a random help file to convince taglist() to do its job. + vim.cmd(string.format('split %s/api.txt', help_dir)) + vim.cmd('lcd %:p:h') + for _, item in ipairs(vim.fn.taglist('.*')) do + if vim.endswith(item.filename, '.txt') then + m[item.name] = item.filename + end + end + vim.cmd('q!') + return m +end + +--- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system. +local function ensure_runtimepath() + if not vim.o.runtimepath:find('build/lib/nvim/') then + vim.cmd [[set runtimepath^=./build/lib/nvim/]] + end +end + +--- Opens `fname` (or `text`, if given) in a buffer and gets a treesitter parser for the buffer contents. +--- +--- @param fname string :help file to parse +--- @param text string? :help file contents +--- @param parser_path string? path to non-default vimdoc.so +--- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr) +local function parse_buf(fname, text, parser_path) + local buf ---@type integer + if text then + vim.cmd('split new') -- Text contents. + vim.api.nvim_put(vim.split(text, '\n'), '', false, false) + vim.cmd('setfiletype help') + -- vim.treesitter.language.add('vimdoc') + buf = vim.api.nvim_get_current_buf() + elseif type(fname) == 'string' then + vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename. + buf = vim.api.nvim_get_current_buf() + else + -- Left for debugging + ---@diagnostic disable-next-line: no-unknown + buf = fname + vim.cmd('sbuffer ' .. tostring(fname)) -- Buffer number. + end + if parser_path then + vim.treesitter.language.add('vimdoc', { path = parser_path }) + end + local lang_tree = assert(vim.treesitter.get_parser(buf, nil, { error = false })) + return lang_tree, buf +end + +--- Validates one :help file `fname`: +--- - checks that |tag| links point to valid helptags. +--- - recursively counts parse errors ("ERROR" nodes) +--- +--- @param fname string help file to validate +--- @param parser_path string? path to non-default vimdoc.so +--- @return { invalid_links: number, parse_errors: string[] } +local function validate_one(fname, parser_path) + local stats = { + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname, nil, parser_path) + for _, tree in ipairs(lang_tree:trees()) do + visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats) + end + lang_tree:destroy() + vim.cmd.close() + return stats +end + +--- Generates HTML from one :help file `fname` and writes the result to `to_fname`. +--- +--- @param fname string Source :help file. +--- @param text string|nil Source :help file contents, or nil to read `fname`. +--- @param to_fname string Destination .html file +--- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) +--- @param parser_path string? path to non-default vimdoc.so +--- +--- @return string html +--- @return table stats +local function gen_one(fname, text, to_fname, old, commit, parser_path) + local stats = { + noise_lines = {}, + parse_errors = {}, + first_tags = {}, -- Track the first few tags in doc. + } + local lang_tree, buf = parse_buf(fname, text, parser_path) + ---@type nvim.gen_help_html.heading[] + local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. + local title = to_titlecase(basename_noext(fname)) + + local html = ([[ + <!DOCTYPE html> + <html> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="description" content="Neovim user documentation"> + + <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ --> + <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@docsearch/css@3" /> + <link rel="preconnect" href="https://X185E15FPG-dsn.algolia.net" crossorigin /> + + <link href="/css/bootstrap.min.css" rel="stylesheet"> + <link href="/css/main.css" rel="stylesheet"> + <link href="help.css" rel="stylesheet"> + <link href="/highlight/styles/neovim.min.css" rel="stylesheet"> + + <script src="/highlight/highlight.min.js"></script> + <script>hljs.highlightAll();</script> + <title>%s - Neovim docs</title> + </head> + <body> + ]]):format(title) + + local logo_svg = [[ + <svg xmlns="http://www.w3.org/2000/svg" role="img" width="173" height="50" viewBox="0 0 742 214" aria-label="Neovim"> + <title>Neovim</title> + <defs> + <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="a"> + <stop stop-color="#16B0ED" stop-opacity=".8" offset="0%" /> + <stop stop-color="#0F59B2" stop-opacity=".837" offset="100%" /> + </linearGradient> + <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="b"> + <stop stop-color="#7DB643" offset="0%" /> + <stop stop-color="#367533" offset="100%" /> + </linearGradient> + <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="c"> + <stop stop-color="#88C649" stop-opacity=".8" offset="0%" /> + <stop stop-color="#439240" stop-opacity=".84" offset="100%" /> + </linearGradient> + </defs> + <g fill="none" fill-rule="evenodd"> + <path + d="M.027 45.459L45.224-.173v212.171L.027 166.894V45.459z" + fill="url(#a)" + transform="translate(1 1)" + /> + <path + d="M129.337 45.89L175.152-.149l-.928 212.146-45.197-45.104.31-121.005z" + fill="url(#b)" + transform="matrix(-1 0 0 1 305 1)" + /> + <path + d="M45.194-.137L162.7 179.173l-32.882 32.881L12.25 33.141 45.194-.137z" + fill="url(#c)" + transform="translate(1 1)" + /> + <path + d="M46.234 84.032l-.063 7.063-36.28-53.563 3.36-3.422 32.983 49.922z" + fill-opacity=".13" + fill="#000" + /> + <g fill="#444"> + <path + d="M227 154V64.44h4.655c1.55 0 2.445.75 2.685 2.25l.806 13.502c4.058-5.16 8.786-9.316 14.188-12.466 5.4-3.15 11.413-4.726 18.037-4.726 4.893 0 9.205.781 12.935 2.34 3.729 1.561 6.817 3.811 9.264 6.751 2.448 2.942 4.297 6.48 5.55 10.621 1.253 4.14 1.88 8.821 1.88 14.042V154h-8.504V96.754c0-8.402-1.91-14.987-5.729-19.757-3.82-4.771-9.667-7.156-17.544-7.156-5.851 0-11.28 1.516-16.292 4.545-5.013 3.032-9.489 7.187-13.427 12.467V154H227zM350.624 63c5.066 0 9.755.868 14.069 2.605 4.312 1.738 8.052 4.268 11.219 7.592s5.638 7.412 7.419 12.264C385.11 90.313 386 95.883 386 102.17c0 1.318-.195 2.216-.588 2.696-.393.48-1.01.719-1.851.719h-64.966v1.70c0 6.708.784 12.609 2.353 17.7 1.567 5.09 3.8 9.357 6.695 12.802 2.895 3.445 6.393 6.034 10.495 7.771 4.1 1.738 8.686 2.606 13.752 2.606 4.524 0 8.446-.494 11.762-1.483 3.317-.988 6.108-2.097 8.37-3.324 2.261-1.227 4.056-2.336 5.383-3.324 1.326-.988 2.292-1.482 2.895-1.482.784 0 1.388.3 1.81.898l2.352 2.875c-1.448 1.797-3.362 3.475-5.745 5.031-2.383 1.558-5.038 2.891-7.962 3.998-2.926 1.109-6.062 1.991-9.41 2.65a52.21 52.21 0 01-10.088.989c-6.152 0-11.762-1.064-16.828-3.19-5.067-2.125-9.415-5.225-13.043-9.298-3.63-4.074-6.435-9.06-8.415-14.96C310.99 121.655 310 114.9 310 107.294c0-6.408.92-12.323 2.76-17.744 1.84-5.421 4.493-10.093 7.961-14.016 3.467-3.922 7.72-6.991 12.758-9.209C338.513 64.11 344.229 63 350.624 63zm.573 6c-4.696 0-8.904.702-12.623 2.105-3.721 1.404-6.936 3.421-9.65 6.053-2.713 2.631-4.908 5.79-6.586 9.474S319.55 94.439 319 99h60c0-4.679-.672-8.874-2.013-12.588-1.343-3.712-3.232-6.856-5.67-9.43-2.44-2.571-5.367-4.545-8.782-5.92-3.413-1.374-7.192-2.062-11.338-2.062zM435.546 63c6.526 0 12.368 1.093 17.524 3.28 5.154 2.186 9.5 5.286 13.04 9.298 3.538 4.013 6.238 8.85 8.099 14.51 1.861 5.66 2.791 11.994 2.791 19.002 0 7.008-.932 13.327-2.791 18.957-1.861 5.631-4.561 10.452-8.099 14.465-3.54 4.012-7.886 7.097-13.04 9.254-5.156 2.156-10.998 3.234-17.524 3.234-6.529 0-12.369-1.078-17.525-3.234-5.155-2.157-9.517-5.242-13.085-9.254-3.57-4.013-6.285-8.836-8.145-14.465-1.861-5.63-2.791-11.95-2.791-18.957 0-7.008.93-13.342 2.791-19.002 1.861-5.66 4.576-10.496 8.145-14.51 3.568-4.012 7.93-7.112 13.085-9.299C423.177 64.094 429.017 63 435.546 63zm-.501 86c5.341 0 10.006-.918 13.997-2.757 3.99-1.838 7.32-4.474 9.992-7.909 2.67-3.435 4.664-7.576 5.986-12.428 1.317-4.85 1.98-10.288 1.98-16.316 0-5.965-.66-11.389-1.98-16.27-1.322-4.88-3.316-9.053-5.986-12.519-2.67-3.463-6-6.13-9.992-7.999-3.991-1.867-8.657-2.802-13.997-2.802s-10.008.935-13.997 2.802c-3.991 1.87-7.322 4.536-9.992 8-2.671 3.465-4.68 7.637-6.03 12.518-1.35 4.881-2.026 10.305-2.026 16.27 0 6.026.675 11.465 2.025 16.316 1.35 4.852 3.36 8.993 6.031 12.428 2.67 3.435 6 6.07 9.992 7.91 3.99 1.838 8.656 2.756 13.997 2.756z" + fill="currentColor" + /> + <path + d="M530.57 152h-20.05L474 60h18.35c1.61 0 2.967.39 4.072 1.166 1.103.778 1.865 1.763 2.283 2.959l17.722 49.138a92.762 92.762 0 012.551 8.429c.686 2.751 1.298 5.5 1.835 8.25.537-2.75 1.148-5.499 1.835-8.25a77.713 77.713 0 012.64-8.429l18.171-49.138c.417-1.196 1.164-2.181 2.238-2.96 1.074-.776 2.356-1.165 3.849-1.165H567l-36.43 92zM572 61h23v92h-23zM610 153V60.443h13.624c2.887 0 4.78 1.354 5.682 4.06l1.443 6.856a52.7 52.7 0 015.097-4.962 32.732 32.732 0 015.683-3.879 30.731 30.731 0 016.496-2.57c2.314-.632 4.855-.948 7.624-.948 5.832 0 10.63 1.579 14.39 4.736 3.758 3.157 6.57 7.352 8.434 12.585 1.444-3.068 3.248-5.698 5.413-7.894 2.165-2.194 4.541-3.984 7.127-5.367a32.848 32.848 0 018.254-3.068 39.597 39.597 0 018.796-.992c5.111 0 9.653.783 13.622 2.345 3.97 1.565 7.307 3.849 10.014 6.857 2.706 3.007 4.766 6.675 6.18 11.005C739.29 83.537 740 88.5 740 94.092V153h-22.284V94.092c0-5.894-1.294-10.329-3.878-13.306-2.587-2.977-6.376-4.465-11.368-4.465-2.286 0-4.404.391-6.358 1.172a15.189 15.189 0 00-5.144 3.383c-1.473 1.474-2.631 3.324-3.474 5.548-.842 2.225-1.263 4.781-1.263 7.668V153h-22.37V94.092c0-6.194-1.249-10.704-3.744-13.532-2.497-2.825-6.18-4.24-11.051-4.24-3.19 0-6.18.798-8.976 2.391-2.799 1.593-5.399 3.775-7.804 6.54V153H610zM572 30h23v19h-23z" + fill="currentColor" + fill-opacity=".8" + /> + </g> + </g> + </svg> + ]] + + local main = '' + for _, tree in ipairs(lang_tree:trees()) do + main = main + .. ( + visit_node( + tree:root(), + 0, + tree, + headings, + { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1 }, + stats + ) + ) + end + + main = ([[ + <header class="container"> + <nav class="navbar navbar-expand-lg"> + <div class="container-fluid"> + <a href="/" class="navbar-brand" aria-label="logo"> + <!--TODO: use <img src="….svg"> here instead. Need one that has green lettering instead of gray. --> + %s + <!--<img src="https://neovim.io/logos/neovim-logo.svg" width="173" height="50" alt="Neovim" />--> + </a> + <div id="docsearch"></div> <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ --> + </div> + </nav> + </header> + + <div class="container golden-grid help-body"> + <div class="col-wide"> + <a name="%s" href="#%s"><h1 id="%s">%s</h1></a> + <p> + <i> + Nvim <code>:help</code> pages, <a href="https://github.com/neovim/neovim/blob/master/scripts/gen_help_html.lua">generated</a> + from <a href="https://github.com/neovim/neovim/blob/master/runtime/doc/%s">source</a> + using the <a href="https://github.com/neovim/tree-sitter-vimdoc">tree-sitter-vimdoc</a> parser. + </i> + </p> + <hr/> + %s + </div> + ]]):format( + logo_svg, + stats.first_tags[1] or '', + stats.first_tags[2] or '', + stats.first_tags[2] or '', + title, + vim.fs.basename(fname), + main + ) + + ---@type string + local toc = [[ + <div class="col-narrow toc"> + <div><a href="index.html">Main</a></div> + <div><a href="vimindex.html">Commands index</a></div> + <div><a href="quickref.html">Quick reference</a></div> + <hr/> + ]] + + local n = 0 -- Count of all headings + subheadings. + for _, h1 in ipairs(headings) do + n = n + 1 + #h1.subheadings + end + for _, h1 in ipairs(headings) do + ---@type string + toc = toc .. ('<div class="help-toc-h1"><a href="#%s">%s</a>\n'):format(h1.tag, h1.name) + if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. + for _, h2 in ipairs(h1.subheadings) do + toc = toc + .. ('<div class="help-toc-h2"><a href="#%s">%s</a></div>\n'):format(h2.tag, h2.name) + end + end + toc = toc .. '</div>' + end + toc = toc .. '</div>\n' + + local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil) + local bug_link = string.format('(<a href="%s" target="_blank">report docs bug...</a>)', bug_url) + + local footer = ([[ + <footer> + <div class="container flex"> + <div class="generator-stats"> + Generated at %s from <code><a href="https://github.com/neovim/neovim/commit/%s">%s</a></code> + </div> + <div class="generator-stats"> + parse_errors: %d %s | <span title="%s">noise_lines: %d</span> + </div> + <div> + + <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ --> + <script src="https://cdn.jsdelivr.net/npm/@docsearch/js@3"></script> + <script type="module"> + docsearch({ + container: '#docsearch', + appId: 'X185E15FPG', + apiKey: 'b5e6b2f9c636b2b471303205e59832ed', + indexName: 'nvim', + }); + </script> + + </footer> + ]]):format( + os.date('%Y-%m-%d %H:%M'), + commit, + commit:sub(1, 7), + #stats.parse_errors, + bug_link, + html_esc(table.concat(stats.noise_lines, '\n')), + #stats.noise_lines + ) + + html = ('%s%s%s</div>\n%s</body>\n</html>\n'):format(html, main, toc, footer) + vim.cmd('q!') + lang_tree:destroy() + return html, stats +end + +local function gen_css(fname) + local css = [[ + :root { + --code-color: #004b4b; + --tag-color: #095943; + } + @media (prefers-color-scheme: dark) { + :root { + --code-color: #00c243; + --tag-color: #00b7b7; + } + } + @media (min-width: 40em) { + .toc { + position: fixed; + left: 67%; + } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + } + @media (max-width: 40em) { + .golden-grid { + /* Disable grid for narrow viewport (mobile phone). */ + display: block; + } + } + .toc { + /* max-width: 12rem; */ + height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + } + .toc > div { + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + } + html { + scroll-behavior: auto; + } + body { + font-size: 18px; + line-height: 1.5; + } + h1, h2, h3, h4, h5 { + font-family: sans-serif; + border-bottom: 1px solid var(--tag-color); /*rgba(0, 0, 0, .9);*/ + } + h3, h4, h5 { + border-bottom-style: dashed; + } + .help-column_heading { + color: var(--code-color); + } + .help-body { + padding-bottom: 2em; + } + .help-line { + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + } + .help-li { + white-space: normal; + display: list-item; + margin-left: 1.5rem; /* padding-left: 1rem; */ + } + .help-para { + padding-top: 10px; + padding-bottom: 10px; + } + + .old-help-para { + padding-top: 10px; + padding-bottom: 10px; + /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */ + tab-size: 8; + white-space: pre-wrap; + font-size: 16px; + font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; + word-wrap: break-word; + } + .old-help-para pre, .old-help-para pre:hover { + /* Text following <pre> is already visually separated by the linebreak. */ + margin-bottom: 0; + /* Long lines that exceed the textwidth should not be wrapped (no "pre-wrap"). + Since text may overflow horizontally, we make the contents to be scrollable + (only if necessary) to prevent overlapping with the navigation bar at the right. */ + white-space: pre; + overflow-x: auto; + } + + /* TODO: should this rule be deleted? help tags are rendered as <code> or <span>, not <a> */ + a.help-tag, a.help-tag:focus, a.help-tag:hover { + color: inherit; + text-decoration: none; + } + .help-tag { + color: var(--tag-color); + } + /* Tag pseudo-header common in :help docs. */ + .help-tag-right { + color: var(--tag-color); + margin-left: auto; + margin-right: 0; + float: right; + display: block; + } + .help-tag a, + .help-tag-right a { + color: inherit; + } + .help-tag a:not(:hover), + .help-tag-right a:not(:hover) { + text-decoration: none; + } + h1 .help-tag, h2 .help-tag, h3 .help-tag { + font-size: smaller; + } + .help-heading { + white-space: normal; + display: flex; + flex-flow: row wrap; + justify-content: space-between; + gap: 0 15px; + } + /* The (right-aligned) "tags" part of a section heading. */ + .help-heading-tags { + margin-right: 10px; + } + .help-toc-h1 { + } + .help-toc-h2 { + margin-left: 1em; + } + .parse-error { + background-color: red; + } + .unknown-token { + color: black; + background-color: yellow; + } + code { + color: var(--code-color); + font-size: 16px; + } + pre { + /* Tabs are used in codeblocks only for indentation, not alignment, so we can aggressively shrink them. */ + tab-size: 2; + white-space: pre-wrap; + line-height: 1.3; /* Important for ascii art. */ + overflow: visible; + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + font-size: 16px; + margin-top: 10px; + } + pre:last-child { + margin-bottom: 0; + } + pre:hover { + overflow: visible; + } + .generator-stats { + color: gray; + font-size: smaller; + } + ]] + tofile(fname, css) +end + +-- Testing + +local function ok(cond, expected, actual, message) + assert( + (not expected and not actual) or (expected and actual), + 'if "expected" is given, "actual" is also required' + ) + if expected then + assert( + cond, + ('%sexpected %s, got: %s'):format( + message and (message .. '\n') or '', + vim.inspect(expected), + vim.inspect(actual) + ) + ) + return cond + else + return assert(cond) + end +end +local function eq(expected, actual, message) + return ok(vim.deep_equal(expected, actual), expected, actual, message) +end + +function M._test() + tagmap = get_helptags('$VIMRUNTIME/doc') + helpfiles = get_helpfiles(vim.fs.normalize('$VIMRUNTIME/doc')) + + ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) + ok( + vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), + tagmap['vim.diagnostic.set()'], + 'diagnostic.txt' + ) + ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt') + ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]])) + ok(is_noise([[ NVIM REFERENCE MANUAL by Thiago de Arruda ]])) + ok(not is_noise([[vim:tw=78]])) + + eq(0, get_indent('a')) + eq(1, get_indent(' a')) + eq(2, get_indent(' a\n b\n c\n')) + eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) + eq( + 'a\n \n b\n c\n d\n e\n', + trim_indent(' a\n \n b\n c\n d\n e\n') + ) + + local fixed_url, removed_chars = fix_url('https://example.com).') + eq('https://example.com', fixed_url) + eq(').', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.)') + eq('https://example.com.', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.') + eq('https://example.com', fixed_url) + eq('.', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com)') + eq('https://example.com', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com') + eq('https://example.com', fixed_url) + eq('', removed_chars) + + print('all tests passed.\n') +end + +--- @class nvim.gen_help_html.gen_result +--- @field helpfiles string[] list of generated HTML files, from the source docs {include} +--- @field err_count integer number of parse errors in :help docs +--- @field invalid_links table<string, any> + +--- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`. +--- +--- Example: +--- +--- gen('$VIMRUNTIME/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) +--- +--- @param help_dir string Source directory containing the :help files. Must run `make helptags` first. +--- @param to_dir string Target directory where the .html files will be written. +--- @param include string[]|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} +--- +--- @return nvim.gen_help_html.gen_result result +function M.gen(help_dir, to_dir, include, commit, parser_path) + vim.validate('help_dir', help_dir, function(d) + return vim.fn.isdirectory(vim.fs.normalize(d)) == 1 + end, 'valid directory') + vim.validate('to_dir', to_dir, 'string') + vim.validate('include', include, 'table', true) + vim.validate('commit', commit, 'string', true) + vim.validate('parser_path', parser_path, function(f) + return vim.fn.filereadable(vim.fs.normalize(f)) == 1 + end, true, 'valid vimdoc.{so,dll} filepath') + + local err_count = 0 + local redirects_count = 0 + ensure_runtimepath() + tagmap = get_helptags(vim.fs.normalize(help_dir)) + helpfiles = get_helpfiles(help_dir, include) + to_dir = vim.fs.normalize(to_dir) + parser_path = parser_path and vim.fs.normalize(parser_path) or nil + + print(('output dir: %s\n\n'):format(to_dir)) + vim.fn.mkdir(to_dir, 'p') + gen_css(('%s/help.css'):format(to_dir)) + + for _, f in ipairs(helpfiles) do + -- "foo.txt" + local helpfile = vim.fs.basename(f) + -- "to/dir/foo.html" + local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) + local html, stats = + gen_one(f, nil, to_fname, not new_layout[helpfile], commit or '?', parser_path) + tofile(to_fname, html) + print( + ('generated (%-2s errors): %-15s => %s'):format( + #stats.parse_errors, + helpfile, + vim.fs.basename(to_fname) + ) + ) + + -- Generate redirect pages for renamed help files. + local helpfile_tag = (helpfile:gsub('%.txt$', '')) + local redirect_from = redirects[helpfile_tag] + if redirect_from then + local redirect_text = ([[ +*%s* Nvim + +This document moved to: |%s| + +============================================================================== +This document moved to: |%s| + +This document moved to: |%s| + +============================================================================== + vim:tw=78:ts=8:ft=help:norl: + ]]):format( + redirect_from, + helpfile_tag, + helpfile_tag, + helpfile_tag, + helpfile_tag, + helpfile_tag + ) + local redirect_to = ('%s/%s'):format(to_dir, get_helppage(redirect_from)) + local redirect_html, _ = + gen_one(redirect_from, redirect_text, redirect_to, false, commit or '?', parser_path) + assert(redirect_html:find(helpfile_tag)) + tofile(redirect_to, redirect_html) + + print( + ('generated (redirect) : %-15s => %s'):format( + redirect_from .. '.txt', + vim.fs.basename(to_fname) + ) + ) + redirects_count = redirects_count + 1 + end + + err_count = err_count + #stats.parse_errors + end + + print(('\ngenerated %d html pages'):format(#helpfiles + redirects_count)) + print(('total errors: %d'):format(err_count)) + print(('invalid tags: %s'):format(vim.inspect(invalid_links))) + assert(#(include or {}) > 0 or redirects_count == vim.tbl_count(redirects)) -- sanity check + print(('redirects: %d'):format(redirects_count)) + print('\n') + + --- @type nvim.gen_help_html.gen_result + return { + helpfiles = helpfiles, + err_count = err_count, + invalid_links = invalid_links, + } +end + +--- @class nvim.gen_help_html.validate_result +--- @field helpfiles integer number of generated helpfiles +--- @field err_count integer number of parse errors +--- @field parse_errors table<string, string[]> +--- @field invalid_links table<string, any> invalid tags in :help docs +--- @field invalid_urls table<string, any> invalid URLs in :help docs +--- @field invalid_spelling table<string, table<string, string>> invalid spelling in :help docs + +--- Validates all :help files found in `help_dir`: +--- - checks that |tag| links point to valid helptags. +--- - recursively counts parse errors ("ERROR" nodes) +--- +--- This is 10x faster than gen(), for use in CI. +--- +--- @return nvim.gen_help_html.validate_result result +function M.validate(help_dir, include, parser_path) + vim.validate('help_dir', help_dir, function(d) + return vim.fn.isdirectory(vim.fs.normalize(d)) == 1 + end, 'valid directory') + vim.validate('include', include, 'table', true) + vim.validate('parser_path', parser_path, function(f) + return vim.fn.filereadable(vim.fs.normalize(f)) == 1 + end, true, 'valid vimdoc.{so,dll} filepath') + local err_count = 0 ---@type integer + local files_to_errors = {} ---@type table<string, string[]> + ensure_runtimepath() + tagmap = get_helptags(vim.fs.normalize(help_dir)) + helpfiles = get_helpfiles(help_dir, include) + parser_path = parser_path and vim.fs.normalize(parser_path) or nil + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local rv = validate_one(f, parser_path) + print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) + if #rv.parse_errors > 0 then + files_to_errors[helpfile] = rv.parse_errors + vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v) + return s .. '\n ' .. v + end))) + end + err_count = err_count + #rv.parse_errors + end + + ---@type nvim.gen_help_html.validate_result + return { + helpfiles = #helpfiles, + err_count = err_count, + parse_errors = files_to_errors, + invalid_links = invalid_links, + invalid_urls = invalid_urls, + invalid_spelling = invalid_spelling, + } +end + +--- Validates vimdoc files on $VIMRUNTIME. and print human-readable error messages if fails. +--- +--- If this fails, try these steps (in order): +--- 1. Fix/cleanup the :help docs. +--- 2. Fix the parser: https://github.com/neovim/tree-sitter-vimdoc +--- 3. File a parser bug, and adjust the tolerance of this test in the meantime. +--- +--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc' +function M.run_validate(help_dir) + help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc') + print('doc path = ' .. vim.uv.fs_realpath(help_dir)) + + local rv = M.validate(help_dir) + + -- Check that we actually found helpfiles. + ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles) + + eq({}, rv.parse_errors, 'no parse errors') + eq(0, rv.err_count, 'no parse errors') + eq({}, rv.invalid_links, 'invalid tags in :help docs') + eq({}, rv.invalid_urls, 'invalid URLs in :help docs') + eq( + {}, + rv.invalid_spelling, + 'invalid spelling in :help docs (see spell_dict in scripts/gen_help_html.lua)' + ) +end + +--- Test-generates HTML from docs. +--- +--- 1. Test that gen_help_html.lua actually works. +--- 2. Test that parse errors did not increase wildly. Because we explicitly test only a few +--- :help files, we can be precise about the tolerances here. +--- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc' +function M.test_gen(help_dir) + local tmpdir = vim.fs.dirname(vim.fn.tempname()) + help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc') + print('doc path = ' .. vim.uv.fs_realpath(help_dir)) + + -- Because gen() is slow (~30s), this test is limited to a few files. + local input = { 'help.txt', 'index.txt', 'nvim.txt' } + local rv = M.gen(help_dir, tmpdir, input) + eq(#input, #rv.helpfiles) + eq(0, rv.err_count, 'parse errors in :help docs') + eq({}, rv.invalid_links, 'invalid tags in :help docs') +end + +return M diff --git a/src/gen/gen_lsp.lua b/src/gen/gen_lsp.lua new file mode 100644 index 0000000000..38792307e4 --- /dev/null +++ b/src/gen/gen_lsp.lua @@ -0,0 +1,514 @@ +-- Generates lua-ls annotations for lsp. + +local USAGE = [[ +Generates lua-ls annotations for lsp. + +USAGE: +nvim -l src/gen/gen_lsp.lua gen # by default, this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua +nvim -l src/gen/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua +nvim -l src/gen/gen_lsp.lua gen --version 3.18 --methods --capabilities +]] + +local DEFAULT_LSP_VERSION = '3.18' + +local M = {} + +local function tofile(fname, text) + local f = io.open(fname, 'w') + if not f then + error(('failed to write: %s'):format(f)) + else + print(('Written to: %s'):format(fname)) + f:write(text) + f:close() + end +end + +--- The LSP protocol JSON data (it's partial, non-exhaustive). +--- https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/3.18/metaModel/metaModel.schema.json +--- @class vim._gen_lsp.Protocol +--- @field requests vim._gen_lsp.Request[] +--- @field notifications vim._gen_lsp.Notification[] +--- @field structures vim._gen_lsp.Structure[] +--- @field enumerations vim._gen_lsp.Enumeration[] +--- @field typeAliases vim._gen_lsp.TypeAlias[] + +---@param opt vim._gen_lsp.opt +---@return vim._gen_lsp.Protocol +local function read_json(opt) + local uri = 'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/' + .. opt.version + .. '/metaModel/metaModel.json' + print('Reading ' .. uri) + + local res = vim.system({ 'curl', '--no-progress-meter', uri, '-o', '-' }):wait() + if res.code ~= 0 or (res.stdout or ''):len() < 999 then + print(('URL failed: %s'):format(uri)) + vim.print(res) + error(res.stdout) + end + return vim.json.decode(res.stdout) +end + +-- Gets the Lua symbol for a given fully-qualified LSP method name. +local function to_luaname(s) + -- "$/" prefix is special: https://microsoft.github.io/language-server-protocol/specification/#dollarRequests + return s:gsub('^%$', 'dollar'):gsub('/', '_') +end + +---@param protocol vim._gen_lsp.Protocol +---@param gen_methods boolean +---@param gen_capabilities boolean +local function write_to_protocol(protocol, gen_methods, gen_capabilities) + if not gen_methods and not gen_capabilities then + return + end + + local indent = (' '):rep(2) + + --- @class vim._gen_lsp.Request + --- @field deprecated? string + --- @field documentation? string + --- @field messageDirection string + --- @field clientCapability? string + --- @field serverCapability? string + --- @field method string + --- @field params? any + --- @field proposed? boolean + --- @field registrationMethod? string + --- @field registrationOptions? any + --- @field since? string + + --- @class vim._gen_lsp.Notification + --- @field deprecated? string + --- @field documentation? string + --- @field errorData? any + --- @field messageDirection string + --- @field clientCapability? string + --- @field serverCapability? string + --- @field method string + --- @field params? any[] + --- @field partialResult? any + --- @field proposed? boolean + --- @field registrationMethod? string + --- @field registrationOptions? any + --- @field result any + --- @field since? string + + ---@type (vim._gen_lsp.Request|vim._gen_lsp.Notification)[] + local all = vim.list_extend(protocol.requests, protocol.notifications) + table.sort(all, function(a, b) + return to_luaname(a.method) < to_luaname(b.method) + end) + + local output = { '-- Generated by gen_lsp.lua, keep at end of file.' } + + if gen_methods then + output[#output + 1] = '--- @alias vim.lsp.protocol.Method.ClientToServer' + + for _, item in ipairs(all) do + if item.method and item.messageDirection == 'clientToServer' then + output[#output + 1] = ("--- | '%s',"):format(item.method) + end + end + + vim.list_extend(output, { + '', + '--- @alias vim.lsp.protocol.Method.ServerToClient', + }) + for _, item in ipairs(all) do + if item.method and item.messageDirection == 'serverToClient' then + output[#output + 1] = ("--- | '%s',"):format(item.method) + end + end + + vim.list_extend(output, { + '', + '--- @alias vim.lsp.protocol.Method', + '--- | vim.lsp.protocol.Method.ClientToServer', + '--- | vim.lsp.protocol.Method.ServerToClient', + '', + '-- Generated by gen_lsp.lua, keep at end of file.', + '--- @enum vim.lsp.protocol.Methods', + '--- @see https://microsoft.github.io/language-server-protocol/specification/#metaModel', + '--- LSP method names.', + 'protocol.Methods = {', + }) + + for _, item in ipairs(all) do + if item.method then + if item.documentation then + local document = vim.split(item.documentation, '\n?\n', { trimempty = true }) + for _, docstring in ipairs(document) do + output[#output + 1] = indent .. '--- ' .. docstring + end + end + output[#output + 1] = ("%s%s = '%s',"):format(indent, to_luaname(item.method), item.method) + end + end + output[#output + 1] = '}' + end + + if gen_capabilities then + vim.list_extend(output, { + '', + '-- stylua: ignore start', + '-- Generated by gen_lsp.lua, keep at end of file.', + '--- Maps method names to the required server capability', + 'protocol._request_name_to_capability = {', + }) + + for _, item in ipairs(all) do + if item.serverCapability then + output[#output + 1] = ("%s['%s'] = { %s },"):format( + indent, + item.method, + table.concat( + vim + .iter(vim.split(item.serverCapability, '.', { plain = true })) + :map(function(segment) + return "'" .. segment .. "'" + end) + :totable(), + ', ' + ) + ) + end + end + + output[#output + 1] = '}' + output[#output + 1] = '-- stylua: ignore end' + end + + output[#output + 1] = '' + output[#output + 1] = 'return protocol' + + local fname = './runtime/lua/vim/lsp/protocol.lua' + local bufnr = vim.fn.bufadd(fname) + vim.fn.bufload(bufnr) + vim.api.nvim_set_current_buf(bufnr) + local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) + local index = vim.iter(ipairs(lines)):find(function(key, item) + return vim.startswith(item, '-- Generated by') and key or nil + end) + index = index and index - 1 or vim.api.nvim_buf_line_count(bufnr) - 1 + vim.api.nvim_buf_set_lines(bufnr, index, -1, true, output) + vim.cmd.write() +end + +---@class vim._gen_lsp.opt +---@field output_file string +---@field version string +---@field methods boolean +---@field capabilities boolean + +---@param opt vim._gen_lsp.opt +function M.gen(opt) + --- @type vim._gen_lsp.Protocol + local protocol = read_json(opt) + + write_to_protocol(protocol, opt.methods, opt.capabilities) + + local output = { + '--' .. '[[', + 'THIS FILE IS GENERATED by scr/gen/gen_lsp.lua', + 'DO NOT EDIT MANUALLY', + '', + 'Based on LSP protocol ' .. opt.version, + '', + 'Regenerate:', + ([=[nvim -l scr/gen/gen_lsp.lua gen --version %s]=]):format(DEFAULT_LSP_VERSION), + '--' .. ']]', + '', + '---@meta', + "error('Cannot require a meta file')", + '', + '---@alias lsp.null nil', + '---@alias uinteger integer', + '---@alias decimal number', + '---@alias lsp.DocumentUri string', + '---@alias lsp.URI string', + '', + } + + local anonymous_num = 0 + + ---@type string[] + local anonym_classes = {} + + local simple_types = { + 'string', + 'boolean', + 'integer', + 'uinteger', + 'decimal', + } + + ---@param documentation string + local _process_documentation = function(documentation) + documentation = documentation:gsub('\n', '\n---') + -- Remove <200b> (zero-width space) unicode characters: e.g., `**/<200b>*` + documentation = documentation:gsub('\226\128\139', '') + -- Escape annotations that are not recognized by lua-ls + documentation = documentation:gsub('%^---@sample', '---\\@sample') + return '---' .. documentation + end + + --- @class vim._gen_lsp.Type + --- @field kind string a common field for all Types. + --- @field name? string for ReferenceType, BaseType + --- @field element? any for ArrayType + --- @field items? vim._gen_lsp.Type[] for OrType, AndType + --- @field key? vim._gen_lsp.Type for MapType + --- @field value? string|vim._gen_lsp.Type for StringLiteralType, MapType, StructureLiteralType + + ---@param type vim._gen_lsp.Type + ---@param prefix? string Optional prefix associated with the this type, made of (nested) field name. + --- Used to generate class name for structure literal types. + ---@return string + local function parse_type(type, prefix) + -- ReferenceType | BaseType + if type.kind == 'reference' or type.kind == 'base' then + if vim.tbl_contains(simple_types, type.name) then + return type.name + end + return 'lsp.' .. type.name + + -- ArrayType + elseif type.kind == 'array' then + local parsed_items = parse_type(type.element, prefix) + if type.element.items and #type.element.items > 1 then + parsed_items = '(' .. parsed_items .. ')' + end + return parsed_items .. '[]' + + -- OrType + elseif type.kind == 'or' then + local val = '' + for _, item in ipairs(type.items) do + val = val .. parse_type(item, prefix) .. '|' --[[ @as string ]] + end + val = val:sub(0, -2) + return val + + -- StringLiteralType + elseif type.kind == 'stringLiteral' then + return '"' .. type.value .. '"' + + -- MapType + elseif type.kind == 'map' then + local key = assert(type.key) + local value = type.value --[[ @as vim._gen_lsp.Type ]] + return 'table<' .. parse_type(key, prefix) .. ', ' .. parse_type(value, prefix) .. '>' + + -- StructureLiteralType + elseif type.kind == 'literal' then + -- can I use ---@param disabled? {reason: string} + -- use | to continue the inline class to be able to add docs + -- https://github.com/LuaLS/lua-language-server/issues/2128 + anonymous_num = anonymous_num + 1 + local anonymous_classname = 'lsp._anonym' .. anonymous_num + if prefix then + anonymous_classname = anonymous_classname .. '.' .. prefix + end + local anonym = vim + .iter({ + (anonymous_num > 1 and { '' } or {}), + { '---@class ' .. anonymous_classname }, + }) + :flatten() + :totable() + + --- @class vim._gen_lsp.StructureLiteral translated to anonymous @class. + --- @field deprecated? string + --- @field description? string + --- @field properties vim._gen_lsp.Property[] + --- @field proposed? boolean + --- @field since? string + + ---@type vim._gen_lsp.StructureLiteral + local structural_literal = assert(type.value) --[[ @as vim._gen_lsp.StructureLiteral ]] + for _, field in ipairs(structural_literal.properties) do + anonym[#anonym + 1] = '---' + if field.documentation then + anonym[#anonym + 1] = _process_documentation(field.documentation) + end + anonym[#anonym + 1] = '---@field ' + .. field.name + .. (field.optional and '?' or '') + .. ' ' + .. parse_type(field.type, prefix .. '.' .. field.name) + end + -- anonym[#anonym + 1] = '' + for _, line in ipairs(anonym) do + if line then + anonym_classes[#anonym_classes + 1] = line + end + end + return anonymous_classname + + -- TupleType + elseif type.kind == 'tuple' then + local tuple = '[' + for _, value in ipairs(type.items) do + tuple = tuple .. parse_type(value, prefix) .. ', ' + end + -- remove , at the end + tuple = tuple:sub(0, -3) + return tuple .. ']' + end + + vim.print('WARNING: Unknown type ', type) + return '' + end + + --- @class vim._gen_lsp.Structure translated to @class + --- @field deprecated? string + --- @field documentation? string + --- @field extends? { kind: string, name: string }[] + --- @field mixins? { kind: string, name: string }[] + --- @field name string + --- @field properties? vim._gen_lsp.Property[] members, translated to @field + --- @field proposed? boolean + --- @field since? string + for _, structure in ipairs(protocol.structures) do + -- output[#output + 1] = '' + if structure.documentation then + output[#output + 1] = _process_documentation(structure.documentation) + end + local class_string = ('---@class lsp.%s'):format(structure.name) + if structure.extends or structure.mixins then + local inherits_from = table.concat( + vim.list_extend( + vim.tbl_map(parse_type, structure.extends or {}), + vim.tbl_map(parse_type, structure.mixins or {}) + ), + ', ' + ) + class_string = class_string .. ': ' .. inherits_from + end + output[#output + 1] = class_string + + --- @class vim._gen_lsp.Property translated to @field + --- @field deprecated? string + --- @field documentation? string + --- @field name string + --- @field optional? boolean + --- @field proposed? boolean + --- @field since? string + --- @field type { kind: string, name: string } + for _, field in ipairs(structure.properties or {}) do + output[#output + 1] = '---' -- Insert a single newline between @fields (and after @class) + if field.documentation then + output[#output + 1] = _process_documentation(field.documentation) + end + output[#output + 1] = '---@field ' + .. field.name + .. (field.optional and '?' or '') + .. ' ' + .. parse_type(field.type, field.name) + end + output[#output + 1] = '' + end + + --- @class vim._gen_lsp.Enumeration translated to @enum + --- @field deprecated string? + --- @field documentation string? + --- @field name string? + --- @field proposed boolean? + --- @field since string? + --- @field suportsCustomValues boolean? + --- @field values { name: string, value: string, documentation?: string, since?: string }[] + for _, enum in ipairs(protocol.enumerations) do + if enum.documentation then + output[#output + 1] = _process_documentation(enum.documentation) + end + local enum_type = '---@alias lsp.' .. enum.name + for _, value in ipairs(enum.values) do + enum_type = enum_type + .. '\n---| ' + .. (type(value.value) == 'string' and '"' .. value.value .. '"' or value.value) + .. ' # ' + .. value.name + end + output[#output + 1] = enum_type + output[#output + 1] = '' + end + + --- @class vim._gen_lsp.TypeAlias translated to @alias + --- @field deprecated? string? + --- @field documentation? string + --- @field name string + --- @field proposed? boolean + --- @field since? string + --- @field type vim._gen_lsp.Type + for _, alias in ipairs(protocol.typeAliases) do + if alias.documentation then + output[#output + 1] = _process_documentation(alias.documentation) + end + if alias.type.kind == 'or' then + local alias_type = '---@alias lsp.' .. alias.name .. ' ' + for _, item in ipairs(alias.type.items) do + alias_type = alias_type .. parse_type(item, alias.name) .. '|' + end + alias_type = alias_type:sub(0, -2) + output[#output + 1] = alias_type + else + output[#output + 1] = '---@alias lsp.' + .. alias.name + .. ' ' + .. parse_type(alias.type, alias.name) + end + output[#output + 1] = '' + end + + -- anonymous classes + for _, line in ipairs(anonym_classes) do + output[#output + 1] = line + end + + tofile(opt.output_file, table.concat(output, '\n') .. '\n') +end + +---@type vim._gen_lsp.opt +local opt = { + output_file = 'runtime/lua/vim/lsp/_meta/protocol.lua', + version = DEFAULT_LSP_VERSION, + methods = false, + capabilities = false, +} + +local command = nil +local i = 1 +while i <= #_G.arg do + if _G.arg[i] == '--out' then + opt.output_file = assert(_G.arg[i + 1], '--out <outfile> needed') + i = i + 1 + elseif _G.arg[i] == '--version' then + opt.version = assert(_G.arg[i + 1], '--version <version> needed') + i = i + 1 + elseif _G.arg[i] == '--methods' then + opt.methods = true + elseif _G.arg[i] == '--capabilities' then + opt.capabilities = true + elseif vim.startswith(_G.arg[i], '-') then + error('Unrecognized args: ' .. _G.arg[i]) + else + if command then + error('More than one command was given: ' .. _G.arg[i]) + else + command = _G.arg[i] + end + end + i = i + 1 +end + +if not command then + print(USAGE) +elseif M[command] then + M[command](opt) -- see M.gen() +else + error('Unknown command: ' .. command) +end + +return M diff --git a/src/nvim/generators/gen_options.lua b/src/gen/gen_options.lua index e5dba90925..1947297a0e 100644 --- a/src/nvim/generators/gen_options.lua +++ b/src/gen/gen_options.lua @@ -1,5 +1,5 @@ --- @module 'nvim.options' -local options = require('options') +local options = require('nvim.options') local options_meta = options.options local cstr = options.cstr local valid_scopes = options.valid_scopes @@ -418,7 +418,7 @@ end --- @param option_index table<string,string> local function gen_map(output_file, option_index) -- Generate option index map. - local hashy = require('generators.hashy') + local hashy = require('gen.hashy') local neworder, hashfun = hashy.hashy_hash( 'find_option', diff --git a/src/gen/gen_vimdoc.lua b/src/gen/gen_vimdoc.lua new file mode 100755 index 0000000000..2fe7224ea5 --- /dev/null +++ b/src/gen/gen_vimdoc.lua @@ -0,0 +1,1041 @@ +#!/usr/bin/env -S nvim -l +--- Generates Nvim :help docs from Lua/C docstrings +--- +--- The generated :help text for each function is formatted as follows: +--- - Max width of 78 columns (`TEXT_WIDTH`). +--- - Indent with spaces (not tabs). +--- - Indent of 4 columns for body text (`INDENTATION`). +--- - Function signature and helptag (right-aligned) on the same line. +--- - Signature and helptag must have a minimum of 8 spaces between them. +--- - If the signature is too long, it is placed on the line after the helptag. +--- Signature wraps with subsequent lines indented to the open parenthesis. +--- - Subsection bodies are indented an additional 4 spaces. +--- - Body consists of function description, parameters, return description, and +--- C declaration (`INCLUDE_C_DECL`). +--- - Parameters are omitted for the `void` and `Error *` types, or if the +--- parameter is marked as [out]. +--- - Each function documentation is separated by a single line. + +local luacats_parser = require('gen.luacats_parser') +local cdoc_parser = require('gen.cdoc_parser') +local util = require('gen.util') + +local fmt = string.format + +local wrap = util.wrap +local md_to_vimdoc = util.md_to_vimdoc + +local TEXT_WIDTH = 78 +local INDENTATION = 4 + +--- @class (exact) nvim.gen_vimdoc.Config +--- +--- Generated documentation target, e.g. api.txt +--- @field filename string +--- +--- @field section_order string[] +--- +--- List of files/directories for doxygen to read, relative to `base_dir`. +--- @field files string[] +--- +--- @field exclude_types? true +--- +--- Section name overrides. Key: filename (e.g., vim.c) +--- @field section_name? table<string,string> +--- +--- @field fn_name_pat? string +--- +--- @field fn_xform? fun(fun: nvim.luacats.parser.fun) +--- +--- For generated section names. +--- @field section_fmt fun(name: string): string +--- +--- @field helptag_fmt fun(name: string): string|string[] +--- +--- Per-function helptag. +--- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string +--- +--- @field append_only? string[] + +local function contains(t, xs) + return vim.tbl_contains(xs, t) +end + +--- @type {level:integer, prerelease:boolean}? +local nvim_api_info_ + +--- @return {level: integer, prerelease:boolean} +local function nvim_api_info() + if not nvim_api_info_ then + --- @type integer?, boolean? + local level, prerelease + for l in io.lines('CMakeLists.txt') do + --- @cast l string + if level and prerelease then + break + end + local m1 = l:match('^set%(NVIM_API_LEVEL%s+(%d+)%)') + if m1 then + level = tonumber(m1) --[[@as integer]] + end + local m2 = l:match('^set%(NVIM_API_PRERELEASE%s+(%w+)%)') + if m2 then + prerelease = m2 == 'true' + end + end + nvim_api_info_ = { level = level, prerelease = prerelease } + end + + return nvim_api_info_ +end + +--- @param fun nvim.luacats.parser.fun +--- @return string +local function fn_helptag_fmt_common(fun) + local fn_sfx = fun.table and '' or '()' + if fun.classvar then + return fmt('%s:%s%s', fun.classvar, fun.name, fn_sfx) + end + if fun.module then + return fmt('%s.%s%s', fun.module, fun.name, fn_sfx) + end + return fun.name .. fn_sfx +end + +--- @type table<string,nvim.gen_vimdoc.Config> +local config = { + api = { + filename = 'api.txt', + section_order = { + 'vim.c', + 'vimscript.c', + 'command.c', + 'options.c', + 'buffer.c', + 'extmark.c', + 'window.c', + 'win_config.c', + 'tabpage.c', + 'autocmd.c', + 'ui.c', + }, + exclude_types = true, + fn_name_pat = 'nvim_.*', + files = { 'src/nvim/api' }, + section_name = { + ['vim.c'] = 'Global', + }, + section_fmt = function(name) + return name .. ' Functions' + end, + helptag_fmt = function(name) + return fmt('api-%s', name:lower()) + end, + }, + lua = { + filename = 'lua.txt', + section_order = { + 'hl.lua', + 'diff.lua', + 'mpack.lua', + 'json.lua', + 'base64.lua', + 'spell.lua', + 'builtin.lua', + '_options.lua', + '_editor.lua', + '_inspector.lua', + 'shared.lua', + 'loader.lua', + 'uri.lua', + 'ui.lua', + 'filetype.lua', + 'keymap.lua', + 'fs.lua', + 'glob.lua', + 'lpeg.lua', + 're.lua', + 'regex.lua', + 'secure.lua', + 'version.lua', + 'iter.lua', + 'snippet.lua', + 'text.lua', + 'tohtml.lua', + }, + files = { + 'runtime/lua/vim/iter.lua', + 'runtime/lua/vim/_editor.lua', + 'runtime/lua/vim/_options.lua', + 'runtime/lua/vim/shared.lua', + 'runtime/lua/vim/loader.lua', + 'runtime/lua/vim/uri.lua', + 'runtime/lua/vim/ui.lua', + 'runtime/lua/vim/filetype.lua', + 'runtime/lua/vim/keymap.lua', + 'runtime/lua/vim/fs.lua', + 'runtime/lua/vim/hl.lua', + 'runtime/lua/vim/secure.lua', + 'runtime/lua/vim/version.lua', + 'runtime/lua/vim/_inspector.lua', + 'runtime/lua/vim/snippet.lua', + 'runtime/lua/vim/text.lua', + 'runtime/lua/vim/glob.lua', + 'runtime/lua/vim/_meta/builtin.lua', + 'runtime/lua/vim/_meta/diff.lua', + 'runtime/lua/vim/_meta/mpack.lua', + 'runtime/lua/vim/_meta/json.lua', + 'runtime/lua/vim/_meta/base64.lua', + 'runtime/lua/vim/_meta/regex.lua', + 'runtime/lua/vim/_meta/lpeg.lua', + 'runtime/lua/vim/_meta/re.lua', + 'runtime/lua/vim/_meta/spell.lua', + 'runtime/lua/tohtml.lua', + }, + fn_xform = function(fun) + if contains(fun.module, { 'vim.uri', 'vim.shared', 'vim._editor' }) then + fun.module = 'vim' + end + + if fun.module == 'vim' and contains(fun.name, { 'cmd', 'inspect' }) then + fun.table = nil + end + + if fun.classvar or vim.startswith(fun.name, 'vim.') or fun.module == 'vim.iter' then + return + end + + fun.name = fmt('%s.%s', fun.module, fun.name) + end, + section_name = { + ['_inspector.lua'] = 'inspector', + }, + section_fmt = function(name) + name = name:lower() + if name == '_editor' then + return 'Lua module: vim' + elseif name == '_options' then + return 'LUA-VIMSCRIPT BRIDGE' + elseif name == 'builtin' then + return 'VIM' + end + if + contains(name, { + 'hl', + 'mpack', + 'json', + 'base64', + 'diff', + 'spell', + 'regex', + 'lpeg', + 're', + }) + then + return 'VIM.' .. name:upper() + end + if name == 'tohtml' then + return 'Lua module: tohtml' + end + return 'Lua module: vim.' .. name + end, + helptag_fmt = function(name) + if name == '_editor' then + return 'lua-vim' + elseif name == '_options' then + return 'lua-vimscript' + elseif name == 'tohtml' then + return 'tohtml' + end + return 'vim.' .. name:lower() + end, + fn_helptag_fmt = function(fun) + local name = fun.name + + if vim.startswith(name, 'vim.') then + local fn_sfx = fun.table and '' or '()' + return name .. fn_sfx + elseif fun.classvar == 'Option' then + return fmt('vim.opt:%s()', name) + end + + return fn_helptag_fmt_common(fun) + end, + append_only = { + 'shared.lua', + }, + }, + lsp = { + filename = 'lsp.txt', + section_order = { + 'lsp.lua', + 'client.lua', + 'buf.lua', + 'diagnostic.lua', + 'codelens.lua', + 'completion.lua', + 'folding_range.lua', + 'inlay_hint.lua', + 'tagfunc.lua', + 'semantic_tokens.lua', + 'handlers.lua', + 'util.lua', + 'log.lua', + 'rpc.lua', + 'protocol.lua', + }, + files = { + 'runtime/lua/vim/lsp', + 'runtime/lua/vim/lsp.lua', + }, + fn_xform = function(fun) + fun.name = fun.name:gsub('result%.', '') + if fun.module == 'vim.lsp.protocol' then + fun.classvar = nil + end + end, + section_fmt = function(name) + if name:lower() == 'lsp' then + return 'Lua module: vim.lsp' + end + return 'Lua module: vim.lsp.' .. name:lower() + end, + helptag_fmt = function(name) + if name:lower() == 'lsp' then + return 'lsp-core' + end + return fmt('lsp-%s', name:lower()) + end, + }, + diagnostic = { + filename = 'diagnostic.txt', + section_order = { + 'diagnostic.lua', + }, + files = { 'runtime/lua/vim/diagnostic.lua' }, + section_fmt = function() + return 'Lua module: vim.diagnostic' + end, + helptag_fmt = function() + return 'diagnostic-api' + end, + }, + treesitter = { + filename = 'treesitter.txt', + section_order = { + 'tstree.lua', + 'tsnode.lua', + 'treesitter.lua', + 'language.lua', + 'query.lua', + 'highlighter.lua', + 'languagetree.lua', + 'dev.lua', + }, + files = { + 'runtime/lua/vim/treesitter/_meta/', + 'runtime/lua/vim/treesitter.lua', + 'runtime/lua/vim/treesitter/', + }, + section_fmt = function(name) + if name:lower() == 'treesitter' then + return 'Lua module: vim.treesitter' + elseif name:lower() == 'tstree' then + return 'TREESITTER TREES' + elseif name:lower() == 'tsnode' then + return 'TREESITTER NODES' + end + return 'Lua module: vim.treesitter.' .. name:lower() + end, + helptag_fmt = function(name) + if name:lower() == 'treesitter' then + return 'lua-treesitter-core' + elseif name:lower() == 'query' then + return 'lua-treesitter-query' + elseif name:lower() == 'tstree' then + return { 'treesitter-tree', 'TSTree' } + elseif name:lower() == 'tsnode' then + return { 'treesitter-node', 'TSNode' } + end + return 'treesitter-' .. name:lower() + end, + }, + editorconfig = { + filename = 'editorconfig.txt', + files = { + 'runtime/lua/editorconfig.lua', + }, + section_order = { + 'editorconfig.lua', + }, + section_fmt = function(_name) + return 'EditorConfig integration' + end, + helptag_fmt = function(name) + return name:lower() + end, + fn_xform = function(fun) + fun.table = true + fun.name = vim.split(fun.name, '.', { plain = true })[2] + end, + }, + health = { + filename = 'health.txt', + files = { + 'runtime/lua/vim/health.lua', + }, + section_order = { + 'health.lua', + }, + section_fmt = function(_name) + return 'Checkhealth' + end, + helptag_fmt = function() + return { 'vim.health', 'health' } + end, + }, +} + +--- @param ty string +--- @param generics table<string,string> +--- @return string +local function replace_generics(ty, generics) + if ty:sub(-2) == '[]' then + local ty0 = ty:sub(1, -3) + if generics[ty0] then + return generics[ty0] .. '[]' + end + elseif ty:sub(-1) == '?' then + local ty0 = ty:sub(1, -2) + if generics[ty0] then + return generics[ty0] .. '?' + end + end + + return generics[ty] or ty +end + +--- @param name string +local function fmt_field_name(name) + local name0, opt = name:match('^([^?]*)(%??)$') + return fmt('{%s}%s', name0, opt) +end + +--- @param ty string +--- @param generics? table<string,string> +--- @param default? string +local function render_type(ty, generics, default) + if generics then + ty = replace_generics(ty, generics) + end + ty = ty:gsub('%s*|%s*nil', '?') + ty = ty:gsub('nil%s*|%s*(.*)', '%1?') + ty = ty:gsub('%s*|%s*', '|') + if default then + return fmt('(`%s`, default: %s)', ty, default) + end + return fmt('(`%s`)', ty) +end + +--- @param p nvim.luacats.parser.param|nvim.luacats.parser.field +local function should_render_field_or_param(p) + return not p.nodoc + and not p.access + and not contains(p.name, { '_', 'self' }) + and not vim.startswith(p.name, '_') +end + +--- @param desc? string +--- @return string?, string? +local function get_default(desc) + if not desc then + return + end + + local default = desc:match('\n%s*%([dD]efault: ([^)]+)%)') + if default then + desc = desc:gsub('\n%s*%([dD]efault: [^)]+%)', '') + end + + return desc, default +end + +--- @param ty string +--- @param classes? table<string,nvim.luacats.parser.class> +--- @return nvim.luacats.parser.class? +local function get_class(ty, classes) + if not classes then + return + end + + local cty = ty:gsub('%s*|%s*nil', '?'):gsub('?$', ''):gsub('%[%]$', '') + + return classes[cty] +end + +--- @param obj nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.field +--- @param classes? table<string,nvim.luacats.parser.class> +local function inline_type(obj, classes) + local ty = obj.type + if not ty then + return + end + + local cls = get_class(ty, classes) + + if not cls or cls.nodoc then + return + end + + if not cls.inlinedoc then + -- Not inlining so just add a: "See |tag|." + local tag = fmt('|%s|', cls.name) + if obj.desc and obj.desc:find(tag) then + -- Tag already there + return + end + + -- TODO(lewis6991): Aim to remove this. Need this to prevent dead + -- references to types defined in runtime/lua/vim/lsp/_meta/protocol.lua + if not vim.startswith(cls.name, 'vim.') then + return + end + + obj.desc = obj.desc or '' + local period = (obj.desc == '' or vim.endswith(obj.desc, '.')) and '' or '.' + obj.desc = obj.desc .. fmt('%s See %s.', period, tag) + return + end + + local ty_isopt = (ty:match('%?$') or ty:match('%s*|%s*nil')) ~= nil + local ty_islist = (ty:match('%[%]$')) ~= nil + ty = ty_isopt and 'table?' or ty_islist and 'table[]' or 'table' + + local desc = obj.desc or '' + if cls.desc then + desc = desc .. cls.desc + elseif desc == '' then + if ty_islist then + desc = desc .. 'A list of objects with the following fields:' + elseif cls.parent then + desc = desc .. fmt('Extends |%s| with the additional fields:', cls.parent) + else + desc = desc .. 'A table with the following fields:' + end + end + + local desc_append = {} + for _, f in ipairs(cls.fields) do + if not f.access then + local fdesc, default = get_default(f.desc) + local fty = render_type(f.type, nil, default) + local fnm = fmt_field_name(f.name) + table.insert(desc_append, table.concat({ '-', fnm, fty, fdesc }, ' ')) + end + end + + desc = desc .. '\n' .. table.concat(desc_append, '\n') + obj.type = ty + obj.desc = desc +end + +--- @param xs (nvim.luacats.parser.param|nvim.luacats.parser.field)[] +--- @param generics? table<string,string> +--- @param classes? table<string,nvim.luacats.parser.class> +--- @param exclude_types? true +--- @param cfg nvim.gen_vimdoc.Config +local function render_fields_or_params(xs, generics, classes, exclude_types, cfg) + local ret = {} --- @type string[] + + xs = vim.tbl_filter(should_render_field_or_param, xs) + + local indent = 0 + for _, p in ipairs(xs) do + if p.type or p.desc then + indent = math.max(indent, #p.name + 3) + end + if exclude_types then + p.type = nil + end + end + + for _, p in ipairs(xs) do + local pdesc, default = get_default(p.desc) + p.desc = pdesc + + inline_type(p, classes) + local nm, ty = p.name, p.type + + local desc = p.classvar and string.format('See |%s|.', cfg.fn_helptag_fmt(p)) or p.desc + + local fnm = p.kind == 'operator' and fmt('op(%s)', nm) or fmt_field_name(nm) + local pnm = fmt(' • %-' .. indent .. 's', fnm) + + if ty then + local pty = render_type(ty, generics, default) + + if desc then + table.insert(ret, pnm) + if #pty > TEXT_WIDTH - indent then + vim.list_extend(ret, { ' ', pty, '\n' }) + table.insert(ret, md_to_vimdoc(desc, 9 + indent, 9 + indent, TEXT_WIDTH, true)) + else + desc = fmt('%s %s', pty, desc) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + end + else + table.insert(ret, fmt('%s %s\n', pnm, pty)) + end + else + if desc then + table.insert(ret, pnm) + table.insert(ret, md_to_vimdoc(desc, 1, 9 + indent, TEXT_WIDTH, true)) + end + end + end + + return table.concat(ret) +end + +--- @param class nvim.luacats.parser.class +--- @param classes table<string,nvim.luacats.parser.class> +--- @param cfg nvim.gen_vimdoc.Config +local function render_class(class, classes, cfg) + if class.access or class.nodoc or class.inlinedoc then + return + end + + local ret = {} --- @type string[] + + table.insert(ret, fmt('*%s*\n', class.name)) + + if class.parent then + local txt = fmt('Extends: |%s|', class.parent) + table.insert(ret, md_to_vimdoc(txt, INDENTATION, INDENTATION, TEXT_WIDTH)) + table.insert(ret, '\n') + end + + if class.desc then + table.insert(ret, md_to_vimdoc(class.desc, INDENTATION, INDENTATION, TEXT_WIDTH)) + end + + local fields_txt = render_fields_or_params(class.fields, nil, classes, nil, cfg) + if not fields_txt:match('^%s*$') then + table.insert(ret, '\n Fields: ~\n') + table.insert(ret, fields_txt) + end + table.insert(ret, '\n') + + return table.concat(ret) +end + +--- @param classes table<string,nvim.luacats.parser.class> +--- @param cfg nvim.gen_vimdoc.Config +local function render_classes(classes, cfg) + local ret = {} --- @type string[] + + for _, class in vim.spairs(classes) do + ret[#ret + 1] = render_class(class, classes, cfg) + end + + return table.concat(ret) +end + +--- @param fun nvim.luacats.parser.fun +--- @param cfg nvim.gen_vimdoc.Config +local function render_fun_header(fun, cfg) + local ret = {} --- @type string[] + + local args = {} --- @type string[] + for _, p in ipairs(fun.params or {}) do + if p.name ~= 'self' then + args[#args + 1] = fmt_field_name(p.name) + end + end + + local nm = fun.name + if fun.classvar then + nm = fmt('%s:%s', fun.classvar, nm) + end + if nm == 'vim.bo' then + nm = 'vim.bo[{bufnr}]' + end + if nm == 'vim.wo' then + nm = 'vim.wo[{winid}][{bufnr}]' + end + + local proto = fun.table and nm or nm .. '(' .. table.concat(args, ', ') .. ')' + + local tag = '*' .. cfg.fn_helptag_fmt(fun) .. '*' + + if #proto + #tag > TEXT_WIDTH - 8 then + table.insert(ret, fmt('%78s\n', tag)) + local name, pargs = proto:match('([^(]+%()(.*)') + table.insert(ret, name) + table.insert(ret, wrap(pargs, 0, #name, TEXT_WIDTH)) + else + local pad = TEXT_WIDTH - #proto - #tag + table.insert(ret, proto .. string.rep(' ', pad) .. tag) + end + + return table.concat(ret) +end + +--- @param returns nvim.luacats.parser.return[] +--- @param generics? table<string,string> +--- @param classes? table<string,nvim.luacats.parser.class> +--- @param exclude_types boolean +local function render_returns(returns, generics, classes, exclude_types) + local ret = {} --- @type string[] + + returns = vim.deepcopy(returns) + if exclude_types then + for _, r in ipairs(returns) do + r.type = nil + end + end + + if #returns > 1 then + table.insert(ret, ' Return (multiple): ~\n') + elseif #returns == 1 and next(returns[1]) then + table.insert(ret, ' Return: ~\n') + end + + for _, p in ipairs(returns) do + inline_type(p, classes) + local rnm, ty, desc = p.name, p.type, p.desc + + local blk = {} --- @type string[] + if ty then + blk[#blk + 1] = render_type(ty, generics) + end + blk[#blk + 1] = rnm + blk[#blk + 1] = desc + + table.insert(ret, md_to_vimdoc(table.concat(blk, ' '), 8, 8, TEXT_WIDTH, true)) + end + + return table.concat(ret) +end + +--- @param fun nvim.luacats.parser.fun +--- @param classes table<string,nvim.luacats.parser.class> +--- @param cfg nvim.gen_vimdoc.Config +local function render_fun(fun, classes, cfg) + if fun.access or fun.deprecated or fun.nodoc then + return + end + + if cfg.fn_name_pat and not fun.name:match(cfg.fn_name_pat) then + return + end + + if vim.startswith(fun.name, '_') or fun.name:find('[:.]_') then + return + end + + local ret = {} --- @type string[] + + table.insert(ret, render_fun_header(fun, cfg)) + table.insert(ret, '\n') + + if fun.since then + local since = assert(tonumber(fun.since), 'invalid @since on ' .. fun.name) + local info = nvim_api_info() + if since == 0 or (info.prerelease and since == info.level) then + -- Experimental = (since==0 or current prerelease) + local s = 'WARNING: This feature is experimental/unstable.' + table.insert(ret, md_to_vimdoc(s, INDENTATION, INDENTATION, TEXT_WIDTH)) + table.insert(ret, '\n') + else + local v = assert(util.version_level[since], 'invalid @since on ' .. fun.name) + fun.attrs = fun.attrs or {} + table.insert(fun.attrs, ('Since: %s'):format(v)) + end + end + + if fun.desc then + table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH)) + end + + if fun.notes then + table.insert(ret, '\n Note: ~\n') + for _, p in ipairs(fun.notes) do + table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true)) + end + end + + if fun.attrs then + table.insert(ret, '\n Attributes: ~\n') + for _, attr in ipairs(fun.attrs) do + local attr_str = ({ + textlock = 'not allowed when |textlock| is active or in the |cmdwin|', + textlock_allow_cmdwin = 'not allowed when |textlock| is active', + fast = '|api-fast|', + remote_only = '|RPC| only', + lua_only = 'Lua |vim.api| only', + })[attr] or attr + table.insert(ret, fmt(' %s\n', attr_str)) + end + end + + if fun.params and #fun.params > 0 then + local param_txt = + render_fields_or_params(fun.params, fun.generics, classes, cfg.exclude_types, cfg) + if not param_txt:match('^%s*$') then + table.insert(ret, '\n Parameters: ~\n') + ret[#ret + 1] = param_txt + end + end + + if fun.returns then + local txt = render_returns(fun.returns, fun.generics, classes, cfg.exclude_types) + if not txt:match('^%s*$') then + table.insert(ret, '\n') + ret[#ret + 1] = txt + end + end + + if fun.see then + table.insert(ret, '\n See also: ~\n') + for _, p in ipairs(fun.see) do + table.insert(ret, ' • ' .. md_to_vimdoc(p.desc, 0, 8, TEXT_WIDTH, true)) + end + end + + table.insert(ret, '\n') + return table.concat(ret) +end + +--- @param funs nvim.luacats.parser.fun[] +--- @param classes table<string,nvim.luacats.parser.class> +--- @param cfg nvim.gen_vimdoc.Config +local function render_funs(funs, classes, cfg) + local ret = {} --- @type string[] + + for _, f in ipairs(funs) do + if cfg.fn_xform then + cfg.fn_xform(f) + end + ret[#ret + 1] = render_fun(f, classes, cfg) + end + + -- Sort via prototype. Experimental API functions ("nvim__") sort last. + table.sort(ret, function(a, b) + local a1 = ('\n' .. a):match('\n[a-zA-Z_][^\n]+\n') + local b1 = ('\n' .. b):match('\n[a-zA-Z_][^\n]+\n') + + local a1__ = a1:find('^%s*nvim__') and 1 or 0 + local b1__ = b1:find('^%s*nvim__') and 1 or 0 + if a1__ ~= b1__ then + return a1__ < b1__ + end + + return a1:lower() < b1:lower() + end) + + return table.concat(ret) +end + +--- @return string +local function get_script_path() + local str = debug.getinfo(2, 'S').source:sub(2) + return str:match('(.*[/\\])') or './' +end + +local script_path = get_script_path() +local base_dir = vim.fs.dirname(vim.fs.dirname(vim.fs.dirname(script_path))) + +local function delete_lines_below(doc_file, tokenstr) + local lines = {} --- @type string[] + local found = false + for line in io.lines(doc_file) do + if line:find(vim.pesc(tokenstr)) then + found = true + break + end + lines[#lines + 1] = line + end + if not found then + error(fmt('not found: %s in %s', tokenstr, doc_file)) + end + lines[#lines] = nil + local fp = assert(io.open(doc_file, 'w')) + fp:write(table.concat(lines, '\n')) + fp:write('\n') + fp:close() +end + +--- @param x string +local function mktitle(x) + if x == 'ui' then + return 'UI' + end + return x:sub(1, 1):upper() .. x:sub(2) +end + +--- @class nvim.gen_vimdoc.Section +--- @field name string +--- @field title string +--- @field help_tag string +--- @field funs_txt string +--- @field doc? string[] + +--- @param filename string +--- @param cfg nvim.gen_vimdoc.Config +--- @param section_docs table<string,nvim.gen_vimdoc.Section> +--- @param funs_txt string +--- @return nvim.gen_vimdoc.Section? +local function make_section(filename, cfg, section_docs, funs_txt) + -- filename: e.g., 'autocmd.c' + -- name: e.g. 'autocmd' + local name = filename:match('(.*)%.[a-z]+') + + -- Formatted (this is what's going to be written in the vimdoc) + -- e.g., "Autocmd Functions" + local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name) + + -- section tag: e.g., "*api-autocmd*" + local help_labels = cfg.helptag_fmt(sectname) + if type(help_labels) == 'table' then + help_labels = table.concat(help_labels, '* *') + end + local help_tags = '*' .. help_labels .. '*' + + if funs_txt == '' and #section_docs == 0 then + return + end + + return { + name = sectname, + title = cfg.section_fmt(sectname), + help_tag = help_tags, + funs_txt = funs_txt, + doc = section_docs, + } +end + +--- @param section nvim.gen_vimdoc.Section +--- @param add_header? boolean +local function render_section(section, add_header) + local doc = {} --- @type string[] + + if add_header ~= false then + vim.list_extend(doc, { + string.rep('=', TEXT_WIDTH), + '\n', + section.title, + fmt('%' .. (TEXT_WIDTH - section.title:len()) .. 's', section.help_tag), + }) + end + + local sdoc = '\n\n' .. table.concat(section.doc or {}, '\n') + if sdoc:find('[^%s]') then + doc[#doc + 1] = sdoc + end + + if section.funs_txt then + table.insert(doc, '\n\n') + table.insert(doc, section.funs_txt) + end + + return table.concat(doc) +end + +local parsers = { + lua = luacats_parser.parse, + c = cdoc_parser.parse, + h = cdoc_parser.parse, +} + +--- @param files string[] +local function expand_files(files) + for k, f in pairs(files) do + if vim.fn.isdirectory(f) == 1 then + table.remove(files, k) + for path, ty in vim.fs.dir(f) do + if ty == 'file' then + table.insert(files, vim.fs.joinpath(f, path)) + end + end + end + end +end + +--- @param cfg nvim.gen_vimdoc.Config +local function gen_target(cfg) + cfg.fn_helptag_fmt = cfg.fn_helptag_fmt or fn_helptag_fmt_common + print('Target:', cfg.filename) + local sections = {} --- @type table<string,nvim.gen_vimdoc.Section> + + expand_files(cfg.files) + + --- @type table<string,[table<string,nvim.luacats.parser.class>, nvim.luacats.parser.fun[], string[]]> + local file_results = {} + + --- @type table<string,nvim.luacats.parser.class> + local all_classes = {} + + --- First pass so we can collect all classes + for _, f in vim.spairs(cfg.files) do + local ext = assert(f:match('%.([^.]+)$')) --[[@as 'h'|'c'|'lua']] + local parser = assert(parsers[ext]) + local classes, funs, briefs = parser(f) + file_results[f] = { classes, funs, briefs } + all_classes = vim.tbl_extend('error', all_classes, classes) + end + + for f, r in vim.spairs(file_results) do + local classes, funs, briefs = r[1], r[2], r[3] + + local briefs_txt = {} --- @type string[] + for _, b in ipairs(briefs) do + briefs_txt[#briefs_txt + 1] = md_to_vimdoc(b, 0, 0, TEXT_WIDTH) + end + print(' Processing file:', f) + local funs_txt = render_funs(funs, all_classes, cfg) + if next(classes) then + local classes_txt = render_classes(classes, cfg) + if vim.trim(classes_txt) ~= '' then + funs_txt = classes_txt .. '\n' .. funs_txt + end + end + -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua` + local f_base = vim.fs.basename(f) + sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt) + end + + local first_section_tag = sections[cfg.section_order[1]].help_tag + local docs = {} --- @type string[] + for _, f in ipairs(cfg.section_order) do + local section = sections[f] + if section then + print(string.format(" Rendering section: '%s'", section.title)) + local add_sep_and_header = not vim.tbl_contains(cfg.append_only or {}, f) + docs[#docs + 1] = render_section(section, add_sep_and_header) + end + end + + table.insert( + docs, + fmt(' vim:tw=78:ts=8:sw=%d:sts=%d:et:ft=help:norl:\n', INDENTATION, INDENTATION) + ) + + local doc_file = vim.fs.joinpath(base_dir, 'runtime', 'doc', cfg.filename) + + if vim.uv.fs_stat(doc_file) then + delete_lines_below(doc_file, first_section_tag) + end + + local fp = assert(io.open(doc_file, 'a')) + fp:write(table.concat(docs, '\n')) + fp:close() +end + +local function run() + for _, cfg in vim.spairs(config) do + gen_target(cfg) + end +end + +run() diff --git a/src/nvim/generators/gen_vimvim.lua b/src/gen/gen_vimvim.lua index 3817735a55..d2b1f48a4c 100644 --- a/src/nvim/generators/gen_vimvim.lua +++ b/src/gen/gen_vimvim.lua @@ -15,9 +15,9 @@ local function w(s) end end -local options = require('options') -local auevents = require('auevents') -local ex_cmds = require('ex_cmds') +local options = require('nvim.options') +local auevents = require('nvim.auevents') +local ex_cmds = require('nvim.ex_cmds') local function cmd_kw(prev_cmd, cmd) if not prev_cmd then diff --git a/src/nvim/generators/hashy.lua b/src/gen/hashy.lua index 74b7655324..74b7655324 100644 --- a/src/nvim/generators/hashy.lua +++ b/src/gen/hashy.lua diff --git a/src/gen/luacats_grammar.lua b/src/gen/luacats_grammar.lua new file mode 100644 index 0000000000..b700bcf58f --- /dev/null +++ b/src/gen/luacats_grammar.lua @@ -0,0 +1,207 @@ +--[[! +LPEG grammar for LuaCATS +]] + +local lpeg = vim.lpeg +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local C, Ct, Cg = lpeg.C, lpeg.Ct, lpeg.Cg + +--- @param x vim.lpeg.Pattern +local function rep(x) + return x ^ 0 +end + +--- @param x vim.lpeg.Pattern +local function rep1(x) + return x ^ 1 +end + +--- @param x vim.lpeg.Pattern +local function opt(x) + return x ^ -1 +end + +local ws = rep1(S(' \t')) +local fill = opt(ws) +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') +local num = R('09') + +--- @param x string | vim.lpeg.Pattern +local function Pf(x) + return fill * P(x) * fill +end + +--- @param x string | vim.lpeg.Pattern +local function Plf(x) + return fill * P(x) +end + +--- @param x string +local function Sf(x) + return fill * S(x) * fill +end + +--- @param x vim.lpeg.Pattern +local function paren(x) + return Pf('(') * x * fill * P(')') +end + +--- @param x vim.lpeg.Pattern +local function parenOpt(x) + return paren(x) + x +end + +--- @param x vim.lpeg.Pattern +local function comma1(x) + return parenOpt(x * rep(Pf(',') * x)) +end + +--- @param x vim.lpeg.Pattern +local function comma(x) + return opt(comma1(x)) +end + +--- @type table<string,vim.lpeg.Pattern> +local v = setmetatable({}, { + __index = function(_, k) + return lpeg.V(k) + end, +}) + +--- @class nvim.luacats.Param +--- @field kind 'param' +--- @field name string +--- @field type string +--- @field desc? string + +--- @class nvim.luacats.Return +--- @field kind 'return' +--- @field [integer] { type: string, name?: string} +--- @field desc? string + +--- @class nvim.luacats.Generic +--- @field kind 'generic' +--- @field name string +--- @field type? string + +--- @class nvim.luacats.Class +--- @field kind 'class' +--- @field name string +--- @field parent? string +--- @field access? 'private'|'protected'|'package' + +--- @class nvim.luacats.Field +--- @field kind 'field' +--- @field name string +--- @field type string +--- @field desc? string +--- @field access? 'private'|'protected'|'package' + +--- @class nvim.luacats.Note +--- @field desc? string + +--- @alias nvim.luacats.grammar.result +--- | nvim.luacats.Param +--- | nvim.luacats.Return +--- | nvim.luacats.Generic +--- | nvim.luacats.Class +--- | nvim.luacats.Field +--- | nvim.luacats.Note + +--- @class nvim.luacats.grammar +--- @field match fun(self, input: string): nvim.luacats.grammar.result? + +local function annot(nm, pat) + if type(nm) == 'string' then + nm = P(nm) + end + if pat then + return Ct(Cg(P(nm), 'kind') * fill * pat) + end + return Ct(Cg(P(nm), 'kind')) +end + +local colon = Pf(':') +local ellipsis = P('...') +local ident_first = P('_') + letter +local ident = ident_first * rep(ident_first + num) +local opt_ident = ident * opt(P('?')) +local ty_ident_sep = S('-._') +local ty_ident = ident * rep(ty_ident_sep * ident) +local string_single = P "'" * rep(any - P "'") * P "'" +local string_double = P('"') * rep(any - P('"')) * P('"') +local generic = P('`') * ty_ident * P('`') +local literal = string_single + string_double + (opt(P('-')) * rep1(num)) + P('false') + P('true') +local ty_prims = ty_ident + literal + generic + +local array_postfix = rep1(Plf('[]')) +local opt_postfix = rep1(Plf('?')) +local rep_array_opt_postfix = rep(array_postfix + opt_postfix) + +local typedef = P({ + 'typedef', + typedef = C(v.type), + + type = v.ty * rep_array_opt_postfix * rep(Pf('|') * v.ty * rep_array_opt_postfix), + ty = v.composite + paren(v.typedef), + composite = (v.types * array_postfix) + (v.types * opt_postfix) + v.types, + types = v.generics + v.kv_table + v.tuple + v.dict + v.table_literal + v.fun + ty_prims, + + tuple = Pf('[') * comma1(v.type) * Plf(']'), + dict = Pf('{') * comma1(Pf('[') * v.type * Pf(']') * colon * v.type) * Plf('}'), + kv_table = Pf('table') * Pf('<') * v.type * Pf(',') * v.type * Plf('>'), + table_literal = Pf('{') * comma1(opt_ident * Pf(':') * v.type) * Plf('}'), + fun_param = (opt_ident + ellipsis) * opt(colon * v.type), + fun_ret = v.type + (ellipsis * opt(colon * v.type)), + fun = Pf('fun') * paren(comma(v.fun_param)) * opt(Pf(':') * comma1(v.fun_ret)), + generics = P(ty_ident) * Pf('<') * comma1(v.type) * Plf('>'), +}) / function(match) + return vim.trim(match):gsub('^%((.*)%)$', '%1'):gsub('%?+', '?') +end + +local access = P('private') + P('protected') + P('package') +local caccess = Cg(access, 'access') +local cattr = Cg(comma(access + P('exact')), 'access') +local desc_delim = Sf '#:' + ws +local desc = Cg(rep(any), 'desc') +local opt_desc = opt(desc_delim * desc) +local ty_name = Cg(ty_ident, 'name') +local opt_parent = opt(colon * Cg(ty_ident, 'parent')) +local lname = (ident + ellipsis) * opt(P('?')) + +local grammar = P { + rep1(P('@') * (v.ats + v.ext_ats)), + + ats = annot('param', Cg(lname, 'name') * ws * v.ctype * opt_desc) + + annot('return', comma1(Ct(v.ctype * opt(ws * (ty_name + Cg(ellipsis, 'name'))))) * opt_desc) + + annot('type', comma1(Ct(v.ctype)) * opt_desc) + + annot('cast', ty_name * ws * opt(Sf('+-')) * v.ctype) + + annot('generic', ty_name * opt(colon * v.ctype)) + + annot('class', opt(paren(cattr)) * fill * ty_name * opt_parent) + + annot('field', opt(caccess * ws) * v.field_name * ws * v.ctype * opt_desc) + + annot('operator', ty_name * opt(paren(Cg(v.ctype, 'argtype'))) * colon * v.ctype) + + annot(access) + + annot('deprecated') + + annot('alias', ty_name * opt(ws * v.ctype)) + + annot('enum', ty_name) + + annot('overload', v.ctype) + + annot('see', opt(desc_delim) * desc) + + annot('diagnostic', opt(desc_delim) * desc) + + annot('meta'), + + --- Custom extensions + ext_ats = ( + annot('note', desc) + + annot('since', desc) + + annot('nodoc') + + annot('inlinedoc') + + annot('brief', desc) + ), + + field_name = Cg(lname + (v.ty_index * opt(P('?'))), 'name'), + ty_index = C(Pf('[') * typedef * fill * P(']')), + ctype = Cg(typedef, 'type'), +} + +return grammar --[[@as nvim.luacats.grammar]] diff --git a/src/gen/luacats_parser.lua b/src/gen/luacats_parser.lua new file mode 100644 index 0000000000..36bdc44076 --- /dev/null +++ b/src/gen/luacats_parser.lua @@ -0,0 +1,535 @@ +local luacats_grammar = require('gen.luacats_grammar') + +--- @class nvim.luacats.parser.param : nvim.luacats.Param + +--- @class nvim.luacats.parser.return +--- @field name string +--- @field type string +--- @field desc string + +--- @class nvim.luacats.parser.note +--- @field desc string + +--- @class nvim.luacats.parser.brief +--- @field kind 'brief' +--- @field desc string + +--- @class nvim.luacats.parser.alias +--- @field kind 'alias' +--- @field type string[] +--- @field desc string + +--- @class nvim.luacats.parser.fun +--- @field name string +--- @field params nvim.luacats.parser.param[] +--- @field returns nvim.luacats.parser.return[] +--- @field desc string +--- @field access? 'private'|'package'|'protected' +--- @field class? string +--- @field module? string +--- @field modvar? string +--- @field classvar? string +--- @field deprecated? true +--- @field since? string +--- @field attrs? string[] +--- @field nodoc? true +--- @field generics? table<string,string> +--- @field table? true +--- @field notes? nvim.luacats.parser.note[] +--- @field see? nvim.luacats.parser.note[] + +--- @class nvim.luacats.parser.field : nvim.luacats.Field +--- @field classvar? string +--- @field nodoc? true + +--- @class nvim.luacats.parser.class : nvim.luacats.Class +--- @field desc? string +--- @field nodoc? true +--- @field inlinedoc? true +--- @field fields nvim.luacats.parser.field[] +--- @field notes? string[] + +--- @class nvim.luacats.parser.State +--- @field doc_lines? string[] +--- @field cur_obj? nvim.luacats.parser.obj +--- @field last_doc_item? nvim.luacats.parser.param|nvim.luacats.parser.return|nvim.luacats.parser.note +--- @field last_doc_item_indent? integer + +--- @alias nvim.luacats.parser.obj +--- | nvim.luacats.parser.class +--- | nvim.luacats.parser.fun +--- | nvim.luacats.parser.brief +--- | nvim.luacats.parser.alias + +-- Remove this when we document classes properly +--- Some doc lines have the form: +--- param name some.complex.type (table) description +--- if so then transform the line to remove the complex type: +--- param name (table) description +--- @param line string +local function use_type_alt(line) + for _, type in ipairs({ 'table', 'function' }) do + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', '@param %1 %2') + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', '@param %1 %2') + line = line:gsub('@param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '%?)%)', '@param %1 %2') + + line = line:gsub('@return%s+.*%((' .. type .. ')%)', '@return %1') + line = line:gsub('@return%s+.*%((' .. type .. '|nil)%)', '@return %1') + line = line:gsub('@return%s+.*%((' .. type .. '%?)%)', '@return %1') + end + return line +end + +--- If we collected any `---` lines. Add them to the existing (or new) object +--- Used for function/class descriptions and multiline param descriptions. +--- @param state nvim.luacats.parser.State +local function add_doc_lines_to_obj(state) + if state.doc_lines then + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + local txt = table.concat(state.doc_lines, '\n') + if cur_obj.desc then + cur_obj.desc = cur_obj.desc .. '\n' .. txt + else + cur_obj.desc = txt + end + state.doc_lines = nil + end +end + +--- @param line string +--- @param state nvim.luacats.parser.State +local function process_doc_line(line, state) + line = line:sub(4):gsub('^%s+@', '@') + line = use_type_alt(line) + + local parsed = luacats_grammar:match(line) + + if not parsed then + if line:match('^ ') then + line = line:sub(2) + end + + if state.last_doc_item then + if not state.last_doc_item_indent then + state.last_doc_item_indent = #line:match('^%s*') + 1 + end + state.last_doc_item.desc = (state.last_doc_item.desc or '') + .. '\n' + .. line:sub(state.last_doc_item_indent or 1) + else + state.doc_lines = state.doc_lines or {} + table.insert(state.doc_lines, line) + end + return + end + + state.last_doc_item_indent = nil + state.last_doc_item = nil + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + local kind = parsed.kind + + if kind == 'brief' then + state.cur_obj = { + kind = 'brief', + desc = parsed.desc, + } + elseif kind == 'class' then + --- @cast parsed nvim.luacats.Class + cur_obj.kind = 'class' + cur_obj.name = parsed.name + cur_obj.parent = parsed.parent + cur_obj.access = parsed.access + cur_obj.desc = state.doc_lines and table.concat(state.doc_lines, '\n') or nil + state.doc_lines = nil + cur_obj.fields = {} + elseif kind == 'field' then + --- @cast parsed nvim.luacats.Field + parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil + if parsed.desc then + parsed.desc = vim.trim(parsed.desc) + end + table.insert(cur_obj.fields, parsed) + state.doc_lines = nil + elseif kind == 'operator' then + parsed.desc = parsed.desc or state.doc_lines and table.concat(state.doc_lines, '\n') or nil + if parsed.desc then + parsed.desc = vim.trim(parsed.desc) + end + table.insert(cur_obj.fields, parsed) + state.doc_lines = nil + elseif kind == 'param' then + state.last_doc_item_indent = nil + cur_obj.params = cur_obj.params or {} + if vim.endswith(parsed.name, '?') then + parsed.name = parsed.name:sub(1, -2) + parsed.type = parsed.type .. '?' + end + state.last_doc_item = { + name = parsed.name, + type = parsed.type, + desc = parsed.desc, + } + table.insert(cur_obj.params, state.last_doc_item) + elseif kind == 'return' then + cur_obj.returns = cur_obj.returns or {} + for _, t in ipairs(parsed) do + table.insert(cur_obj.returns, { + name = t.name, + type = t.type, + desc = parsed.desc, + }) + end + state.last_doc_item_indent = nil + state.last_doc_item = cur_obj.returns[#cur_obj.returns] + elseif kind == 'private' then + cur_obj.access = 'private' + elseif kind == 'package' then + cur_obj.access = 'package' + elseif kind == 'protected' then + cur_obj.access = 'protected' + elseif kind == 'deprecated' then + cur_obj.deprecated = true + elseif kind == 'inlinedoc' then + cur_obj.inlinedoc = true + elseif kind == 'nodoc' then + cur_obj.nodoc = true + elseif kind == 'since' then + cur_obj.since = parsed.desc + elseif kind == 'see' then + cur_obj.see = cur_obj.see or {} + table.insert(cur_obj.see, { desc = parsed.desc }) + elseif kind == 'note' then + state.last_doc_item_indent = nil + state.last_doc_item = { + desc = parsed.desc, + } + cur_obj.notes = cur_obj.notes or {} + table.insert(cur_obj.notes, state.last_doc_item) + elseif kind == 'type' then + cur_obj.desc = parsed.desc + parsed.desc = nil + parsed.kind = nil + cur_obj.type = parsed + elseif kind == 'alias' then + state.cur_obj = { + kind = 'alias', + desc = parsed.desc, + } + elseif kind == 'enum' then + -- TODO + state.doc_lines = nil + elseif + vim.tbl_contains({ + 'diagnostic', + 'cast', + 'overload', + 'meta', + }, kind) + then + -- Ignore + return + elseif kind == 'generic' then + cur_obj.generics = cur_obj.generics or {} + cur_obj.generics[parsed.name] = parsed.type or 'any' + else + error('Unhandled' .. vim.inspect(parsed)) + end +end + +--- @param fun nvim.luacats.parser.fun +--- @return nvim.luacats.parser.field +local function fun2field(fun) + local parts = { 'fun(' } + + local params = {} ---@type string[] + for _, p in ipairs(fun.params or {}) do + params[#params + 1] = string.format('%s: %s', p.name, p.type) + end + parts[#parts + 1] = table.concat(params, ', ') + parts[#parts + 1] = ')' + if fun.returns then + parts[#parts + 1] = ': ' + local tys = {} --- @type string[] + for _, p in ipairs(fun.returns) do + tys[#tys + 1] = p.type + end + parts[#parts + 1] = table.concat(tys, ', ') + end + + return { + name = fun.name, + type = table.concat(parts, ''), + access = fun.access, + desc = fun.desc, + nodoc = fun.nodoc, + } +end + +--- Function to normalize known form for declaring functions and normalize into a more standard +--- form. +--- @param line string +--- @return string +local function filter_decl(line) + -- M.fun = vim._memoize(function(...) + -- -> + -- function M.fun(...) + line = line:gsub('^local (.+) = memoize%([^,]+, function%((.*)%)$', 'local function %1(%2)') + line = line:gsub('^(.+) = memoize%([^,]+, function%((.*)%)$', 'function %1(%2)') + return line +end + +--- @param line string +--- @param state nvim.luacats.parser.State +--- @param classes table<string,nvim.luacats.parser.class> +--- @param classvars table<string,string> +--- @param has_indent boolean +local function process_lua_line(line, state, classes, classvars, has_indent) + line = filter_decl(line) + + if state.cur_obj and state.cur_obj.kind == 'class' then + local nm = line:match('^local%s+([a-zA-Z0-9_]+)%s*=') + if nm then + classvars[nm] = state.cur_obj.name + end + return + end + + do + local parent_tbl, sep, fun_or_meth_nm = + line:match('^function%s+([a-zA-Z0-9_]+)([.:])([a-zA-Z0-9_]+)%s*%(') + if parent_tbl then + -- Have a decl. Ensure cur_obj + state.cur_obj = state.cur_obj or {} + local cur_obj = assert(state.cur_obj) + + -- Match `Class:foo` methods for defined classes + local class = classvars[parent_tbl] + if class then + --- @cast cur_obj nvim.luacats.parser.fun + cur_obj.name = fun_or_meth_nm + cur_obj.class = class + cur_obj.classvar = parent_tbl + -- Add self param to methods + if sep == ':' then + cur_obj.params = cur_obj.params or {} + table.insert(cur_obj.params, 1, { + name = 'self', + type = class, + }) + end + + -- Add method as the field to the class + local cls = classes[class] + local field = fun2field(cur_obj) + field.classvar = cur_obj.classvar + table.insert(cls.fields, field) + return + end + + -- Match `M.foo` + if cur_obj and parent_tbl == cur_obj.modvar then + cur_obj.name = fun_or_meth_nm + return + end + end + end + + do + -- Handle: `function A.B.C.foo(...)` + local fn_nm = line:match('^function%s+([.a-zA-Z0-9_]+)%s*%(') + if fn_nm then + state.cur_obj = state.cur_obj or {} + state.cur_obj.name = fn_nm + return + end + end + + do + -- Handle: `M.foo = {...}` where `M` is the modvar + local parent_tbl, tbl_nm = line:match('([a-zA-Z_]+)%.([a-zA-Z0-9_]+)%s*=') + if state.cur_obj and parent_tbl and parent_tbl == state.cur_obj.modvar then + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + do + -- Handle: `foo = {...}` + local tbl_nm = line:match('^([a-zA-Z0-9_]+)%s*=') + if tbl_nm and not has_indent then + state.cur_obj = state.cur_obj or {} + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + do + -- Handle: `vim.foo = {...}` + local tbl_nm = line:match('^(vim%.[a-zA-Z0-9_]+)%s*=') + if state.cur_obj and tbl_nm and not has_indent then + state.cur_obj.name = tbl_nm + state.cur_obj.table = true + return + end + end + + if state.cur_obj then + if line:find('^%s*%-%- luacheck:') then + state.cur_obj = nil + elseif line:find('^%s*local%s+') then + state.cur_obj = nil + elseif line:find('^%s*return%s+') then + state.cur_obj = nil + elseif line:find('^%s*[a-zA-Z_.]+%(%s+') then + state.cur_obj = nil + end + end +end + +--- Determine the table name used to export functions of a module +--- Usually this is `M`. +--- @param str string +--- @return string? +local function determine_modvar(str) + local modvar --- @type string? + for line in vim.gsplit(str, '\n') do + do + --- @type string? + local m = line:match('^return%s+([a-zA-Z_]+)') + if m then + modvar = m + end + end + do + --- @type string? + local m = line:match('^return%s+setmetatable%(([a-zA-Z_]+),') + if m then + modvar = m + end + end + end + return modvar +end + +--- @param obj nvim.luacats.parser.obj +--- @param funs nvim.luacats.parser.fun[] +--- @param classes table<string,nvim.luacats.parser.class> +--- @param briefs string[] +--- @param uncommitted nvim.luacats.parser.obj[] +local function commit_obj(obj, classes, funs, briefs, uncommitted) + local commit = false + if obj.kind == 'class' then + --- @cast obj nvim.luacats.parser.class + if not classes[obj.name] then + classes[obj.name] = obj + commit = true + end + elseif obj.kind == 'alias' then + -- Just pretend + commit = true + elseif obj.kind == 'brief' then + --- @cast obj nvim.luacats.parser.brief` + briefs[#briefs + 1] = obj.desc + commit = true + else + --- @cast obj nvim.luacats.parser.fun` + if obj.name then + funs[#funs + 1] = obj + commit = true + end + end + if not commit then + table.insert(uncommitted, obj) + end + return commit +end + +--- @param filename string +--- @param uncommitted nvim.luacats.parser.obj[] +-- luacheck: no unused +local function dump_uncommitted(filename, uncommitted) + local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt' + if #uncommitted > 0 then + print(string.format('Could not commit %d objects in %s', #uncommitted, filename)) + vim.fn.mkdir(vim.fs.dirname(out_path), 'p') + local f = assert(io.open(out_path, 'w')) + for i, x in ipairs(uncommitted) do + f:write(i) + f:write(': ') + f:write(vim.inspect(x)) + f:write('\n') + end + f:close() + else + vim.fn.delete(out_path) + end +end + +local M = {} + +function M.parse_str(str, filename) + local funs = {} --- @type nvim.luacats.parser.fun[] + local classes = {} --- @type table<string,nvim.luacats.parser.class> + local briefs = {} --- @type string[] + + local mod_return = determine_modvar(str) + + --- @type string + local module = filename:match('.*/lua/([a-z_][a-z0-9_/]+)%.lua') or filename + module = module:gsub('/', '.') + + local classvars = {} --- @type table<string,string> + + local state = {} --- @type nvim.luacats.parser.State + + -- Keep track of any partial objects we don't commit + local uncommitted = {} --- @type nvim.luacats.parser.obj[] + + for line in vim.gsplit(str, '\n') do + local has_indent = line:match('^%s+') ~= nil + line = vim.trim(line) + if vim.startswith(line, '---') then + process_doc_line(line, state) + else + add_doc_lines_to_obj(state) + + if state.cur_obj then + state.cur_obj.modvar = mod_return + state.cur_obj.module = module + end + + process_lua_line(line, state, classes, classvars, has_indent) + + -- Commit the object + local cur_obj = state.cur_obj + if cur_obj then + if not commit_obj(cur_obj, classes, funs, briefs, uncommitted) then + --- @diagnostic disable-next-line:inject-field + cur_obj.line = line + end + end + + state = {} + end + end + + -- dump_uncommitted(filename, uncommitted) + + return classes, funs, briefs, uncommitted +end + +--- @param filename string +function M.parse(filename) + local f = assert(io.open(filename, 'r')) + local txt = f:read('*all') + f:close() + + return M.parse_str(txt, filename) +end + +return M diff --git a/src/nvim/generators/nvim_version.lua.in b/src/gen/nvim_version.lua.in index c29141fc68..c29141fc68 100644 --- a/src/nvim/generators/nvim_version.lua.in +++ b/src/gen/nvim_version.lua.in diff --git a/src/gen/preload.lua b/src/gen/preload.lua new file mode 100644 index 0000000000..4856d8d7a1 --- /dev/null +++ b/src/gen/preload.lua @@ -0,0 +1,6 @@ +local srcdir = table.remove(arg, 1) + +package.path = (srcdir .. '/src/?.lua;') .. (srcdir .. '/runtime/lua/?.lua;') .. package.path + +arg[0] = table.remove(arg, 1) +return loadfile(arg[0])() diff --git a/src/nvim/generators/preload.lua b/src/gen/preload_nlua.lua index e14671074c..a1d89105bc 100644 --- a/src/nvim/generators/preload.lua +++ b/src/gen/preload_nlua.lua @@ -1,8 +1,12 @@ local srcdir = table.remove(arg, 1) local nlualib = table.remove(arg, 1) local gendir = table.remove(arg, 1) -package.path = srcdir .. '/src/nvim/?.lua;' .. srcdir .. '/runtime/lua/?.lua;' .. package.path -package.path = gendir .. '/?.lua;' .. package.path + +package.path = (srcdir .. '/src/?.lua;') + .. (srcdir .. '/runtime/lua/?.lua;') + .. (gendir .. '/?.lua;') + .. package.path + _G.vim = require 'vim.shared' _G.vim.inspect = require 'vim.inspect' package.cpath = package.cpath .. ';' .. nlualib diff --git a/src/gen/util.lua b/src/gen/util.lua new file mode 100644 index 0000000000..5940221abe --- /dev/null +++ b/src/gen/util.lua @@ -0,0 +1,399 @@ +-- TODO(justinmk): move most of this to `vim.text`. + +local fmt = string.format + +--- @class nvim.util.MDNode +--- @field [integer] nvim.util.MDNode +--- @field type string +--- @field text? string + +local INDENTATION = 4 + +local NBSP = string.char(160) + +local M = {} + +local function contains(t, xs) + return vim.tbl_contains(xs, t) +end + +-- Map of api_level:version, by inspection of: +-- :lua= vim.mpack.decode(vim.fn.readfile('test/functional/fixtures/api_level_9.mpack','B')).version +M.version_level = { + [13] = '0.11.0', + [12] = '0.10.0', + [11] = '0.9.0', + [10] = '0.8.0', + [9] = '0.7.0', + [8] = '0.6.0', + [7] = '0.5.0', + [6] = '0.4.0', + [5] = '0.3.2', + [4] = '0.3.0', + [3] = '0.2.1', + [2] = '0.2.0', + [1] = '0.1.0', +} + +--- @param txt string +--- @param srow integer +--- @param scol integer +--- @param erow? integer +--- @param ecol? integer +--- @return string +local function slice_text(txt, srow, scol, erow, ecol) + local lines = vim.split(txt, '\n') + + if srow == erow then + return lines[srow + 1]:sub(scol + 1, ecol) + end + + if erow then + -- Trim the end + for _ = erow + 2, #lines do + table.remove(lines, #lines) + end + end + + -- Trim the start + for _ = 1, srow do + table.remove(lines, 1) + end + + lines[1] = lines[1]:sub(scol + 1) + lines[#lines] = lines[#lines]:sub(1, ecol) + + return table.concat(lines, '\n') +end + +--- @param text string +--- @return nvim.util.MDNode +local function parse_md_inline(text) + local parser = vim.treesitter.languagetree.new(text, 'markdown_inline') + local root = parser:parse(true)[1]:root() + + --- @param node TSNode + --- @return nvim.util.MDNode? + local function extract(node) + local ntype = node:type() + + if ntype:match('^%p$') then + return + end + + --- @type table<any,any> + local ret = { type = ntype } + ret.text = vim.treesitter.get_node_text(node, text) + + local row, col = 0, 0 + + for child, child_field in node:iter_children() do + local e = extract(child) + if e and ntype == 'inline' then + local srow, scol = child:start() + if (srow == row and scol > col) or srow > row then + local t = slice_text(ret.text, row, col, srow, scol) + if t and t ~= '' then + table.insert(ret, { type = 'text', j = true, text = t }) + end + end + row, col = child:end_() + end + + if child_field then + ret[child_field] = e + else + table.insert(ret, e) + end + end + + if ntype == 'inline' and (row > 0 or col > 0) then + local t = slice_text(ret.text, row, col) + if t and t ~= '' then + table.insert(ret, { type = 'text', text = t }) + end + end + + return ret + end + + return extract(root) or {} +end + +--- @param text string +--- @return nvim.util.MDNode +local function parse_md(text) + local parser = vim.treesitter.languagetree.new(text, 'markdown', { + injections = { markdown = '' }, + }) + + local root = parser:parse(true)[1]:root() + + local EXCLUDE_TEXT_TYPE = { + list = true, + list_item = true, + section = true, + document = true, + fenced_code_block = true, + fenced_code_block_delimiter = true, + } + + --- @param node TSNode + --- @return nvim.util.MDNode? + local function extract(node) + local ntype = node:type() + + if ntype:match('^%p$') or contains(ntype, { 'block_continuation' }) then + return + end + + --- @type table<any,any> + local ret = { type = ntype } + + if not EXCLUDE_TEXT_TYPE[ntype] then + ret.text = vim.treesitter.get_node_text(node, text) + end + + if ntype == 'inline' then + ret = parse_md_inline(ret.text) + end + + for child, child_field in node:iter_children() do + local e = extract(child) + if child_field then + ret[child_field] = e + else + table.insert(ret, e) + end + end + + return ret + end + + return extract(root) or {} +end + +--- Prefixes each line in `text`. +--- +--- Does not wrap, not important for "meta" files? (You probably want md_to_vimdoc instead.) +--- +--- @param text string +--- @param prefix_ string +function M.prefix_lines(prefix_, text) + local r = '' + for _, l in ipairs(vim.split(text, '\n', { plain = true })) do + r = r .. vim.trim(prefix_ .. l) .. '\n' + end + return r +end + +--- @param x string +--- @param start_indent integer +--- @param indent integer +--- @param text_width integer +--- @return string +function M.wrap(x, start_indent, indent, text_width) + local words = vim.split(vim.trim(x), '%s+') + local parts = { string.rep(' ', start_indent) } --- @type string[] + local count = indent + + for i, w in ipairs(words) do + if count > indent and count + #w > text_width - 1 then + parts[#parts + 1] = '\n' + parts[#parts + 1] = string.rep(' ', indent) + count = indent + elseif i ~= 1 then + parts[#parts + 1] = ' ' + count = count + 1 + end + count = count + #w + parts[#parts + 1] = w + end + + return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', '')) +end + +--- @param node nvim.util.MDNode +--- @param start_indent integer +--- @param indent integer +--- @param text_width integer +--- @param level integer +--- @return string[] +local function render_md(node, start_indent, indent, text_width, level, is_list) + local parts = {} --- @type string[] + + -- For debugging + local add_tag = false + -- local add_tag = true + + local ntype = node.type + + if add_tag then + parts[#parts + 1] = '<' .. ntype .. '>' + end + + if ntype == 'text' then + parts[#parts + 1] = node.text + elseif ntype == 'html_tag' then + error('html_tag: ' .. node.text) + elseif ntype == 'inline_link' then + vim.list_extend(parts, { '*', node[1].text, '*' }) + elseif ntype == 'shortcut_link' then + if node[1].text:find('^<.*>$') then + parts[#parts + 1] = node[1].text + elseif node[1].text:find('^%d+$') then + vim.list_extend(parts, { '[', node[1].text, ']' }) + else + vim.list_extend(parts, { '|', node[1].text, '|' }) + end + elseif ntype == 'backslash_escape' then + parts[#parts + 1] = node.text + elseif ntype == 'emphasis' then + parts[#parts + 1] = node.text:sub(2, -2) + elseif ntype == 'code_span' then + vim.list_extend(parts, { '`', node.text:sub(2, -2):gsub(' ', NBSP), '`' }) + elseif ntype == 'inline' then + if #node == 0 then + local text = assert(node.text) + parts[#parts + 1] = M.wrap(text, start_indent, indent, text_width) + else + for _, child in ipairs(node) do + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1)) + end + end + elseif ntype == 'paragraph' then + local pparts = {} + for _, child in ipairs(node) do + vim.list_extend(pparts, render_md(child, start_indent, indent, text_width, level + 1)) + end + parts[#parts + 1] = M.wrap(table.concat(pparts), start_indent, indent, text_width) + parts[#parts + 1] = '\n' + elseif ntype == 'code_fence_content' then + local lines = vim.split(node.text:gsub('\n%s*$', ''), '\n') + + local cindent = indent + INDENTATION + if level > 3 then + -- The tree-sitter markdown parser doesn't parse the code blocks indents + -- correctly in lists. Fudge it! + lines[1] = ' ' .. lines[1] -- ¯\_(ツ)_/¯ + cindent = indent - level + local _, initial_indent = lines[1]:find('^%s*') + initial_indent = initial_indent + cindent + if initial_indent < indent then + cindent = indent - INDENTATION + end + end + + for _, l in ipairs(lines) do + if #l > 0 then + parts[#parts + 1] = string.rep(' ', cindent) + parts[#parts + 1] = l + end + parts[#parts + 1] = '\n' + end + elseif ntype == 'fenced_code_block' then + parts[#parts + 1] = '>' + for _, child in ipairs(node) do + if child.type == 'info_string' then + parts[#parts + 1] = child.text + break + end + end + parts[#parts + 1] = '\n' + for _, child in ipairs(node) do + if child.type ~= 'info_string' then + vim.list_extend(parts, render_md(child, start_indent, indent, text_width, level + 1)) + end + end + parts[#parts + 1] = '<\n' + elseif ntype == 'html_block' then + local text = node.text:gsub('^<pre>help', '') + text = text:gsub('</pre>%s*$', '') + parts[#parts + 1] = text + elseif ntype == 'list_marker_dot' then + parts[#parts + 1] = node.text + elseif contains(ntype, { 'list_marker_minus', 'list_marker_star' }) then + parts[#parts + 1] = '• ' + elseif ntype == 'list_item' then + parts[#parts + 1] = string.rep(' ', indent) + local offset = node[1].type == 'list_marker_dot' and 3 or 2 + for i, child in ipairs(node) do + local sindent = i <= 2 and 0 or (indent + offset) + vim.list_extend( + parts, + render_md(child, sindent, indent + offset, text_width, level + 1, true) + ) + end + else + if node.text then + error(fmt('cannot render:\n%s', vim.inspect(node))) + end + for i, child in ipairs(node) do + local start_indent0 = i == 1 and start_indent or indent + vim.list_extend( + parts, + render_md(child, start_indent0, indent, text_width, level + 1, is_list) + ) + if ntype ~= 'list' and i ~= #node then + if (node[i + 1] or {}).type ~= 'list' then + parts[#parts + 1] = '\n' + end + end + end + end + + if add_tag then + parts[#parts + 1] = '</' .. ntype .. '>' + end + + return parts +end + +--- @param text_width integer +local function align_tags(text_width) + --- @param line string + --- @return string + return function(line) + local tag_pat = '%s*(%*.+%*)%s*$' + local tags = {} + for m in line:gmatch(tag_pat) do + table.insert(tags, m) + end + + if #tags > 0 then + line = line:gsub(tag_pat, '') + local tags_str = ' ' .. table.concat(tags, ' ') + --- @type integer + local conceal_offset = select(2, tags_str:gsub('%*', '')) - 2 + local pad = string.rep(' ', text_width - #line - #tags_str + conceal_offset) + return line .. pad .. tags_str + end + + return line + end +end + +--- @param text string +--- @param start_indent integer +--- @param indent integer +--- @param is_list? boolean +--- @return string +function M.md_to_vimdoc(text, start_indent, indent, text_width, is_list) + -- Add an extra newline so the parser can properly capture ending ``` + local parsed = parse_md(text .. '\n') + local ret = render_md(parsed, start_indent, indent, text_width, 0, is_list) + + local lines = vim.split(table.concat(ret):gsub(NBSP, ' '), '\n') + + lines = vim.tbl_map(align_tags(text_width), lines) + + local s = table.concat(lines, '\n') + + -- Reduce whitespace in code-blocks + s = s:gsub('\n+%s*>([a-z]+)\n', ' >%1\n') + s = s:gsub('\n+%s*>\n?\n', ' >\n') + + return s +end + +return M diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index 111a6567ec..8112045d11 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -292,7 +292,7 @@ set(UI_METADATA ${PROJECT_BINARY_DIR}/ui_metadata.mpack) set(BINARY_LIB_DIR ${PROJECT_BINARY_DIR}/lib/nvim) set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto) set(GENERATED_INCLUDES_DIR ${PROJECT_BINARY_DIR}/include) -set(GENERATOR_DIR ${CMAKE_CURRENT_LIST_DIR}/generators) +set(GENERATOR_DIR ${PROJECT_SOURCE_DIR}/src/gen) set(GEN_EVAL_TOUCH ${TOUCHES_DIR}/gen_doc_eval) set(LUAJIT_RUNTIME_DIR ${DEPS_PREFIX}/share/luajit-2.1/jit) set(NVIM_RUNTIME_DIR ${PROJECT_SOURCE_DIR}/runtime) @@ -306,7 +306,8 @@ set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua) set(FUNCS_GENERATOR ${GENERATOR_DIR}/gen_eval.lua) set(GENERATOR_C_GRAMMAR ${GENERATOR_DIR}/c_grammar.lua) set(GENERATOR_HASHY ${GENERATOR_DIR}/hashy.lua) -set(GENERATOR_PRELOAD ${GENERATOR_DIR}/preload.lua) +set(GENERATOR_PRELOAD ${GENERATOR_DIR}/preload_nlua.lua) +set(NVIM_LUA_PRELOAD ${GENERATOR_DIR}/preload.lua) set(HEADER_GENERATOR ${GENERATOR_DIR}/gen_declarations.lua) set(OPTIONS_GENERATOR ${GENERATOR_DIR}/gen_options.lua) @@ -514,6 +515,9 @@ add_custom_command( set(LUA_GEN ${LUA_GEN_PRG} ${GENERATOR_PRELOAD} ${PROJECT_SOURCE_DIR} $<TARGET_FILE:nlua0> ${PROJECT_BINARY_DIR}) set(LUA_GEN_DEPS ${GENERATOR_PRELOAD} $<TARGET_FILE:nlua0>) +# Like LUA_GEN but includes also vim.fn, vim.api, vim.uv, etc +set(NVIM_LUA $<TARGET_FILE:nvim_bin> -u NONE -l ${NVIM_LUA_PRELOAD} ${PROJECT_SOURCE_DIR}) + # NVIM_GENERATED_FOR_HEADERS: generated headers to be included in headers # NVIM_GENERATED_FOR_SOURCES: generated headers to be included in sources # These lists must be mutually exclusive. @@ -937,12 +941,12 @@ file(GLOB LUA_SOURCES CONFIGURE_DEPENDS ) add_target(doc-vim - COMMAND $<TARGET_FILE:nvim_bin> -u NONE -l scripts/gen_vimdoc.lua + COMMAND ${NVIM_LUA} src/gen/gen_vimdoc.lua DEPENDS nvim ${API_SOURCES} ${LUA_SOURCES} - ${PROJECT_SOURCE_DIR}/scripts/gen_vimdoc.lua + ${PROJECT_SOURCE_DIR}/src/gen/gen_vimdoc.lua ${NVIM_RUNTIME_DIR}/doc/api.txt ${NVIM_RUNTIME_DIR}/doc/diagnostic.txt ${NVIM_RUNTIME_DIR}/doc/lsp.txt @@ -951,11 +955,11 @@ add_target(doc-vim ) add_target(doc-eval - COMMAND $<TARGET_FILE:nvim_bin> -u NONE -l ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua + COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua DEPENDS nvim ${FUNCS_METADATA} - ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua + ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${PROJECT_SOURCE_DIR}/src/nvim/eval.lua ${PROJECT_SOURCE_DIR}/src/nvim/options.lua ${PROJECT_SOURCE_DIR}/src/nvim/vvars.lua @@ -966,7 +970,7 @@ add_custom_target(doc) add_dependencies(doc doc-vim doc-eval) add_target(lintdoc - COMMAND $<TARGET_FILE:nvim_bin> -u NONE -l scripts/lintdoc.lua + COMMAND ${NVIM_LUA} scripts/lintdoc.lua DEPENDS ${DOCFILES} CUSTOM_COMMAND_ARGS USES_TERMINAL) add_dependencies(lintdoc nvim) diff --git a/src/nvim/func_attr.h b/src/nvim/func_attr.h index 43af880767..e19a0acd5d 100644 --- a/src/nvim/func_attr.h +++ b/src/nvim/func_attr.h @@ -1,6 +1,6 @@ // Undefined DEFINE_FUNC_ATTRIBUTES and undefined DEFINE_EMPTY_ATTRIBUTES // leaves file with untouched FUNC_ATTR_* macros. This variant is used for -// scripts/gen_declarations.lua. +// src/gen/gen_declarations.lua. // // Empty macros are used for *.c files. // (undefined DEFINE_FUNC_ATTRIBUTES and defined DEFINE_EMPTY_ATTRIBUTES) |