Merge remote-tracking branch 'upstream/master' into mix_20240309

10 files changed, 443 insertions, 227 deletions

diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua
index c5294893e0..e332ef475f 100755
--- a/scripts/bump_deps.lua
+++ b/scripts/bump_deps.lua
@@ -81,6 +81,14 @@ local function get_dependency(dependency_name)
       repo = 'luvit/luv',
       symbol = 'LUV',
     },
+    ['unibilium'] = {
+      repo = 'neovim/unibilium',
+      symbol = 'UNIBILIUM',
+    },
+    ['utf8proc'] = {
+      repo = 'JuliaStrings/utf8proc',
+      symbol = 'UTF8PROC',
+    },
     ['tree-sitter'] = {
       repo = 'tree-sitter/tree-sitter',
       symbol = 'TREESITTER',
@@ -90,11 +98,11 @@ local function get_dependency(dependency_name)
       symbol = 'TREESITTER_C',
     },
     ['tree-sitter-lua'] = {
-      repo = 'MunifTanjim/tree-sitter-lua',
+      repo = 'tree-sitter-grammars/tree-sitter-lua',
       symbol = 'TREESITTER_LUA',
     },
     ['tree-sitter-vim'] = {
-      repo = 'neovim/tree-sitter-vim',
+      repo = 'tree-sitter-grammars/tree-sitter-vim',
       symbol = 'TREESITTER_VIM',
     },
     ['tree-sitter-vimdoc'] = {
@@ -102,9 +110,21 @@ local function get_dependency(dependency_name)
       symbol = 'TREESITTER_VIMDOC',
     },
     ['tree-sitter-query'] = {
-      repo = 'nvim-treesitter/tree-sitter-query',
+      repo = 'tree-sitter-grammars/tree-sitter-query',
       symbol = 'TREESITTER_QUERY',
     },
+    ['tree-sitter-markdown'] = {
+      repo = 'tree-sitter-grammars/tree-sitter-markdown',
+      symbol = 'TREESITTER_MARKDOWN',
+    },
+    ['wasmtime'] = {
+      repo = 'bytecodealliance/wasmtime',
+      symbol = 'WASMTIME',
+    },
+    ['uncrustify'] = {
+      repo = 'uncrustify/uncrustify',
+      symbol = 'UNCRUSTIFY',
+    },
   }
   local dependency = dependency_table[dependency_name]
   if dependency == nil then
diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua
index f1bba5c0a2..b9ea4e73f0 100755
--- a/scripts/gen_eval_files.lua
+++ b/scripts/gen_eval_files.lua
@@ -2,11 +2,15 @@
 
 -- Generator for various vimdoc and Lua type files
 
+local util = require('scripts.util')
+local fmt = string.format
+
 local DEP_API_METADATA = 'build/funcs_metadata.mpack'
+local TEXT_WIDTH = 78
 
 --- @class vim.api.metadata
 --- @field name string
---- @field parameters {[1]:string,[2]:string}[]
+--- @field parameters [string,string][]
 --- @field return_type string
 --- @field deprecated_since integer
 --- @field eval boolean
@@ -20,7 +24,7 @@ local DEP_API_METADATA = 'build/funcs_metadata.mpack'
 
 local LUA_API_RETURN_OVERRIDES = {
   nvim_buf_get_command = 'table<string,vim.api.keyset.command_info>',
-  nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item',
+  nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item_by_id',
   nvim_buf_get_extmarks = 'vim.api.keyset.get_extmark_item[]',
   nvim_buf_get_keymap = 'vim.api.keyset.keymap[]',
   nvim_get_autocmds = 'vim.api.keyset.get_autocmds.ret[]',
@@ -29,11 +33,11 @@ local LUA_API_RETURN_OVERRIDES = {
   nvim_get_keymap = 'vim.api.keyset.keymap[]',
   nvim_get_mark = 'vim.api.keyset.get_mark',
 
-  -- Can also return table<string,vim.api.keyset.hl_info>, however we need to
+  -- Can also return table<string,vim.api.keyset.get_hl_info>, however we need to
   -- pick one to get some benefit.
   -- REVISIT lewrus01 (26/01/24): we can maybe add
-  -- @overload fun(ns: integer, {}): table<string,vim.api.keyset.hl_info>
-  nvim_get_hl = 'vim.api.keyset.hl_info',
+  -- @overload fun(ns: integer, {}): table<string,vim.api.keyset.get_hl_info>
+  nvim_get_hl = 'vim.api.keyset.get_hl_info',
 
   nvim_get_mode = 'vim.api.keyset.get_mode',
   nvim_get_namespaces = 'table<string,integer>',
@@ -112,7 +116,7 @@ local API_TYPES = {
   String = 'string',
   Array = 'any[]',
   LuaRef = 'function',
-  Dictionary = 'table<string,any>',
+  Dict = 'table<string,any>',
   Float = 'number',
   HLGroupID = 'number|string',
   void = '',
@@ -140,7 +144,7 @@ local function api_type(t)
     return 'vim.api.keyset.' .. d
   end
 
-  local d0 = t:match('^DictionaryOf%((.*)%)')
+  local d0 = t:match('^DictOf%((.*)%)')
   if d0 then
     return 'table<string,' .. api_type(d0) .. '>'
   end
@@ -149,7 +153,7 @@ local function api_type(t)
 end
 
 --- @param f string
---- @param params {[1]:string,[2]:string}[]|true
+--- @param params [string,string][]|true
 --- @return string
 local function render_fun_sig(f, params)
   local param_str --- @type string
@@ -158,7 +162,7 @@ local function render_fun_sig(f, params)
   else
     param_str = table.concat(
       vim.tbl_map(
-        --- @param v {[1]:string,[2]:string}
+        --- @param v [string,string]
         --- @return string
         function(v)
           return v[1]
@@ -170,16 +174,16 @@ local function render_fun_sig(f, params)
   end
 
   if LUA_KEYWORDS[f] then
-    return string.format("vim.fn['%s'] = function(%s) end", f, param_str)
+    return fmt("vim.fn['%s'] = function(%s) end", f, param_str)
   else
-    return string.format('function vim.fn.%s(%s) end', f, param_str)
+    return fmt('function vim.fn.%s(%s) end', f, param_str)
   end
 end
 
 --- Uniquify names
 --- Fix any names that are lua keywords
---- @param params {[1]:string,[2]:string,[3]:string}[]
---- @return {[1]:string,[2]:string,[3]:string}[]
+--- @param params [string,string,string][]
+--- @return [string,string,string][]
 local function process_params(params)
   local seen = {} --- @type table<string,true>
   local sfx = 1
@@ -199,14 +203,6 @@ local function process_params(params)
   return params
 end
 
---- @class vim.gen_vim_doc_fun
---- @field signature string
---- @field doc string[]
---- @field parameters_doc table<string,string>
---- @field return string[]
---- @field seealso string[]
---- @field annotations string[]
-
 --- @return table<string, vim.EvalFn>
 local function get_api_meta()
   local ret = {} --- @type table<string, vim.EvalFn>
@@ -245,7 +241,17 @@ local function get_api_meta()
   for _, fun in pairs(functions) do
     local deprecated = fun.deprecated_since ~= nil
 
-    local params = {} --- @type {[1]:string,[2]:string}[]
+    local notes = {} --- @type string[]
+    for _, note in ipairs(fun.notes or {}) do
+      notes[#notes + 1] = note.desc
+    end
+
+    local sees = {} --- @type string[]
+    for _, see in ipairs(fun.see or {}) do
+      sees[#sees + 1] = see.desc
+    end
+
+    local params = {} --- @type [string,string][]
     for _, p in ipairs(fun.params) do
       params[#params + 1] = {
         p.name,
@@ -258,13 +264,15 @@ local function get_api_meta()
       signature = 'NA',
       name = fun.name,
       params = params,
+      notes = notes,
+      see = sees,
       returns = api_type(fun.returns[1].type),
       deprecated = deprecated,
     }
 
     if not deprecated then
       r.desc = fun.desc
-      r.return_desc = fun.returns[1].desc
+      r.returns_desc = fun.returns[1].desc
     end
 
     ret[fun.name] = r
@@ -278,8 +286,19 @@ end
 --- Ensure code blocks have one empty line before the start fence and after the closing fence.
 ---
 --- @param x string
+--- @param special string?
+---                | 'see-api-meta' Normalize `@see` for API meta docstrings.
 --- @return string
-local function norm_text(x)
+local function norm_text(x, special)
+  if special == 'see-api-meta' then
+    -- Try to guess a symbol that actually works in @see.
+    -- "nvim_xx()" => "vim.api.nvim_xx"
+    x = x:gsub([=[%|?(nvim_[^.()| ]+)%(?%)?%|?]=], 'vim.api.%1')
+    -- TODO: Remove backticks when LuaLS resolves: https://github.com/LuaLS/lua-language-server/issues/2889
+    -- "|foo|" => "`:help foo`"
+    x = x:gsub([=[|([^ ]+)|]=], '`:help %1`')
+  end
+
   return (
     x:gsub('|([^ ]+)|', '`%1`')
       :gsub('\n*>lua', '\n\n```lua')
@@ -291,14 +310,13 @@ local function norm_text(x)
   )
 end
 
+--- Generates LuaLS docstring for an API function.
 --- @param _f string
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
 local function render_api_meta(_f, fun, write)
   write('')
 
-  local text_utils = require('scripts.text_utils')
-
   if vim.startswith(fun.name, 'nvim__') then
     write('--- @private')
   end
@@ -309,10 +327,18 @@ local function render_api_meta(_f, fun, write)
 
   local desc = fun.desc
   if desc then
-    desc = text_utils.md_to_vimdoc(desc, 0, 0, 74)
-    for _, l in ipairs(split(norm_text(desc))) do
-      write('--- ' .. l)
-    end
+    write(util.prefix_lines('--- ', norm_text(desc)))
+  end
+
+  -- LuaLS doesn't support @note. Render @note items as a markdown list.
+  if fun.notes and #fun.notes > 0 then
+    write('--- Note:')
+    write(util.prefix_lines('--- ', table.concat(fun.notes, '\n')))
+    write('---')
+  end
+
+  for _, see in ipairs(fun.see or {}) do
+    write(util.prefix_lines('--- @see ', norm_text(see, 'see-api-meta')))
   end
 
   local param_names = {} --- @type string[]
@@ -322,8 +348,6 @@ local function render_api_meta(_f, fun, write)
     local pdesc = p[3]
     if pdesc then
       local s = '--- @param ' .. p[1] .. ' ' .. p[2] .. ' '
-      local indent = #('@param ' .. p[1] .. ' ')
-      pdesc = text_utils.md_to_vimdoc(pdesc, #s, indent, 74, true)
       local pdesc_a = split(vim.trim(norm_text(pdesc)))
       write(s .. pdesc_a[1])
       for i = 2, #pdesc_a do
@@ -336,15 +360,15 @@ local function render_api_meta(_f, fun, write)
       write('--- @param ' .. p[1] .. ' ' .. p[2])
     end
   end
+
   if fun.returns ~= '' then
-    local ret_desc = fun.returns_desc and ' : ' .. fun.returns_desc or ''
-    ret_desc = text_utils.md_to_vimdoc(ret_desc, 0, 0, 74)
+    local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or ''
     local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns
-    write('--- @return ' .. ret .. ret_desc)
+    write(util.prefix_lines('--- ', '@return ' .. ret .. ret_desc))
   end
   local param_str = table.concat(param_names, ', ')
 
-  write(string.format('function vim.api.%s(%s) end', fun.name, param_str))
+  write(fmt('function vim.api.%s(%s) end', fun.name, param_str))
 end
 
 --- @return table<string, vim.EvalFn>
@@ -372,10 +396,14 @@ local function get_api_keysets_meta()
   return ret
 end
 
+--- Generates LuaLS docstring for an API keyset.
 --- @param _f string
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
 local function render_api_keyset_meta(_f, fun, write)
+  if string.sub(fun.name, 1, 1) == '_' then
+    return -- not exported
+  end
   write('')
   write('--- @class vim.api.keyset.' .. fun.name)
   for _, p in ipairs(fun.params) do
@@ -388,6 +416,7 @@ local function get_eval_meta()
   return require('src/nvim/eval').funcs
 end
 
+--- Generates LuaLS docstring for a Vimscript "eval" function.
 --- @param f string
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
@@ -397,7 +426,6 @@ local function render_eval_meta(f, fun, write)
   end
 
   local funname = fun.name or f
-
   local params = process_params(fun.params)
 
   write('')
@@ -421,25 +449,28 @@ local function render_eval_meta(f, fun, write)
   for i, param in ipairs(params) do
     local pname, ptype = param[1], param[2]
     local optional = (pname ~= '...' and i > req_args) and '?' or ''
-    write(string.format('--- @param %s%s %s', pname, optional, ptype))
+    write(fmt('--- @param %s%s %s', pname, optional, ptype))
   end
 
   if fun.returns ~= false then
-    write('--- @return ' .. (fun.returns or 'any'))
+    local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or ''
+    write('--- @return ' .. (fun.returns or 'any') .. ret_desc)
   end
 
   write(render_fun_sig(funname, params))
 end
 
+--- Generates vimdoc heading for a Vimscript "eval" function signature.
 --- @param name string
+--- @param name_tag boolean
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
-local function render_sig_and_tag(name, fun, write)
+local function render_sig_and_tag(name, name_tag, fun, write)
   if not fun.signature then
     return
   end
 
-  local tags = { '*' .. name .. '()*' }
+  local tags = name_tag and { '*' .. name .. '()*' } or {}
 
   if fun.tags then
     for _, t in ipairs(fun.tags) do
@@ -447,6 +478,11 @@ local function render_sig_and_tag(name, fun, write)
     end
   end
 
+  if #tags == 0 then
+    write(fun.signature)
+    return
+  end
+
   local tag = table.concat(tags, ' ')
   local siglen = #fun.signature
   local conceal_offset = 2 * (#tags - 1)
@@ -456,32 +492,28 @@ local function render_sig_and_tag(name, fun, write)
     write(string.rep(' ', tag_pad_len) .. tag)
     write(fun.signature)
   else
-    write(string.format('%s%s%s', fun.signature, string.rep(' ', tag_pad_len - siglen), tag))
+    write(fmt('%s%s%s', fun.signature, string.rep(' ', tag_pad_len - siglen), tag))
   end
 end
 
+--- Generates vimdoc for a Vimscript "eval" function.
 --- @param f string
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
 local function render_eval_doc(f, fun, write)
-  if fun.deprecated then
-    return
-  end
-
-  if not fun.signature then
+  if fun.deprecated or not fun.signature then
     return
   end
 
-  if f:find('__%d+$') then
-    write(fun.signature)
-  else
-    render_sig_and_tag(fun.name or f, fun, write)
-  end
+  render_sig_and_tag(fun.name or f, not f:find('__%d+$'), fun, write)
 
   if not fun.desc then
     return
   end
 
+  local params = process_params(fun.params)
+  local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0
+
   local desc_l = split(vim.trim(fun.desc))
   for _, l in ipairs(desc_l) do
     l = l:gsub('^      ', '')
@@ -497,6 +529,26 @@ local function render_eval_doc(f, fun, write)
   if #desc_l > 0 and not desc_l[#desc_l]:match('^<?$') then
     write('')
   end
+
+  if #params > 0 then
+    write(util.md_to_vimdoc('Parameters: ~', 16, 16, TEXT_WIDTH))
+    for i, param in ipairs(params) do
+      local pname, ptype = param[1], param[2]
+      local optional = (pname ~= '...' and i > req_args) and '?' or ''
+      local s = fmt('- %-14s (`%s%s`)', fmt('{%s}', pname), ptype, optional)
+      write(util.md_to_vimdoc(s, 16, 18, TEXT_WIDTH))
+    end
+    write('')
+  end
+
+  if fun.returns ~= false then
+    write(util.md_to_vimdoc('Return: ~', 16, 16, TEXT_WIDTH))
+    local ret = ('(`%s`)'):format((fun.returns or 'any'))
+    ret = ret .. (fun.returns_desc and ' ' .. fun.returns_desc or '')
+    ret = util.md_to_vimdoc(ret, 18, 18, TEXT_WIDTH)
+    write(ret)
+    write('')
+  end
 end
 
 --- @param d vim.option_defaults
@@ -734,9 +786,9 @@ local function render_option_doc(_f, opt, write)
 
   local name_str --- @type string
   if opt.abbreviation then
-    name_str = string.format("'%s' '%s'", opt.full_name, opt.abbreviation)
+    name_str = fmt("'%s' '%s'", opt.full_name, opt.abbreviation)
   else
-    name_str = string.format("'%s'", opt.full_name)
+    name_str = fmt("'%s'", opt.full_name)
   end
 
   local otype = opt.type == 'boolean' and 'boolean' or opt.type
@@ -744,13 +796,13 @@ local function render_option_doc(_f, opt, write)
     local v = render_option_default(opt.defaults, true)
     local pad = string.rep('\t', math.max(1, math.ceil((24 - #name_str) / 8)))
     if opt.defaults.doc then
-      local deflen = #string.format('%s%s%s (', name_str, pad, otype)
+      local deflen = #fmt('%s%s%s (', name_str, pad, otype)
       --- @type string
       v = v:gsub('\n', '\n' .. string.rep(' ', deflen - 2))
     end
-    write(string.format('%s%s%s\t(default %s)', name_str, pad, otype, v))
+    write(fmt('%s%s%s\t(default %s)', name_str, pad, otype, v))
   else
-    write(string.format('%s\t%s', name_str, otype))
+    write(fmt('%s\t%s', name_str, otype))
   end
 
   write('\t\t\t' .. scope_to_doc(opt.scope) .. scope_more_doc(opt))
diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua
index cdfb85bde6..f6e799508b 100644
--- a/scripts/gen_help_html.lua
+++ b/scripts/gen_help_html.lua
@@ -1,6 +1,4 @@
--- Converts Vim :help files to HTML.  Validates |tag| links and document syntax (parser errors).
---
--- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags).
+--- Converts Nvim :help files to HTML.  Validates |tag| links and document syntax (parser errors).
 --
 -- USAGE (For CI/local testing purposes): Simply `make lintdoc` or `scripts/lintdoc.lua`, which
 -- basically does the following:
@@ -23,6 +21,8 @@
 --   1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +q
 --
 -- NOTES:
+--   * This script is used by the automation repo: https://github.com/neovim/doc
+--   * :helptags checks for duplicate tags, whereas this script checks _links_ (to tags).
 --   * gen() and validate() are the primary (programmatic) entrypoints. validate() only exists
 --     because gen() is too slow (~1 min) to run in per-commit CI.
 --   * visit_node() is the core function used by gen() to traverse the document tree and produce HTML.
@@ -68,6 +68,7 @@ local new_layout = {
   ['dev_theme.txt'] = true,
   ['dev_tools.txt'] = true,
   ['dev_vimpatch.txt'] = true,
+  ['editorconfig.txt'] = true,
   ['faq.txt'] = true,
   ['lua.txt'] = true,
   ['luaref.txt'] = true,
@@ -76,10 +77,17 @@ local new_layout = {
   ['news-0.10.txt'] = true,
   ['nvim.txt'] = true,
   ['provider.txt'] = true,
+  ['tui.txt'] = true,
   ['ui.txt'] = true,
   ['vim_diff.txt'] = true,
 }
 
+-- Map of new:old pages, to redirect renamed pages.
+local redirects = {
+  ['tui'] = 'term',
+  ['terminal'] = 'nvim_terminal_emulator',
+}
+
 -- TODO: These known invalid |links| require an update to the relevant docs.
 local exclude_invalid = {
   ["'string'"] = 'eval.txt',
@@ -212,6 +220,7 @@ local function is_noise(line, noise_lines)
 end
 
 --- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content.
+--- @return string
 local function get_bug_url_vimdoc(fname, to_fname, sample_text)
   local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname))
   local bug_url = (
@@ -227,6 +236,7 @@ local function get_bug_url_vimdoc(fname, to_fname, sample_text)
 end
 
 --- Creates a github issue URL at neovim/neovim with prefilled content.
+--- @return string
 local function get_bug_url_nvim(fname, to_fname, sample_text, token_name)
   local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname))
   local bug_url = (
@@ -255,7 +265,7 @@ local function get_helppage(f)
     return 'index.html'
   end
 
-  return (f:gsub('%.txt$', '.html'))
+  return (f:gsub('%.txt$', '')) .. '.html'
 end
 
 --- Counts leading spaces (tab=8) to decide the indent size of multiline text.
@@ -490,7 +500,6 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     or nil
   -- Parent kind (string).
   local parent = root:parent() and root:parent():type() or nil
-  local text = ''
   -- Gets leading whitespace of `node`.
   local function ws(node)
     node = node or root
@@ -508,6 +517,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf))
   end
 
+  local text = ''
   local trimmed ---@type string
   if root:named_child_count() == 0 or node_name == 'ERROR' then
     text = node_text()
@@ -537,12 +547,17 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     if is_noise(text, stats.noise_lines) then
       return '' -- Discard common "noise" lines.
     end
-    -- Remove "===" and tags from ToC text.
-    local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', ''))
+    -- Remove tags from ToC text.
+    local heading_node = first(root, 'heading')
+    local hname = trim(node_text(heading_node):gsub('%*.*%*', ''))
+    if not heading_node or hname == '' then
+      return '' -- Spurious "===" or "---" in the help doc.
+    end
+
     -- Use the first *tag* node as the heading anchor, if any.
-    local tagnode = first(root, 'tag')
+    local tagnode = first(heading_node, 'tag')
     -- Use the *tag* as the heading anchor id, if possible.
-    local tagname = tagnode and url_encode(node_text(tagnode:child(1), false))
+    local tagname = tagnode and url_encode(trim(node_text(tagnode:child(1), false)))
       or to_heading_tag(hname)
     if node_name == 'h1' or #headings == 0 then
       ---@type nvim.gen_help_html.heading
@@ -555,7 +570,9 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       )
     end
     local el = node_name == 'h1' and 'h2' or 'h3'
-    return ('<%s id="%s" class="help-heading">%s</%s>\n'):format(el, tagname, text, el)
+    return ('<%s id="%s" class="help-heading">%s</%s>\n'):format(el, tagname, trimmed, el)
+  elseif node_name == 'heading' then
+    return trimmed
   elseif node_name == 'column_heading' or node_name == 'column_name' then
     if root:has_error() then
       return text
@@ -648,13 +665,13 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       code = ('<pre>%s</pre>'):format(trim(trim_indent(text), 2))
     end
     return code
-  elseif node_name == 'tag' then -- anchor
+  elseif node_name == 'tag' then -- anchor, h4 pseudo-heading
     if root:has_error() then
       return text
     end
     local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent)
-    local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right'
-      or 'help-tag'
+    local h4 = not in_heading and not next_ and get_indent(node_text()) > 8 -- h4 pseudo-heading
+    local cssclass = h4 and 'help-tag-right' or 'help-tag'
     local tagname = node_text(root:child(1), false)
     if vim.tbl_count(stats.first_tags) < 2 then
       -- Force the first 2 tags in the doc to be anchored at the main heading.
@@ -694,8 +711,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       -- End the <span> container for tags in a heading.
       return string.format('%s</span>', s)
     end
-    return s
-  elseif node_name == 'modeline' then
+    return s .. (h4 and '<br>' or '') -- HACK: <br> avoids h4 pseudo-heading mushing with text.
+  elseif node_name == 'delimiter' or node_name == 'modeline' then
     return ''
   elseif node_name == 'ERROR' then
     if ignore_parse_error(opt.fname, trimmed) then
@@ -760,14 +777,21 @@ local function ensure_runtimepath()
   end
 end
 
---- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents.
+--- Opens `fname` (or `text`, if given) in a buffer and gets a treesitter parser for the buffer contents.
 ---
---- @param fname string help file to parse
+--- @param fname string :help file to parse
+--- @param text string? :help file contents
 --- @param parser_path string? path to non-default vimdoc.so
 --- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr)
-local function parse_buf(fname, parser_path)
+local function parse_buf(fname, text, parser_path)
   local buf ---@type integer
-  if type(fname) == 'string' then
+  if text then
+    vim.cmd('split new') -- Text contents.
+    vim.api.nvim_put(vim.split(text, '\n'), '', false, false)
+    vim.cmd('setfiletype help')
+    -- vim.treesitter.language.add('vimdoc')
+    buf = vim.api.nvim_get_current_buf()
+  elseif type(fname) == 'string' then
     vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename.
     buf = vim.api.nvim_get_current_buf()
   else
@@ -779,7 +803,7 @@ local function parse_buf(fname, parser_path)
   if parser_path then
     vim.treesitter.language.add('vimdoc', { path = parser_path })
   end
-  local lang_tree = vim.treesitter.get_parser(buf)
+  local lang_tree = assert(vim.treesitter.get_parser(buf, nil, { error = false }))
   return lang_tree, buf
 end
 
@@ -794,7 +818,7 @@ local function validate_one(fname, parser_path)
   local stats = {
     parse_errors = {},
   }
-  local lang_tree, buf = parse_buf(fname, parser_path)
+  local lang_tree, buf = parse_buf(fname, nil, parser_path)
   for _, tree in ipairs(lang_tree:trees()) do
     visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats)
   end
@@ -805,20 +829,21 @@ end
 
 --- Generates HTML from one :help file `fname` and writes the result to `to_fname`.
 ---
---- @param fname string Source :help file
+--- @param fname string Source :help file.
+--- @param text string|nil Source :help file contents, or nil to read `fname`.
 --- @param to_fname string Destination .html file
 --- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace)
 --- @param parser_path string? path to non-default vimdoc.so
 ---
 --- @return string html
 --- @return table stats
-local function gen_one(fname, to_fname, old, commit, parser_path)
+local function gen_one(fname, text, to_fname, old, commit, parser_path)
   local stats = {
     noise_lines = {},
     parse_errors = {},
     first_tags = {}, -- Track the first few tags in doc.
   }
-  local lang_tree, buf = parse_buf(fname, parser_path)
+  local lang_tree, buf = parse_buf(fname, text, parser_path)
   ---@type nvim.gen_help_html.heading[]
   local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3.
   local title = to_titlecase(basename_noext(fname))
@@ -1126,6 +1151,7 @@ local function gen_css(fname)
       margin-left: auto;
       margin-right: 0;
       float: right;
+      display: block;
     }
     .help-tag a,
     .help-tag-right a {
@@ -1139,10 +1165,11 @@ local function gen_css(fname)
       font-size: smaller;
     }
     .help-heading {
-      overflow: hidden;
-      white-space: nowrap;
+      white-space: normal;
       display: flex;
+      flex-flow: row wrap;
       justify-content: space-between;
+      gap: 0 15px;
     }
     /* The (right-aligned) "tags" part of a section heading. */
     .help-heading-tags {
@@ -1177,8 +1204,7 @@ local function gen_css(fname)
     pre:last-child {
       margin-bottom: 0;
     }
-    pre:hover,
-    .help-heading:hover {
+    pre:hover {
       overflow: visible;
     }
     .generator-stats {
@@ -1216,7 +1242,7 @@ end
 
 function M._test()
   tagmap = get_helptags('$VIMRUNTIME/doc')
-  helpfiles = get_helpfiles(vim.fn.expand('$VIMRUNTIME/doc'))
+  helpfiles = get_helpfiles(vim.fs.normalize('$VIMRUNTIME/doc'))
 
   ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap))
   ok(
@@ -1278,7 +1304,7 @@ function M.gen(help_dir, to_dir, include, commit, parser_path)
     help_dir = {
       help_dir,
       function(d)
-        return vim.fn.isdirectory(vim.fn.expand(d)) == 1
+        return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
       end,
       'valid directory',
     },
@@ -1288,40 +1314,88 @@ function M.gen(help_dir, to_dir, include, commit, parser_path)
     parser_path = {
       parser_path,
       function(f)
-        return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1
+        return f == nil or vim.fn.filereadable(vim.fs.normalize(f)) == 1
       end,
       'valid vimdoc.{so,dll} filepath',
     },
   }
 
   local err_count = 0
+  local redirects_count = 0
   ensure_runtimepath()
-  tagmap = get_helptags(vim.fn.expand(help_dir))
+  tagmap = get_helptags(vim.fs.normalize(help_dir))
   helpfiles = get_helpfiles(help_dir, include)
-  to_dir = vim.fn.expand(to_dir)
-  parser_path = parser_path and vim.fn.expand(parser_path) or nil
+  to_dir = vim.fs.normalize(to_dir)
+  parser_path = parser_path and vim.fs.normalize(parser_path) or nil
 
-  print(('output dir: %s'):format(to_dir))
+  print(('output dir: %s\n\n'):format(to_dir))
   vim.fn.mkdir(to_dir, 'p')
   gen_css(('%s/help.css'):format(to_dir))
 
   for _, f in ipairs(helpfiles) do
+    -- "foo.txt"
     local helpfile = vim.fs.basename(f)
+    -- "to/dir/foo.html"
     local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile))
-    local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?', parser_path)
+    local html, stats =
+      gen_one(f, nil, to_fname, not new_layout[helpfile], commit or '?', parser_path)
     tofile(to_fname, html)
     print(
-      ('generated (%-4s errors): %-15s => %s'):format(
+      ('generated (%-2s errors): %-15s => %s'):format(
         #stats.parse_errors,
         helpfile,
         vim.fs.basename(to_fname)
       )
     )
+
+    -- Generate redirect pages for renamed help files.
+    local helpfile_tag = (helpfile:gsub('%.txt$', ''))
+    local redirect_from = redirects[helpfile_tag]
+    if redirect_from then
+      local redirect_text = ([[
+*%s*      Nvim
+
+This document moved to: |%s|
+
+==============================================================================
+This document moved to: |%s|
+
+This document moved to: |%s|
+
+==============================================================================
+ vim:tw=78:ts=8:ft=help:norl:
+      ]]):format(
+        redirect_from,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag
+      )
+      local redirect_to = ('%s/%s'):format(to_dir, get_helppage(redirect_from))
+      local redirect_html, _ =
+        gen_one(redirect_from, redirect_text, redirect_to, false, commit or '?', parser_path)
+      assert(redirect_html:find(helpfile_tag))
+      tofile(redirect_to, redirect_html)
+
+      print(
+        ('generated (redirect) : %-15s => %s'):format(
+          redirect_from .. '.txt',
+          vim.fs.basename(to_fname)
+        )
+      )
+      redirects_count = redirects_count + 1
+    end
+
     err_count = err_count + #stats.parse_errors
   end
-  print(('generated %d html pages'):format(#helpfiles))
+
+  print(('\ngenerated %d html pages'):format(#helpfiles + redirects_count))
   print(('total errors: %d'):format(err_count))
-  print(('invalid tags:\n%s'):format(vim.inspect(invalid_links)))
+  print(('invalid tags: %s'):format(vim.inspect(invalid_links)))
+  assert(#(include or {}) > 0 or redirects_count == vim.tbl_count(redirects)) -- sanity check
+  print(('redirects: %d'):format(redirects_count))
+  print('\n')
 
   --- @type nvim.gen_help_html.gen_result
   return {
@@ -1351,7 +1425,7 @@ function M.validate(help_dir, include, parser_path)
     help_dir = {
       help_dir,
       function(d)
-        return vim.fn.isdirectory(vim.fn.expand(d)) == 1
+        return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
       end,
       'valid directory',
     },
@@ -1359,7 +1433,7 @@ function M.validate(help_dir, include, parser_path)
     parser_path = {
       parser_path,
       function(f)
-        return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1
+        return f == nil or vim.fn.filereadable(vim.fs.normalize(f)) == 1
       end,
       'valid vimdoc.{so,dll} filepath',
     },
@@ -1367,12 +1441,12 @@ function M.validate(help_dir, include, parser_path)
   local err_count = 0 ---@type integer
   local files_to_errors = {} ---@type table<string, string[]>
   ensure_runtimepath()
-  tagmap = get_helptags(vim.fn.expand(help_dir))
+  tagmap = get_helptags(vim.fs.normalize(help_dir))
   helpfiles = get_helpfiles(help_dir, include)
-  parser_path = parser_path and vim.fn.expand(parser_path) or nil
+  parser_path = parser_path and vim.fs.normalize(parser_path) or nil
 
   for _, f in ipairs(helpfiles) do
-    local helpfile = assert(vim.fs.basename(f))
+    local helpfile = vim.fs.basename(f)
     local rv = validate_one(f, parser_path)
     print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile))
     if #rv.parse_errors > 0 then
@@ -1404,7 +1478,7 @@ end
 ---
 --- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
 function M.run_validate(help_dir)
-  help_dir = vim.fn.expand(help_dir or '$VIMRUNTIME/doc')
+  help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
   print('doc path = ' .. vim.uv.fs_realpath(help_dir))
 
   local rv = M.validate(help_dir)
@@ -1430,16 +1504,14 @@ end
 ---    :help files, we can be precise about the tolerances here.
 --- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
 function M.test_gen(help_dir)
-  local tmpdir = assert(vim.fs.dirname(vim.fn.tempname()))
-  help_dir = vim.fn.expand(help_dir or '$VIMRUNTIME/doc')
+  local tmpdir = vim.fs.dirname(vim.fn.tempname())
+  help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
   print('doc path = ' .. vim.uv.fs_realpath(help_dir))
 
-  local rv = M.gen(
-    help_dir,
-    tmpdir,
-    -- Because gen() is slow (~30s), this test is limited to a few files.
-    { 'help.txt', 'index.txt', 'nvim.txt' }
-  )
+  -- Because gen() is slow (~30s), this test is limited to a few files.
+  local input = { 'help.txt', 'index.txt', 'nvim.txt' }
+  local rv = M.gen(help_dir, tmpdir, input)
+  eq(#input, #rv.helpfiles)
   eq(0, rv.err_count, 'parse errors in :help docs')
   eq({}, rv.invalid_links, 'invalid tags in :help docs')
 end
diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua
index 04d19f22e6..c8dcf8c018 100644
--- a/scripts/gen_lsp.lua
+++ b/scripts/gen_lsp.lua
@@ -60,9 +60,10 @@ end
 local function gen_methods(protocol)
   local output = {
     '-- Generated by gen_lsp.lua, keep at end of file.',
-    '--- LSP method names.',
     '---',
+    '---@enum vim.lsp.protocol.Methods',
     '---@see https://microsoft.github.io/language-server-protocol/specification/#metaModel',
+    '--- LSP method names.',
     'protocol.Methods = {',
   }
   local indent = (' '):rep(2)
@@ -109,26 +110,8 @@ local function gen_methods(protocol)
     end
   end
   output[#output + 1] = '}'
-  output = vim.list_extend(
-    output,
-    vim.split(
-      [[
-local function freeze(t)
-  return setmetatable({}, {
-    __index = t,
-    __newindex = function()
-      error('cannot modify immutable table')
-    end,
-  })
-end
-protocol.Methods = freeze(protocol.Methods)
-
-return protocol
-]],
-      '\n',
-      { trimempty = true }
-    )
-  )
+  output[#output + 1] = ''
+  output[#output + 1] = 'return protocol'
 
   local fname = './runtime/lua/vim/lsp/protocol.lua'
   local bufnr = vim.fn.bufadd(fname)
@@ -297,13 +280,13 @@ function M.gen(opt)
 
     -- TupleType
     elseif type.kind == 'tuple' then
-      local tuple = '{ '
-      for i, value in ipairs(type.items) do
-        tuple = tuple .. '[' .. i .. ']: ' .. parse_type(value, prefix) .. ', '
+      local tuple = '['
+      for _, value in ipairs(type.items) do
+        tuple = tuple .. parse_type(value, prefix) .. ', '
       end
       -- remove , at the end
       tuple = tuple:sub(0, -3)
-      return tuple .. ' }'
+      return tuple .. ']'
     end
 
     vim.print('WARNING: Unknown type ', type)
diff --git a/scripts/gen_vimdoc.lua b/scripts/gen_vimdoc.lua
index 9c6225efc3..8908097397 100755
--- a/scripts/gen_vimdoc.lua
+++ b/scripts/gen_vimdoc.lua
@@ -18,12 +18,12 @@
 
 local luacats_parser = require('scripts.luacats_parser')
 local cdoc_parser = require('scripts.cdoc_parser')
-local text_utils = require('scripts.text_utils')
+local util = require('scripts.util')
 
 local fmt = string.format
 
-local wrap = text_utils.wrap
-local md_to_vimdoc = text_utils.md_to_vimdoc
+local wrap = util.wrap
+local md_to_vimdoc = util.md_to_vimdoc
 
 local TEXT_WIDTH = 78
 local INDENTATION = 4
@@ -50,7 +50,7 @@ local INDENTATION = 4
 --- For generated section names.
 --- @field section_fmt fun(name: string): string
 ---
---- @field helptag_fmt fun(name: string): string
+--- @field helptag_fmt fun(name: string): string|string[]
 ---
 --- Per-function helptag.
 --- @field fn_helptag_fmt? fun(fun: nvim.luacats.parser.fun): string
@@ -273,6 +273,7 @@ local config = {
       'buf.lua',
       'diagnostic.lua',
       'codelens.lua',
+      'completion.lua',
       'inlay_hint.lua',
       'tagfunc.lua',
       'semantic_tokens.lua',
@@ -318,6 +319,8 @@ local config = {
   treesitter = {
     filename = 'treesitter.txt',
     section_order = {
+      'tstree.lua',
+      'tsnode.lua',
       'treesitter.lua',
       'language.lua',
       'query.lua',
@@ -326,18 +329,27 @@ local config = {
       'dev.lua',
     },
     files = {
+      'runtime/lua/vim/treesitter/_meta/',
       'runtime/lua/vim/treesitter.lua',
       'runtime/lua/vim/treesitter/',
     },
     section_fmt = function(name)
       if name:lower() == 'treesitter' then
         return 'Lua module: vim.treesitter'
+      elseif name:lower() == 'tstree' then
+        return 'TREESITTER TREES'
+      elseif name:lower() == 'tsnode' then
+        return 'TREESITTER NODES'
       end
       return 'Lua module: vim.treesitter.' .. name:lower()
     end,
     helptag_fmt = function(name)
       if name:lower() == 'treesitter' then
         return 'lua-treesitter-core'
+      elseif name:lower() == 'tstree' then
+        return { 'treesitter-tree', 'TSTree' }
+      elseif name:lower() == 'tsnode' then
+        return { 'treesitter-node', 'TSNode' }
       end
       return 'lua-treesitter-' .. name:lower()
     end,
@@ -372,8 +384,8 @@ local config = {
     section_fmt = function(_name)
       return 'Checkhealth'
     end,
-    helptag_fmt = function(name)
-      return name:lower()
+    helptag_fmt = function()
+      return { 'vim.health', 'health' }
     end,
   },
 }
@@ -420,8 +432,11 @@ local function render_type(ty, generics, default)
 end
 
 --- @param p nvim.luacats.parser.param|nvim.luacats.parser.field
-local function should_render_param(p)
-  return not p.access and not contains(p.name, { '_', 'self' })
+local function should_render_field_or_param(p)
+  return not p.nodoc
+    and not p.access
+    and not contains(p.name, { '_', 'self' })
+    and not vim.startswith(p.name, '_')
 end
 
 --- @param desc? string
@@ -523,7 +538,7 @@ end
 local function render_fields_or_params(xs, generics, classes, exclude_types)
   local ret = {} --- @type string[]
 
-  xs = vim.tbl_filter(should_render_param, xs)
+  xs = vim.tbl_filter(should_render_field_or_param, xs)
 
   local indent = 0
   for _, p in ipairs(xs) do
@@ -715,19 +730,25 @@ local function render_fun(fun, classes, cfg)
   table.insert(ret, render_fun_header(fun, cfg))
   table.insert(ret, '\n')
 
-  if fun.desc then
-    table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH))
-  end
-
   if fun.since then
-    local since = tonumber(fun.since)
+    local since = assert(tonumber(fun.since), 'invalid @since on ' .. fun.name)
     local info = nvim_api_info()
-    if since and (since > info.level or since == info.level and info.prerelease) then
-      fun.notes = fun.notes or {}
-      table.insert(fun.notes, { desc = 'This API is pre-release (unstable).' })
+    if since == 0 or (info.prerelease and since == info.level) then
+      -- Experimental = (since==0 or current prerelease)
+      local s = 'WARNING: This feature is experimental/unstable.'
+      table.insert(ret, md_to_vimdoc(s, INDENTATION, INDENTATION, TEXT_WIDTH))
+      table.insert(ret, '\n')
+    else
+      local v = assert(util.version_level[since], 'invalid @since on ' .. fun.name)
+      fun.attrs = fun.attrs or {}
+      table.insert(fun.attrs, ('Since: %s'):format(v))
     end
   end
 
+  if fun.desc then
+    table.insert(ret, md_to_vimdoc(fun.desc, INDENTATION, INDENTATION, TEXT_WIDTH))
+  end
+
   if fun.notes then
     table.insert(ret, '\n    Note: ~\n')
     for _, p in ipairs(fun.notes) do
@@ -813,7 +834,7 @@ local function get_script_path()
 end
 
 local script_path = get_script_path()
-local base_dir = vim.fs.dirname(assert(vim.fs.dirname(script_path)))
+local base_dir = vim.fs.dirname(vim.fs.dirname(script_path))
 
 local function delete_lines_below(doc_file, tokenstr)
   local lines = {} --- @type string[]
@@ -865,7 +886,11 @@ local function make_section(filename, cfg, section_docs, funs_txt)
   local sectname = cfg.section_name and cfg.section_name[filename] or mktitle(name)
 
   -- section tag: e.g., "*api-autocmd*"
-  local help_tag = '*' .. cfg.helptag_fmt(sectname) .. '*'
+  local help_labels = cfg.helptag_fmt(sectname)
+  if type(help_labels) == 'table' then
+    help_labels = table.concat(help_labels, '* *')
+  end
+  local help_tags = '*' .. help_labels .. '*'
 
   if funs_txt == '' and #section_docs == 0 then
     return
@@ -874,7 +899,7 @@ local function make_section(filename, cfg, section_docs, funs_txt)
   return {
     name = sectname,
     title = cfg.section_fmt(sectname),
-    help_tag = help_tag,
+    help_tag = help_tags,
     funs_txt = funs_txt,
     doc = section_docs,
   }
@@ -934,7 +959,7 @@ local function gen_target(cfg)
 
   expand_files(cfg.files)
 
-  --- @type table<string,{[1]:table<string,nvim.luacats.parser.class>, [2]: nvim.luacats.parser.fun[], [3]: string[]}>
+  --- @type table<string,[table<string,nvim.luacats.parser.class>, nvim.luacats.parser.fun[], string[]]>
   local file_results = {}
 
   --- @type table<string,nvim.luacats.parser.class>
@@ -965,7 +990,7 @@ local function gen_target(cfg)
       end
     end
     -- FIXME: Using f_base will confuse `_meta/protocol.lua` with `protocol.lua`
-    local f_base = assert(vim.fs.basename(f))
+    local f_base = vim.fs.basename(f)
     sections[f_base] = make_section(f_base, cfg, briefs_txt, funs_txt)
   end
 
diff --git a/scripts/lintcommit.lua b/scripts/lintcommit.lua
index 96f6304247..7cb57de901 100644
--- a/scripts/lintcommit.lua
+++ b/scripts/lintcommit.lua
@@ -41,12 +41,6 @@ end
 -- Returns nil if the given commit message is valid, or returns a string
 -- message explaining why it is invalid.
 local function validate_commit(commit_message)
-  -- Return nil if the commit message starts with "fixup" as it signifies it's
-  -- a work in progress and shouldn't be linted yet.
-  if vim.startswith(commit_message, 'fixup') then
-    return nil
-  end
-
   local commit_split = vim.split(commit_message, ':', { plain = true })
   -- Return nil if the type is vim-patch since most of the normal rules don't
   -- apply.
@@ -74,11 +68,12 @@ local function validate_commit(commit_message)
   if after_idx > vim.tbl_count(commit_split) then
     return [[Commit message does not include colons.]]
   end
-  local after_colon = ''
+  local after_colon_split = {}
   while after_idx <= vim.tbl_count(commit_split) do
-    after_colon = after_colon .. commit_split[after_idx]
+    table.insert(after_colon_split, commit_split[after_idx])
     after_idx = after_idx + 1
   end
+  local after_colon = table.concat(after_colon_split, ':')
 
   -- Check if commit introduces a breaking change.
   if vim.endswith(before_colon, '!') then
@@ -229,9 +224,9 @@ function M._test()
     ['vim-patch:8.2.3374: Pyret files are not recognized (#15642)'] = true,
     ['vim-patch:8.1.1195,8.2.{3417,3419}'] = true,
     ['revert: "ci: use continue-on-error instead of "|| true""'] = true,
-    ['fixup'] = true,
-    ['fixup: commit message'] = true,
-    ['fixup! commit message'] = true,
+    ['fixup'] = false,
+    ['fixup: commit message'] = false,
+    ['fixup! commit message'] = false,
     [':no type before colon 1'] = false,
     [' :no type before colon 2'] = false,
     ['  :no type before colon 3'] = false,
@@ -253,8 +248,10 @@ function M._test()
     ['unknown: using unknown type'] = false,
     ['feat: foo:bar'] = true,
     ['feat: :foo:bar'] = true,
+    ['feat: :Foo:Bar'] = true,
     ['feat(something): foo:bar'] = true,
     ['feat(something): :foo:bar'] = true,
+    ['feat(something): :Foo:Bar'] = true,
     ['feat(:grep): read from pipe'] = true,
     ['feat(:grep/:make): read from pipe'] = true,
     ['feat(:grep): foo:bar'] = true,
diff --git a/scripts/luacats_grammar.lua b/scripts/luacats_grammar.lua
index 29f3bda5aa..34c1470fea 100644
--- a/scripts/luacats_grammar.lua
+++ b/scripts/luacats_grammar.lua
@@ -4,7 +4,7 @@ LPEG grammar for LuaCATS
 
 local lpeg = vim.lpeg
 local P, R, S = lpeg.P, lpeg.R, lpeg.S
-local Ct, Cg = lpeg.Ct, lpeg.Cg
+local C, Ct, Cg = lpeg.C, lpeg.Ct, lpeg.Cg
 
 --- @param x vim.lpeg.Pattern
 local function rep(x)
@@ -23,23 +23,20 @@ end
 
 local ws = rep1(S(' \t'))
 local fill = opt(ws)
-
 local any = P(1) -- (consume one character)
-local letter = R('az', 'AZ') + S('_$')
+local letter = R('az', 'AZ')
 local num = R('09')
-local ident = letter * rep(letter + num + S '-.')
-local string_single = P "'" * rep(any - P "'") * P "'"
-local string_double = P('"') * rep(any - P('"')) * P('"')
-
-local literal = (string_single + string_double + (opt(P('-')) * num) + P('false') + P('true'))
 
-local lname = (ident + P('...')) * opt(P('?'))
-
---- @param x string
+--- @param x string | vim.lpeg.Pattern
 local function Pf(x)
   return fill * P(x) * fill
 end
 
+--- @param x string | vim.lpeg.Pattern
+local function Plf(x)
+  return fill * P(x)
+end
+
 --- @param x string
 local function Sf(x)
   return fill * S(x) * fill
@@ -72,16 +69,6 @@ local v = setmetatable({}, {
   end,
 })
 
-local colon = Pf(':')
-local opt_exact = opt(Cg(Pf('(exact)'), 'access'))
-local access = P('private') + P('protected') + P('package')
-local caccess = Cg(access, 'access')
-local desc_delim = Sf '#:' + ws
-local desc = Cg(rep(any), 'desc')
-local opt_desc = opt(desc_delim * desc)
-local cname = Cg(ident, 'name')
-local opt_parent = opt(colon * Cg(ident, 'parent'))
-
 --- @class nvim.luacats.Param
 --- @field kind 'param'
 --- @field name string
@@ -135,21 +122,69 @@ local function annot(nm, pat)
   return Ct(Cg(P(nm), 'kind'))
 end
 
+local colon = Pf(':')
+local ellipsis = P('...')
+local ident_first = P('_') + letter
+local ident = ident_first * rep(ident_first + num)
+local opt_ident = ident * opt(P('?'))
+local ty_ident_sep = S('-._')
+local ty_ident = ident * rep(ty_ident_sep * ident)
+local string_single = P "'" * rep(any - P "'") * P "'"
+local string_double = P('"') * rep(any - P('"')) * P('"')
+local generic = P('`') * ty_ident * P('`')
+local literal = string_single + string_double + (opt(P('-')) * rep1(num)) + P('false') + P('true')
+local ty_prims = ty_ident + literal + generic
+
+local array_postfix = rep1(Plf('[]'))
+local opt_postfix = rep1(Plf('?'))
+local rep_array_opt_postfix = rep(array_postfix + opt_postfix)
+
+local typedef = P({
+  'typedef',
+  typedef = C(v.type),
+
+  type = v.ty * rep_array_opt_postfix * rep(Pf('|') * v.ty * rep_array_opt_postfix),
+  ty = v.composite + paren(v.typedef),
+  composite = (v.types * array_postfix) + (v.types * opt_postfix) + v.types,
+  types = v.generics + v.kv_table + v.tuple + v.dict + v.table_literal + v.fun + ty_prims,
+
+  tuple = Pf('[') * comma1(v.type) * Plf(']'),
+  dict = Pf('{') * comma1(Pf('[') * v.type * Pf(']') * colon * v.type) * Plf('}'),
+  kv_table = Pf('table') * Pf('<') * v.type * Pf(',') * v.type * Plf('>'),
+  table_literal = Pf('{') * comma1(opt_ident * Pf(':') * v.type) * Plf('}'),
+  fun_param = (opt_ident + ellipsis) * opt(colon * v.type),
+  fun_ret = v.type + (ellipsis * opt(colon * v.type)),
+  fun = Pf('fun') * paren(comma(v.fun_param)) * opt(Pf(':') * comma1(v.fun_ret)),
+  generics = P(ty_ident) * Pf('<') * comma1(v.type) * Plf('>'),
+}) / function(match)
+  return vim.trim(match):gsub('^%((.*)%)$', '%1'):gsub('%?+', '?')
+end
+
+local opt_exact = opt(Cg(Pf('(exact)'), 'access'))
+local access = P('private') + P('protected') + P('package')
+local caccess = Cg(access, 'access')
+local desc_delim = Sf '#:' + ws
+local desc = Cg(rep(any), 'desc')
+local opt_desc = opt(desc_delim * desc)
+local ty_name = Cg(ty_ident, 'name')
+local opt_parent = opt(colon * Cg(ty_ident, 'parent'))
+local lname = (ident + ellipsis) * opt(P('?'))
+
 local grammar = P {
   rep1(P('@') * (v.ats + v.ext_ats)),
 
   ats = annot('param', Cg(lname, 'name') * ws * v.ctype * opt_desc)
-    + annot('return', comma1(Ct(v.ctype * opt(ws * cname))) * opt_desc)
+    + annot('return', comma1(Ct(v.ctype * opt(ws * (ty_name + Cg(ellipsis, 'name'))))) * opt_desc)
     + annot('type', comma1(Ct(v.ctype)) * opt_desc)
-    + annot('cast', cname * ws * opt(Sf('+-')) * v.ctype)
-    + annot('generic', cname * opt(colon * v.ctype))
-    + annot('class', opt_exact * opt(paren(caccess)) * fill * cname * opt_parent)
+    + annot('cast', ty_name * ws * opt(Sf('+-')) * v.ctype)
+    + annot('generic', ty_name * opt(colon * v.ctype))
+    + annot('class', opt_exact * opt(paren(caccess)) * fill * ty_name * opt_parent)
     + annot('field', opt(caccess * ws) * v.field_name * ws * v.ctype * opt_desc)
-    + annot('operator', cname * opt(paren(Cg(v.ltype, 'argtype'))) * colon * v.ctype)
+    + annot('operator', ty_name * opt(paren(Cg(v.ctype, 'argtype'))) * colon * v.ctype)
     + annot(access)
     + annot('deprecated')
-    + annot('alias', cname * opt(ws * v.ctype))
-    + annot('enum', cname)
+    + annot('alias', ty_name * opt(ws * v.ctype))
+    + annot('enum', ty_name)
     + annot('overload', v.ctype)
     + annot('see', opt(desc_delim) * desc)
     + annot('diagnostic', opt(desc_delim) * desc)
@@ -165,21 +200,8 @@ local grammar = P {
   ),
 
   field_name = Cg(lname + (v.ty_index * opt(P('?'))), 'name'),
-
-  ctype = parenOpt(Cg(v.ltype, 'type')),
-  ltype = parenOpt(v.ty_union),
-
-  ty_union = v.ty_opt * rep(Pf('|') * v.ty_opt),
-  ty = v.ty_fun + ident + v.ty_table + literal + paren(v.ty) + v.ty_generic,
-  ty_param = Pf('<') * comma1(v.ltype) * fill * P('>'),
-  ty_opt = v.ty * opt(v.ty_param) * opt(P('[]')) * opt(P('?')),
-  ty_index = (Pf('[') * (v.ltype + ident + rep1(num)) * fill * P(']')),
-  table_key = v.ty_index + lname,
-  table_elem = v.table_key * colon * v.ltype,
-  ty_table = Pf('{') * comma1(v.table_elem) * fill * P('}'),
-  fun_param = lname * opt(colon * v.ltype),
-  ty_fun = Pf('fun') * paren(comma(lname * opt(colon * v.ltype))) * opt(colon * comma1(v.ltype)),
-  ty_generic = P('`') * letter * P('`'),
+  ty_index = C(Pf('[') * typedef * fill * P(']')),
+  ctype = Cg(typedef, 'type'),
 }
 
 return grammar --[[@as nvim.luacats.grammar]]
diff --git a/scripts/luacats_parser.lua b/scripts/luacats_parser.lua
index cb301b32e4..9a763e4d7b 100644
--- a/scripts/luacats_parser.lua
+++ b/scripts/luacats_parser.lua
@@ -46,6 +46,7 @@ local luacats_grammar = require('scripts.luacats_grammar')
 --- @field type string
 --- @field desc string
 --- @field access? 'private'|'package'|'protected'
+--- @field nodoc? true
 
 --- @class nvim.luacats.parser.class
 --- @field kind 'class'
@@ -252,9 +253,12 @@ end
 --- @return nvim.luacats.parser.field
 local function fun2field(fun)
   local parts = { 'fun(' }
+
+  local params = {} ---@type string[]
   for _, p in ipairs(fun.params or {}) do
-    parts[#parts + 1] = string.format('%s: %s', p.name, p.type)
+    params[#params + 1] = string.format('%s: %s', p.name, p.type)
   end
+  parts[#parts + 1] = table.concat(params, ', ')
   parts[#parts + 1] = ')'
   if fun.returns then
     parts[#parts + 1] = ': '
@@ -270,6 +274,7 @@ local function fun2field(fun)
     type = table.concat(parts, ''),
     access = fun.access,
     desc = fun.desc,
+    nodoc = fun.nodoc,
   }
 end
 
@@ -458,7 +463,7 @@ local function dump_uncommitted(filename, uncommitted)
   local out_path = 'luacats-uncommited/' .. filename:gsub('/', '%%') .. '.txt'
   if #uncommitted > 0 then
     print(string.format('Could not commit %d objects in %s', #uncommitted, filename))
-    vim.fn.mkdir(assert(vim.fs.dirname(out_path)), 'p')
+    vim.fn.mkdir(vim.fs.dirname(out_path), 'p')
     local f = assert(io.open(out_path, 'w'))
     for i, x in ipairs(uncommitted) do
       f:write(i)
diff --git a/scripts/text_utils.lua b/scripts/util.lua
index 75b3bfedd5..5940221abe 100644
--- a/scripts/text_utils.lua
+++ b/scripts/util.lua
@@ -1,7 +1,9 @@
+-- TODO(justinmk): move most of this to `vim.text`.
+
 local fmt = string.format
 
---- @class nvim.text_utils.MDNode
---- @field [integer] nvim.text_utils.MDNode
+--- @class nvim.util.MDNode
+--- @field [integer] nvim.util.MDNode
 --- @field type string
 --- @field text? string
 
@@ -15,6 +17,24 @@ local function contains(t, xs)
   return vim.tbl_contains(xs, t)
 end
 
+-- Map of api_level:version, by inspection of:
+--    :lua= vim.mpack.decode(vim.fn.readfile('test/functional/fixtures/api_level_9.mpack','B')).version
+M.version_level = {
+  [13] = '0.11.0',
+  [12] = '0.10.0',
+  [11] = '0.9.0',
+  [10] = '0.8.0',
+  [9] = '0.7.0',
+  [8] = '0.6.0',
+  [7] = '0.5.0',
+  [6] = '0.4.0',
+  [5] = '0.3.2',
+  [4] = '0.3.0',
+  [3] = '0.2.1',
+  [2] = '0.2.0',
+  [1] = '0.1.0',
+}
+
 --- @param txt string
 --- @param srow integer
 --- @param scol integer
@@ -47,13 +67,13 @@ local function slice_text(txt, srow, scol, erow, ecol)
 end
 
 --- @param text string
---- @return nvim.text_utils.MDNode
+--- @return nvim.util.MDNode
 local function parse_md_inline(text)
   local parser = vim.treesitter.languagetree.new(text, 'markdown_inline')
   local root = parser:parse(true)[1]:root()
 
   --- @param node TSNode
-  --- @return nvim.text_utils.MDNode?
+  --- @return nvim.util.MDNode?
   local function extract(node)
     local ntype = node:type()
 
@@ -101,7 +121,7 @@ local function parse_md_inline(text)
 end
 
 --- @param text string
---- @return nvim.text_utils.MDNode
+--- @return nvim.util.MDNode
 local function parse_md(text)
   local parser = vim.treesitter.languagetree.new(text, 'markdown', {
     injections = { markdown = '' },
@@ -119,7 +139,7 @@ local function parse_md(text)
   }
 
   --- @param node TSNode
-  --- @return nvim.text_utils.MDNode?
+  --- @return nvim.util.MDNode?
   local function extract(node)
     local ntype = node:type()
 
@@ -153,6 +173,20 @@ local function parse_md(text)
   return extract(root) or {}
 end
 
+--- Prefixes each line in `text`.
+---
+--- Does not wrap, not important for "meta" files? (You probably want md_to_vimdoc instead.)
+---
+--- @param text string
+--- @param prefix_ string
+function M.prefix_lines(prefix_, text)
+  local r = ''
+  for _, l in ipairs(vim.split(text, '\n', { plain = true })) do
+    r = r .. vim.trim(prefix_ .. l) .. '\n'
+  end
+  return r
+end
+
 --- @param x string
 --- @param start_indent integer
 --- @param indent integer
@@ -179,7 +213,7 @@ function M.wrap(x, start_indent, indent, text_width)
   return (table.concat(parts):gsub('%s+\n', '\n'):gsub('\n+$', ''))
 end
 
---- @param node nvim.text_utils.MDNode
+--- @param node nvim.util.MDNode
 --- @param start_indent integer
 --- @param indent integer
 --- @param text_width integer
@@ -207,6 +241,8 @@ local function render_md(node, start_indent, indent, text_width, level, is_list)
   elseif ntype == 'shortcut_link' then
     if node[1].text:find('^<.*>$') then
       parts[#parts + 1] = node[1].text
+    elseif node[1].text:find('^%d+$') then
+      vim.list_extend(parts, { '[', node[1].text, ']' })
     else
       vim.list_extend(parts, { '|', node[1].text, '|' })
     end
diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh
index e8758c064f..bfa9f6d99c 100755
--- a/scripts/vim-patch.sh
+++ b/scripts/vim-patch.sh
@@ -156,7 +156,7 @@ assign_commit_details() {
     local munge_commit_line=true
   else
     # Interpret parameter as commit hash.
-    vim_version="${1:0:12}"
+    vim_version="${1:0:7}"
     vim_tag=
     vim_commit_ref="$vim_version"
     local munge_commit_line=false
@@ -207,7 +207,7 @@ preprocess_patch() {
   2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('"${na_rt}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove unwanted Vim doc files.
-  local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|print\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags'
+  local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|print\.txt\|term\.txt\|todo\.txt\|vim9\.txt\|tags'
   2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('"${na_doc}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove "Last change ..." changes in doc files.
@@ -293,8 +293,12 @@ preprocess_patch() {
   LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/option\.h/\1\/option_vars.h/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
-  # Rename terminal.txt to nvim_terminal_emulator.txt
-  LC_ALL=C sed -Ee 's/( [ab]\/runtime\/doc)\/terminal\.txt/\1\/nvim_terminal_emulator.txt/g' \
+  # Rename version*.txt to news.txt
+  LC_ALL=C sed -Ee 's/( [ab]\/runtime\/doc)\/version[0-9]+\.txt/\1\/news.txt/g' \
+    "$file" > "$file".tmp && mv "$file".tmp "$file"
+
+  # Rename sponsor.txt to intro.txt
+  LC_ALL=C sed -Ee 's/( [ab]\/runtime\/doc)\/sponsor\.txt/\1\/intro.txt/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename test_urls.vim to check_urls.vim