10 files changed, 443 insertions, 227 deletions

diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua
index c5294893e0..e332ef475f 100755
--- a/scripts/bump_deps.lua
+++ b/scripts/bump_deps.lua
@@ -81,6 +81,14 @@ local function get_dependency(dependency_name)
       repo = 'luvit/luv',
       symbol = 'LUV',
     },
+    ['unibilium'] = {
+      repo = 'neovim/unibilium',
+      symbol = 'UNIBILIUM',
+    },
+    ['utf8proc'] = {
+      repo = 'JuliaStrings/utf8proc',
+      symbol = 'UTF8PROC',
+    },
     ['tree-sitter'] = {
       repo = 'tree-sitter/tree-sitter',
       symbol = 'TREESITTER',
@@ -90,11 +98,11 @@ local function get_dependency(dependency_name)
       symbol = 'TREESITTER_C',
     },
     ['tree-sitter-lua'] = {
-      repo = 'MunifTanjim/tree-sitter-lua',
+      repo = 'tree-sitter-grammars/tree-sitter-lua',
       symbol = 'TREESITTER_LUA',
     },
     ['tree-sitter-vim'] = {
-      repo = 'neovim/tree-sitter-vim',
+      repo = 'tree-sitter-grammars/tree-sitter-vim',
       symbol = 'TREESITTER_VIM',
     },
     ['tree-sitter-vimdoc'] = {
@@ -102,9 +110,21 @@ local function get_dependency(dependency_name)
       symbol = 'TREESITTER_VIMDOC',
     },
     ['tree-sitter-query'] = {
-      repo = 'nvim-treesitter/tree-sitter-query',
+      repo = 'tree-sitter-grammars/tree-sitter-query',
       symbol = 'TREESITTER_QUERY',
     },
+    ['tree-sitter-markdown'] = {
+      repo = 'tree-sitter-grammars/tree-sitter-markdown',
+      symbol = 'TREESITTER_MARKDOWN',
+    },
+    ['wasmtime'] = {
+      repo = 'bytecodealliance/wasmtime',
+      symbol = 'WASMTIME',
+    },
+    ['uncrustify'] = {
+      repo = 'uncrustify/uncrustify',
+      symbol = 'UNCRUSTIFY',
+    },
   }
   local dependency = dependency_table[dependency_name]
   if dependency == nil then
diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua
index f1bba5c0a2..b9ea4e73f0 100755
--- a/scripts/gen_eval_files.lua
+++ b/scripts/gen_eval_files.lua
@@ -2,11 +2,15 @@
 
 -- Generator for various vimdoc and Lua type files
 
+local util = require('scripts.util')
+local fmt = string.format
+
 local DEP_API_METADATA = 'build/funcs_metadata.mpack'
+local TEXT_WIDTH = 78
 
 --- @class vim.api.metadata
 --- @field name string
---- @field parameters {[1]:string,[2]:string}[]
+--- @field parameters [string,string][]
 --- @field return_type string
 --- @field deprecated_since integer
 --- @field eval boolean
@@ -20,7 +24,7 @@ local DEP_API_METADATA = 'build/funcs_metadata.mpack'
 
 local LUA_API_RETURN_OVERRIDES = {
   nvim_buf_get_command = 'table<string,vim.api.keyset.command_info>',
-  nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item',
+  nvim_buf_get_extmark_by_id = 'vim.api.keyset.get_extmark_item_by_id',
   nvim_buf_get_extmarks = 'vim.api.keyset.get_extmark_item[]',
   nvim_buf_get_keymap = 'vim.api.keyset.keymap[]',
   nvim_get_autocmds = 'vim.api.keyset.get_autocmds.ret[]',
@@ -29,11 +33,11 @@ local LUA_API_RETURN_OVERRIDES = {
   nvim_get_keymap = 'vim.api.keyset.keymap[]',
   nvim_get_mark = 'vim.api.keyset.get_mark',
 
-  -- Can also return table<string,vim.api.keyset.hl_info>, however we need to
+  -- Can also return table<string,vim.api.keyset.get_hl_info>, however we need to
   -- pick one to get some benefit.
   -- REVISIT lewrus01 (26/01/24): we can maybe add
-  -- @overload fun(ns: integer, {}): table<string,vim.api.keyset.hl_info>
-  nvim_get_hl = 'vim.api.keyset.hl_info',
+  -- @overload fun(ns: integer, {}): table<string,vim.api.keyset.get_hl_info>
+  nvim_get_hl = 'vim.api.keyset.get_hl_info',
 
   nvim_get_mode = 'vim.api.keyset.get_mode',
   nvim_get_namespaces = 'table<string,integer>',
@@ -112,7 +116,7 @@ local API_TYPES = {
   String = 'string',
   Array = 'any[]',
   LuaRef = 'function',
-  Dictionary = 'table<string,any>',
+  Dict = 'table<string,any>',
   Float = 'number',
   HLGroupID = 'number|string',
   void = '',
@@ -140,7 +144,7 @@ local function api_type(t)
     return 'vim.api.keyset.' .. d
   end
 
-  local d0 = t:match('^DictionaryOf%((.*)%)')
+  local d0 = t:match('^DictOf%((.*)%)')
   if d0 then
     return 'table<string,' .. api_type(d0) .. '>'
   end
@@ -149,7 +153,7 @@ local function api_type(t)
 end
 
 --- @param f string
---- @param params {[1]:string,[2]:string}[]|true
+--- @param params [string,string][]|true
 --- @return string
 local function render_fun_sig(f, params)
   local param_str --- @type string
@@ -158,7 +162,7 @@ local function render_fun_sig(f, params)
   else
     param_str = table.concat(
       vim.tbl_map(
-        --- @param v {[1]:string,[2]:string}
+        --- @param v [string,string]
         --- @return string
         function(v)
           return v[1]
@@ -170,16 +174,16 @@ local function render_fun_sig(f, params)
   end
 
   if LUA_KEYWORDS[f] then
-    return string.format("vim.fn['%s'] = function(%s) end", f, param_str)
+    return fmt("vim.fn['%s'] = function(%s) end", f, param_str)
   else
-    return string.format('function vim.fn.%s(%s) end', f, param_str)
+    return fmt('function vim.fn.%s(%s) end', f, param_str)
   end
 end
 
 --- Uniquify names
 --- Fix any names that are lua keywords
---- @param params {[1]:string,[2]:string,[3]:string}[]
---- @return {[1]:string,[2]:string,[3]:string}[]
+--- @param params [string,string,string][]
+--- @return [string,string,string][]
 local function process_params(params)
   local seen = {} --- @type table<string,true>
   local sfx = 1
@@ -199,14 +203,6 @@ local function process_params(params)
   return params
 end
 
---- @class vim.gen_vim_doc_fun
---- @field signature string
---- @field doc string[]
---- @field parameters_doc table<string,string>
---- @field return string[]
---- @field seealso string[]
---- @field annotations string[]
-
 --- @return table<string, vim.EvalFn>
 local function get_api_meta()
   local ret = {} --- @type table<string, vim.EvalFn>
@@ -245,7 +241,17 @@ local function get_api_meta()
   for _, fun in pairs(functions) do
     local deprecated = fun.deprecated_since ~= nil
 
-    local params = {} --- @type {[1]:string,[2]:string}[]
+    local notes = {} --- @type string[]
+    for _, note in ipairs(fun.notes or {}) do
+      notes[#notes + 1] = note.desc
+    end
+
+    local sees = {} --- @type string[]
+    for _, see in ipairs(fun.see or {}) do
+      sees[#sees + 1] = see.desc
+    end
+
+    local params = {} --- @type [string,string][]
     for _, p in ipairs(fun.params) do
       params[#params + 1] = {
         p.name,
@@ -258,13 +264,15 @@ local function get_api_meta()
       signature = 'NA',
       name = fun.name,
       params = params,
+      notes = notes,
+      see = sees,
       returns = api_type(fun.returns[1].type),
       deprecated = deprecated,
     }
 
     if not deprecated then
       r.desc = fun.desc
-      r.return_desc = fun.returns[1].desc
+      r.returns_desc = fun.returns[1].desc
     end
 
     ret[fun.name] = r
@@ -278,8 +286,19 @@ end
 --- Ensure code blocks have one empty line before the start fence and after the closing fence.
 ---
 --- @param x string
+--- @param special string?
+---                | 'see-api-meta' Normalize `@see` for API meta docstrings.
 --- @return string
-local function norm_text(x)
+local function norm_text(x, special)
+  if special == 'see-api-meta' then
+    -- Try to guess a symbol that actually works in @see.
+    -- "nvim_xx()" => "vim.api.nvim_xx"
+    x = x:gsub([=[%|?(nvim_[^.()| ]+)%(?%)?%|?]=], 'vim.api.%1')
+    -- TODO: Remove backticks when LuaLS resolves: https://github.com/LuaLS/lua-language-server/issues/2889
+    -- "|foo|" => "`:help foo`"
+    x = x:gsub([=[|([^ ]+)|]=], '`:help %1`')
+  end
+
   return (
     x:gsub('|([^ ]+)|', '`%1`')
       :gsub('\n*>lua', '\n\n```lua')
@@ -291,14 +310,13 @@ local function norm_text(x)
   )
 end
 
+--- Generates LuaLS docstring for an API function.
 --- @param _f string
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
 local function render_api_meta(_f, fun, write)
   write('')
 
-  local text_utils = require('scripts.text_utils')
-
   if vim.startswith(fun.name, 'nvim__') then
     write('--- @private')
   end
@@ -309,10 +327,18 @@ local function render_api_meta(_f, fun, write)
 
   local desc = fun.desc
   if desc then
-    desc = text_utils.md_to_vimdoc(desc, 0, 0, 74)
-    for _, l in ipairs(split(norm_text(desc))) do
-      write('--- ' .. l)
-    end
+    write(util.prefix_lines('--- ', norm_text(desc)))
+  end
+
+  -- LuaLS doesn't support @note. Render @note items as a markdown list.
+  if fun.notes and #fun.notes > 0 then
+    write('--- Note:')
+    write(util.prefix_lines('--- ', table.concat(fun.notes, '\n')))
+    write('---')
+  end
+
+  for _, see in ipairs(fun.see or {}) do
+    write(util.prefix_lines('--- @see ', norm_text(see, 'see-api-meta')))
   end
 
   local param_names = {} --- @type string[]
@@ -322,8 +348,6 @@ local function render_api_meta(_f, fun, write)
     local pdesc = p[3]
     if pdesc then
       local s = '--- @param ' .. p[1] .. ' ' .. p[2] .. ' '
-      local indent = #('@param ' .. p[1] .. ' ')
-      pdesc = text_utils.md_to_vimdoc(pdesc, #s, indent, 74, true)
       local pdesc_a = split(vim.trim(norm_text(pdesc)))
       write(s .. pdesc_a[1])
       for i = 2, #pdesc_a do
@@ -336,15 +360,15 @@ local function render_api_meta(_f, fun, write)
       write('--- @param ' .. p[1] .. ' ' .. p[2])
     end
   end
+
   if fun.returns ~= '' then
-    local ret_desc = fun.returns_desc and ' : ' .. fun.returns_desc or ''
-    ret_desc = text_utils.md_to_vimdoc(ret_desc, 0, 0, 74)
+    local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or ''
     local ret = LUA_API_RETURN_OVERRIDES[fun.name] or fun.returns
-    write('--- @return ' .. ret .. ret_desc)
+    write(util.prefix_lines('--- ', '@return ' .. ret .. ret_desc))
   end
   local param_str = table.concat(param_names, ', ')
 
-  write(string.format('function vim.api.%s(%s) end', fun.name, param_str))
+  write(fmt('function vim.api.%s(%s) end', fun.name, param_str))
 end
 
 --- @return table<string, vim.EvalFn>
@@ -372,10 +396,14 @@ local function get_api_keysets_meta()
   return ret
 end
 
+--- Generates LuaLS docstring for an API keyset.
 --- @param _f string
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
 local function render_api_keyset_meta(_f, fun, write)
+  if string.sub(fun.name, 1, 1) == '_' then
+    return -- not exported
+  end
   write('')
   write('--- @class vim.api.keyset.' .. fun.name)
   for _, p in ipairs(fun.params) do
@@ -388,6 +416,7 @@ local function get_eval_meta()
   return require('src/nvim/eval').funcs
 end
 
+--- Generates LuaLS docstring for a Vimscript "eval" function.
 --- @param f string
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
@@ -397,7 +426,6 @@ local function render_eval_meta(f, fun, write)
   end
 
   local funname = fun.name or f
-
   local params = process_params(fun.params)
 
   write('')
@@ -421,25 +449,28 @@ local function render_eval_meta(f, fun, write)
   for i, param in ipairs(params) do
     local pname, ptype = param[1], param[2]
     local optional = (pname ~= '...' and i > req_args) and '?' or ''
-    write(string.format('--- @param %s%s %s', pname, optional, ptype))
+    write(fmt('--- @param %s%s %s', pname, optional, ptype))
   end
 
   if fun.returns ~= false then
-    write('--- @return ' .. (fun.returns or 'any'))
+    local ret_desc = fun.returns_desc and ' # ' .. fun.returns_desc or ''
+    write('--- @return ' .. (fun.returns or 'any') .. ret_desc)
   end
 
   write(render_fun_sig(funname, params))
 end
 
+--- Generates vimdoc heading for a Vimscript "eval" function signature.
 --- @param name string
+--- @param name_tag boolean
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
-local function render_sig_and_tag(name, fun, write)
+local function render_sig_and_tag(name, name_tag, fun, write)
   if not fun.signature then
     return
   end
 
-  local tags = { '*' .. name .. '()*' }
+  local tags = name_tag and { '*' .. name .. '()*' } or {}
 
   if fun.tags then
     for _, t in ipairs(fun.tags) do
@@ -447,6 +478,11 @@ local function render_sig_and_tag(name, fun, write)
     end
   end
 
+  if #tags == 0 then
+    write(fun.signature)
+    return
+  end
+
   local tag = table.concat(tags, ' ')
   local siglen = #fun.signature
   local conceal_offset = 2 * (#tags - 1)
@@ -456,32 +492,28 @@ local function render_sig_and_tag(name, fun, write)
     write(string.rep(' ', tag_pad_len) .. tag)
     write(fun.signature)
   else
-    write(string.format('%s%s%s', fun.signature, string.rep(' ', tag_pad_len - siglen), tag))
+    write(fmt('%s%s%s', fun.signature, string.rep(' ', tag_pad_len - siglen), tag))
   end
 end
 
+--- Generates vimdoc for a Vimscript "eval" function.
 --- @param f string
 --- @param fun vim.EvalFn
 --- @param write fun(line: string)
 local function render_eval_doc(f, fun, write)
-  if fun.deprecated then
-    return
-  end
-
-  if not fun.signature then
+  if fun.deprecated or not fun.signature then
     return
   end
 
-  if f:find('__%d+$') then
-    write(fun.signature)
-  else
-    render_sig_and_tag(fun.name or f, fun, write)
-  end
+  render_sig_and_tag(fun.name or f, not f:find('__%d+$'), fun, write)
 
   if not fun.desc then
     return
   end
 
+  local params = process_params(fun.params)
+  local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0
+
   local desc_l = split(vim.trim(fun.desc))
   for _, l in ipairs(desc_l) do
     l = l:gsub('^      ', '')
@@ -497,6 +529,26 @@ local function render_eval_doc(f, fun, write)
   if #desc_l > 0 and not desc_l[#desc_l]:match('^<?$') then
     write('')
   end
+
+  if #params > 0 then
+    write(util.md_to_vimdoc('Parameters: ~', 16, 16, TEXT_WIDTH))
+    for i, param in ipairs(params) do
+      local pname, ptype = param[1], param[2]
+      local optional = (pname ~= '...' and i > req_args) and '?' or ''
+      local s = fmt('- %-14s (`%s%s`)', fmt('{%s}', pname), ptype, optional)
+      write(util.md_to_vimdoc(s, 16, 18, TEXT_WIDTH))
+    end
+    write('')
+  end
+
+  if fun.returns ~= false then
+    write(util.md_to_vimdoc('Return: ~', 16, 16, TEXT_WIDTH))
+    local ret = ('(`%s`)'):format((fun.returns or 'any'))
+    ret = ret .. (fun.returns_desc and ' ' .. fun.returns_desc or '')
+    ret = util.md_to_vimdoc(ret, 18, 18, TEXT_WIDTH)
+    write(ret)
+    write('')
+  end
 end
 
 --- @param d vim.option_defaults
@@ -734,9 +786,9 @@ local function render_option_doc(_f, opt, write)
 
   local name_str --- @type string
   if opt.abbreviation then
-    name_str = string.format("'%s' '%s'", opt.full_name, opt.abbreviation)
+    name_str = fmt("'%s' '%s'", opt.full_name, opt.abbreviation)
   else
-    name_str = string.format("'%s'", opt.full_name)
+    name_str = fmt("'%s'", opt.full_name)
   end
 
   local otype = opt.type == 'boolean' and 'boolean' or opt.type
@@ -744,13 +796,13 @@ local function render_option_doc(_f, opt, write)
     local v = render_option_default(opt.defaults, true)
     local pad = string.rep('\t', math.max(1, math.ceil((24 - #name_str) / 8)))
     if opt.defaults.doc then
-      local deflen = #string.format('%s%s%s (', name_str, pad, otype)
+      local deflen = #fmt('%s%s%s (', name_str, pad, otype)
       --- @type string
       v = v:gsub('\n', '\n' .. string.rep(' ', deflen - 2))
     end
-    write(string.format('%s%s%s\t(default %s)', name_str, pad, otype, v))
+    write(fmt('%s%s%s\t(default %s)', name_str, pad, otype, v))
   else
-    write(string.format('%s\t%s', name_str, otype))
+    write(fmt('%s\t%s', name_str, otype))
   end
 
   write('\t\t\t' .. scope_to_doc(opt.scope) .. scope_more_doc(opt))
diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua
index cdfb85bde6..f6e799508b 100644
--- a/scripts/gen_help_html.lua
+++ b/scripts/gen_help_html.lua
@@ -1,6 +1,4 @@
--- Converts Vim :help files to HTML.  Validates |tag| links and document syntax (parser errors).
---
--- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags).
+--- Converts Nvim :help files to HTML.  Validates |tag| links and document syntax (parser errors).
 --
 -- USAGE (For CI/local testing purposes): Simply `make lintdoc` or `scripts/lintdoc.lua`, which
 -- basically does the following:
@@ -23,6 +21,8 @@
 --   1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +q
 --
 -- NOTES:
+--   * This script is used by the automation repo: https://github.com/neovim/doc
+--   * :helptags checks for duplicate tags, whereas this script checks _links_ (to tags).
 --   * gen() and validate() are the primary (programmatic) entrypoints. validate() only exists
 --     because gen() is too slow (~1 min) to run in per-commit CI.
 --   * visit_node() is the core function used by gen() to traverse the document tree and produce HTML.
@@ -68,6 +68,7 @@ local new_layout = {
   ['dev_theme.txt'] = true,
   ['dev_tools.txt'] = true,
   ['dev_vimpatch.txt'] = true,
+  ['editorconfig.txt'] = true,
   ['faq.txt'] = true,
   ['lua.txt'] = true,
   ['luaref.txt'] = true,
@@ -76,10 +77,17 @@ local new_layout = {
   ['news-0.10.txt'] = true,
   ['nvim.txt'] = true,
   ['provider.txt'] = true,
+  ['tui.txt'] = true,
   ['ui.txt'] = true,
   ['vim_diff.txt'] = true,
 }
 
+-- Map of new:old pages, to redirect renamed pages.
+local redirects = {
+  ['tui'] = 'term',
+  ['terminal'] = 'nvim_terminal_emulator',
+}
+
 -- TODO: These known invalid |links| require an update to the relevant docs.
 local exclude_invalid = {
   ["'string'"] = 'eval.txt',
@@ -212,6 +220,7 @@ local function is_noise(line, noise_lines)
 end
 
 --- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content.
+--- @return string
 local function get_bug_url_vimdoc(fname, to_fname, sample_text)
   local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname))
   local bug_url = (
@@ -227,6 +236,7 @@ local function get_bug_url_vimdoc(fname, to_fname, sample_text)
 end
 
 --- Creates a github issue URL at neovim/neovim with prefilled content.
+--- @return string
 local function get_bug_url_nvim(fname, to_fname, sample_text, token_name)
   local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname))
   local bug_url = (
@@ -255,7 +265,7 @@ local function get_helppage(f)
     return 'index.html'
   end
 
-  return (f:gsub('%.txt$', '.html'))
+  return (f:gsub('%.txt$', '')) .. '.html'
 end
 
 --- Counts leading spaces (tab=8) to decide the indent size of multiline text.
@@ -490,7 +500,6 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     or nil
   -- Parent kind (string).
   local parent = root:parent() and root:parent():type() or nil
-  local text = ''
   -- Gets leading whitespace of `node`.
   local function ws(node)
     node = node or root
@@ -508,6 +517,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf))
   end
 
+  local text = ''
   local trimmed ---@type string
   if root:named_child_count() == 0 or node_name == 'ERROR' then
     text = node_text()
@@ -537,12 +547,17 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     if is_noise(text, stats.noise_lines) then
       return '' -- Discard common "noise" lines.
     end
-    -- Remove "===" and tags from ToC text.
-    local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', ''))
+    -- Remove tags from ToC text.
+    local heading_node = first(root, 'heading')
+    local hname = trim(node_text(heading_node):gsub('%*.*%*', ''))
+    if not heading_node or hname == '' then
+      return '' -- Spurious "===" or "---" in the help doc.
+    end
+
     -- Use the first *tag* node as the heading anchor, if any.
-    local tagnode = first(root, 'tag')
+    local tagnode = first(heading_node, 'tag')
     -- Use the *tag* as the heading anchor id, if possible.
-    local tagname = tagnode and url_encode(node_text(tagnode:child(1), false))
+    local tagname = tagnode and url_encode(trim(node_text(tagnode:child(1), false)))
       or to_heading_tag(hname)
     if node_name == 'h1' or #headings == 0 then
       ---@type nvim.gen_help_html.heading
@@ -555,7 +570,9 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       )
     end
     local el = node_name == 'h1' and 'h2' or 'h3'
-    return ('<%s id="%s" class="help-heading">%s</%s>\n'):format(el, tagname, text, el)
+    return ('<%s id="%s" class="help-heading">%s</%s>\n'):format(el, tagname, trimmed, el)
+  elseif node_name == 'heading' then
+    return trimmed
   elseif node_name == 'column_heading' or node_name == 'column_name' then
     if root:has_error() then
       return text
@@ -648,13 +665,13 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       code = ('<pre>%s</pre>'):format(trim(trim_indent(text), 2))
     end
     return code
-  elseif node_name == 'tag' then -- anchor
+  elseif node_name == 'tag' then -- anchor, h4 pseudo-heading
     if root:has_error() then
       return text
     end
     local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent)
-    local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right'
-      or 'help-tag'
+    local h4 = not in_heading and not next_ and get_indent(node_text()) > 8 -- h4 pseudo-heading
+    local cssclass = h4 and 'help-tag-right' or 'help-tag'
     local tagname = node_text(root:child(1), false)
     if vim.tbl_count(stats.first_tags) < 2 then
       -- Force the first 2 tags in the doc to be anchored at the main heading.
@@ -694,8 +711,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       -- End the <span> container for tags in a heading.
       return string.format('%s</span>', s)
     end
-    return s
-  elseif node_name == 'modeline' then
+    return s .. (h4 and '<br>' or '') -- HACK: <br> avoids h4 pseudo-heading mushing with text.
+  elseif node_name == 'delimiter' or node_name == 'modeline' then
     return ''
   elseif node_name == 'ERROR' then
     if ignore_parse_error(opt.fname, trimmed) then
@@ -760,14 +777,21 @@ local function ensure_runtimepath()
   end
 end
 
---- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents.
+--- Opens `fname` (or `text`, if given) in a buffer and gets a treesitter parser for the buffer contents.
 ---
---- @param fname string help file to parse
+--- @param fname string :help file to parse
+--- @param text string? :help file contents
 --- @param parser_path string? path to non-default vimdoc.so
 --- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr)
-local function parse_buf(fname, parser_path)
+local function parse_buf(fname, text, parser_path)
   local buf ---@type integer
-  if type(fname) == 'string' then
+  if text then
+    vim.cmd('split new') -- Text contents.
+    vim.api.nvim_put(vim.split(text, '\n'), '', false, false)
+    vim.cmd('setfiletype help')
+    -- vim.treesitter.language.add('vimdoc')
+    buf = vim.api.nvim_get_current_buf()
+  elseif type(fname) == 'string' then
     vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename.
     buf = vim.api.nvim_get_current_buf()
   else
@@ -779,7 +803,7 @@ local function parse_buf(fname, parser_path)
   if parser_path then
     vim.treesitter.language.add('vimdoc', { path = parser_path })
   end
-  local lang_tree = vim.treesitter.get_parser(buf)
+  local lang_tree = assert(vim.treesitter.get_parser(buf, nil, { error = false }))
   return lang_tree, buf
 end
 
@@ -794,7 +818,7 @@ local function validate_one(fname, parser_path)
   local stats = {
     parse_errors = {},
   }
-  local lang_tree, buf = parse_buf(fname, parser_path)
+  local lang_tree, buf = parse_buf(fname, nil, parser_path)
   for _, tree in ipairs(lang_tree:trees()) do
     visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats)
   end
@@ -805,20 +829,21 @@ end
 
 --- Generates HTML from one :help file `fname` and writes the result to `to_fname`.
 ---
---- @param fname string Source :help file
+--- @param fname string Source :help file.
+--- @param text string|nil Source :help file contents, or nil to read `fname`.
 --- @param to_fname string Destination .html file
 --- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace)
 --- @param parser_path string? path to non-default vimdoc.so
 ---
 --- @return string html
 --- @return table stats
-local function gen_one(fname, to_fname, old, commit, parser_path)
+local function gen_one(fname, text, to_fname, old, commit, parser_path)
   local stats = {
     noise_lines = {},
     parse_errors = {},
     first_tags = {}, -- Track the first few tags in doc.
   }
-  local lang_tree, buf = parse_buf(fname, parser_path)
+  local lang_tree, buf = parse_buf(fname, text, parser_path)
   ---@type nvim.gen_help_html.heading[]
   local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3.
   local title = to_titlecase(basename_noext(fname))
@@ -1126,6 +1151,7 @@ local function gen_css(fname)
       margin-left: auto;
       margin-right: 0;
       float: right;
+      display: block;
     }
     .help-tag a,
     .help-tag-right a {
@@ -1139,10 +1165,11 @@ local function gen_css(fname)
       font-size: smaller;
     }
     .help-heading {
-      overflow: hidden;
-      white-space: nowrap;
+      white-space: normal;
       display: flex;
+      flex-flow: row wrap;
       justify-content: space-between;
+      gap: 0 15px;
     }
     /* The (right-aligned) "tags" part of a section heading. */
     .help-heading-tags {
@@ -1177,8 +1204,7 @@ local function gen_css(fname)
     pre:last-child {
       margin-bottom: 0;
     }
-    pre:hover,
-    .help-heading:hover {
+    pre:hover {
       overflow: visible;
     }
     .generator-stats {
@@ -1216,7 +1242,7 @@ end
 
 function M._test()
   tagmap = get_helptags('$VIMRUNTIME/doc')
-  helpfiles = get_helpfiles(vim.fn.expand('$VIMRUNTIME/doc'))
+  helpfiles = get_helpfiles(vim.fs.normalize('$VIMRUNTIME/doc'))
 
   ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap))
   ok(
@@ -1278,7 +1304,7 @@ function M.gen(help_dir, to_dir, include, commit, parser_path)
     help_dir = {
       help_dir,
       function(d)
-        return vim.fn.isdirectory(vim.fn.expand(d)) == 1
+        return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
       end,
       'valid directory',
     },
@@ -1288,40 +1314,88 @@ function M.gen(help_dir, to_dir, include, commit, parser_path)
     parser_path = {
       parser_path,
       function(f)
-        return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1
+        return f == nil or vim.fn.filereadable(vim.fs.normalize(f)) == 1
       end,
       'valid vimdoc.{so,dll} filepath',
     },
   }
 
   local err_count = 0
+  local redirects_count = 0
   ensure_runtimepath()
-  tagmap = get_helptags(vim.fn.expand(help_dir))
+  tagmap = get_helptags(vim.fs.normalize(help_dir))
   helpfiles = get_helpfiles(help_dir, include)
-  to_dir = vim.fn.expand(to_dir)
-  parser_path = parser_path and vim.fn.expand(parser_path) or nil
+  to_dir = vim.fs.normalize(to_dir)
+  parser_path = parser_path and vim.fs.normalize(parser_path) or nil
 
-  print(('output dir: %s'):format(to_dir))
+  print(('output dir: %s\n\n'):format(to_dir))
   vim.fn.mkdir(to_dir, 'p')
   gen_css(('%s/help.css'):format(to_dir))
 
   for _, f in ipairs(helpfiles) do
+    -- "foo.txt"
     local helpfile = vim.fs.basename(f)
+    -- "to/dir/foo.html"
     local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile))
-    local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?', parser_path)
+    local html, stats =
+      gen_one(f, nil, to_fname, not new_layout[helpfile], commit or '?', parser_path)
     tofile(to_fname, html)
     print(
-      ('generated (%-4s errors): %-15s => %s'):format(
+      ('generated (%-2s errors): %-15s => %s'):format(
         #stats.parse_errors,
         helpfile,
         vim.fs.basename(to_fname)
       )
     )
+
+    -- Generate redirect pages for renamed help files.
+    local helpfile_tag = (helpfile:gsub('%.txt$', ''))
+    local redirect_from = redirects[helpfile_tag]
+    if redirect_from then
+      local redirect_text = ([[
+*%s*      Nvim
+
+This document moved to: |%s|
+
+==============================================================================
+This document moved to: |%s|
+
+This document moved to: |%s|
+
+==============================================================================
+ vim:tw=78:ts=8:ft=help:norl:
+      ]]):format(
+        redirect_from,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag,
+        helpfile_tag
+      )
+      local redirect_to = ('%s/%s'):format(to_dir, get_helppage(redirect_from))
+      local redirect_html, _ =
+        gen_one(redirect_from, redirect_text, redirect_to, false, commit or '?', parser_path)
+      assert(redirect_html:find(helpfile_tag))
+      tofile(redirect_to, redirect_html)
+
+      print(
+        ('generated (redirect) : %-15s => %s'):format(
+          redirect_from .. '.txt',
+          vim.fs.basename(to_fname)
+        )
+      )
+      redirects_count = redirects_count + 1
+    end
+
     err_count = err_count + #stats.parse_errors
   end
-  print(('generated %d html pages'):format(#helpfiles))
+
+  print(('\ngenerated %d html pages'):format(#helpfiles + redirects_count))
   print(('total errors: %d'):format(err_count))
-  print(('invalid tags:\n%s'):format(vim.inspect(invalid_links)))
+  print(('invalid tags: %s'):format(vim.inspect(invalid_links)))
+  assert(#(include or {}) > 0 or redirects_count == vim.tbl_count(redirects)) -- sanity check
+  print(('redirects: %d'):format(redirects_count))
+  print('\n')
 
   --- @type nvim.gen_help_html.gen_result
   return {
@@ -1351,7 +1425,7 @@ function M.validate(help_dir, include, parser_path)
     help_dir = {
       help_dir,
       function(d)
-        return vim.fn.isdirectory(vim.fn.expand(d)) == 1
+        return vim.fn.isdirectory(vim.fs.normalize(d)) == 1
       end,
       'valid directory',
     },
@@ -1359,7 +1433,7 @@ function M.validate(help_dir, include, parser_path)
     parser_path = {
       parser_path,
       function(f)
-        return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1
+        return f == nil or vim.fn.filereadable(vim.fs.normalize(f)) == 1
       end,
       'valid vimdoc.{so,dll} filepath',
     },
@@ -1367,12 +1441,12 @@ function M.validate(help_dir, include, parser_path)
   local err_count = 0 ---@type integer
   local files_to_errors = {} ---@type table<string, string[]>
   ensure_runtimepath()
-  tagmap = get_helptags(vim.fn.expand(help_dir))
+  tagmap = get_helptags(vim.fs.normalize(help_dir))
   helpfiles = get_helpfiles(help_dir, include)
-  parser_path = parser_path and vim.fn.expand(parser_path) or nil
+  parser_path = parser_path and vim.fs.normalize(parser_path) or nil
 
   for _, f in ipairs(helpfiles) do
-    local helpfile = assert(vim.fs.basename(f))
+    local helpfile = vim.fs.basename(f)
     local rv = validate_one(f, parser_path)
     print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile))
     if #rv.parse_errors > 0 then
@@ -1404,7 +1478,7 @@ end
 ---
 --- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
 function M.run_validate(help_dir)
-  help_dir = vim.fn.expand(help_dir or '$VIMRUNTIME/doc')
+  help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
   print('doc path = ' .. vim.uv.fs_realpath(help_dir))
 
   local rv = M.validate(help_dir)
@@ -1430,16 +1504,14 @@ end
 ---    :help files, we can be precise about the tolerances here.
 --- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc'
 function M.test_gen(help_dir)
-  local tmpdir = assert(vim.fs.dirname(vim.fn.tempname()))
-  help_dir = vim.fn.expand(help_dir or '$VIMRUNTIME/doc')
+  local tmpdir = vim.fs.dirname(vim.fn.tempname())
+  help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc')
   print('doc path = ' .. vim.uv.fs_realpath(help_dir))
 
-  local rv = M.gen(
-    help_dir,
-    tmpdir,
-    -- Because gen() is slow (~30s), this test is limited to a few files.
-    { 'help.txt', 'index.txt', 'nvim.txt' }
-  )
+  -- Because gen() is slow (~30s), this test is limited to a few files.
+  local input = { 'help.txt', 'index.txt', 'nvim.txt' }
+  local rv = M.gen(help_dir, tmpdir, input)
+  eq(#input, #rv.helpfiles)
   eq(0, rv.err_count, 'parse errors in :help docs')
   eq({}, rv.invalid_links, 'invalid tags in :help docs')
 end
diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua
index 04d19f22e6..c8dcf8c018 100644
--- a/scripts/gen_lsp.lua
+++ b/scripts/gen_lsp.lua
@@ -60,9 +60,10 @@ end
 local function gen_methods(protocol)
   local output = {
     '-- Generated by gen_lsp.lua, keep at end of file.',
-    '--- LSP method names.',
     '---',
+    '---@enum vim.lsp.protocol.Methods',
     '---@see https://microsoft.github.io/language-server-protocol/specification/#metaModel',
+    '--- LSP method names.',
     'protocol.Methods = {',
   }
   local indent = (' '):rep(2)
@@ -109,26 +110,8 @@ local function gen_methods(protocol)
     end
   end
   output[#output + 1] = '}'
-  output = vim.list_extend(
-    output,
-    vim.split(
-      [[
-local function freeze(t)
-  return setmetatable({}, {
-    __index = t,
-    __newindex = function()
-      error('cannot modify immutable table')
-    end,
-  })
-end
-protocol.Methods = freeze(protocol.Methods)
-
-return protocol
-]],
-      '\n',
-      { trimempty = true }
-    )
-  )
+  output[#output + 1] = ''
+  output[#output + 1] = 'return protocol'
 
   local fname = './runtime/lua/vim/lsp/protocol.lua'
   local bufnr = vim.fn.bufadd(fname)
@@ -297,13 +280,13 @@ function M.gen(opt)
 
     -- TupleType
     elseif type.kind == 'tuple' then
-      local tuple = '{ '
-      for i, value in ipairs(type.items) do
-        tuple = tuple .. '[' .. i .. ']: ' .. parse_type(value, prefix) .. ', '
+      local tuple = '['
+      for _, value in ipairs(type.items) do