Merge remote-tracking branch 'upstream/master' into colorcolcharcolorcolchar

12 files changed, 2100 insertions, 1368 deletions

diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua
index 1873c3cd0d..076ad374cf 100755
--- a/scripts/bump_deps.lua
+++ b/scripts/bump_deps.lua
@@ -54,7 +54,7 @@ local function run_die(cmd, err_msg)
 end
 
 local function require_executable(cmd)
-  local cmd_path = run_die({ 'command', '-v', cmd }, cmd .. ' not found!')
+  local cmd_path = run_die({ 'sh', '-c', 'command -v ' .. cmd }, cmd .. ' not found!')
   run_die({ 'test', '-x', cmd_path }, cmd .. ' is not executable')
 end
 
@@ -63,12 +63,13 @@ local function rm_file_if_present(path_to_file)
 end
 
 local nvim_src_dir = vim.fn.getcwd()
+local deps_file = nvim_src_dir .. '/' .. 'cmake.deps/deps.txt'
 local temp_dir = nvim_src_dir .. '/tmp'
 run({ 'mkdir', '-p', temp_dir })
 
 local function get_dependency(dependency_name)
   local dependency_table = {
-    ['LuaJIT'] = {
+    ['luajit'] = {
       repo = 'LuaJIT/LuaJIT',
       symbol = 'LUAJIT',
     },
@@ -76,7 +77,7 @@ local function get_dependency(dependency_name)
       repo = 'libuv/libuv',
       symbol = 'LIBUV',
     },
-    ['Luv'] = {
+    ['luv'] = {
       repo = 'luvit/luv',
       symbol = 'LUV',
     },
@@ -84,6 +85,26 @@ local function get_dependency(dependency_name)
       repo = 'tree-sitter/tree-sitter',
       symbol = 'TREESITTER',
     },
+    ['tree-sitter-c'] = {
+      repo = 'tree-sitter/tree-sitter-c',
+      symbol = 'TREESITTER_C',
+    },
+    ['tree-sitter-lua'] = {
+      repo = 'MunifTanjim/tree-sitter-lua',
+      symbol = 'TREESITTER_LUA',
+    },
+    ['tree-sitter-vim'] = {
+      repo = 'neovim/tree-sitter-vim',
+      symbol = 'TREESITTER_VIM',
+    },
+    ['tree-sitter-vimdoc'] = {
+      repo = 'neovim/tree-sitter-vimdoc',
+      symbol = 'TREESITTER_VIMDOC',
+    },
+    ['tree-sitter-query'] = {
+      repo = 'nvim-treesitter/tree-sitter-query',
+      symbol = 'TREESITTER_QUERY',
+    },
   }
   local dependency = dependency_table[dependency_name]
   if dependency == nil then
@@ -127,26 +148,24 @@ end
 local function write_cmakelists_line(symbol, kind, value)
   require_executable('sed')
 
-  local cmakelists_path = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt'
   run_die({
     'sed',
     '-i',
     '-e',
-    's/set('
+    's/'
       .. symbol
       .. '_'
       .. kind
       .. '.*$'
-      .. '/set('
+      .. '/'
       .. symbol
       .. '_'
       .. kind
       .. ' '
       .. value
-      .. ')'
       .. '/',
-    cmakelists_path,
-  }, 'Failed to write ' .. cmakelists_path)
+    deps_file,
+  }, 'Failed to write ' .. deps_file)
 end
 
 local function explicit_create_branch(dep)
@@ -181,8 +200,6 @@ local function update_cmakelists(dependency, archive, comment)
 
   verify_branch(dependency.name)
 
-  local changed_file = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt'
-
   p('Updating ' .. dependency.name .. ' to ' .. archive.url .. '\n')
   write_cmakelists_line(dependency.symbol, 'URL', archive.url:gsub('/', '\\/'))
   write_cmakelists_line(dependency.symbol, 'SHA256', archive.sha)
@@ -190,7 +207,7 @@ local function update_cmakelists(dependency, archive, comment)
     {
       'git',
       'commit',
-      changed_file,
+      deps_file,
       '-m',
       commit_prefix .. 'bump ' .. dependency.name .. ' to ' .. comment,
     },
@@ -201,10 +218,9 @@ end
 local function verify_cmakelists_committed()
   require_executable('git')
 
-  local cmakelists_path = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt'
   run_die(
-    { 'git', 'diff', '--quiet', 'HEAD', '--', cmakelists_path },
-    cmakelists_path .. ' has uncommitted changes'
+    { 'git', 'diff', '--quiet', 'HEAD', '--', deps_file },
+    deps_file .. ' has uncommitted changes'
   )
 end
 
diff --git a/scripts/gen_eval_files.lua b/scripts/gen_eval_files.lua
new file mode 100755
index 0000000000..e331dd996e
--- /dev/null
+++ b/scripts/gen_eval_files.lua
@@ -0,0 +1,815 @@
+-- Generator for various vimdoc and Lua type files
+
+local DEP_API_METADATA = 'build/api_metadata.mpack'
+local DEP_API_DOC  = 'runtime/doc/api.mpack'
+
+--- @class vim.api.metadata
+--- @field name string
+--- @field parameters {[1]:string,[2]:string}[]
+--- @field return_type string
+--- @field deprecated_since integer
+--- @field eval boolean
+--- @field fast boolean
+--- @field handler_id integer
+--- @field impl_name string
+--- @field lua boolean
+--- @field method boolean
+--- @field remote boolean
+--- @field since integer
+
+local LUA_META_HEADER = {
+  '--- @meta _',
+  '-- THIS FILE IS GENERATED',
+  '-- DO NOT EDIT',
+  "error('Cannot require a meta file')",
+}
+
+local LUA_API_META_HEADER = {
+  '--- @meta _',
+  '-- THIS FILE IS GENERATED',
+  '-- DO NOT EDIT',
+  "error('Cannot require a meta file')",
+  '',
+  'vim.api = {}',
+}
+
+local LUA_OPTION_META_HEADER = {
+  '--- @meta _',
+  '-- THIS FILE IS GENERATED',
+  '-- DO NOT EDIT',
+  "error('Cannot require a meta file')",
+  '',
+  '---@class vim.bo',
+  '---@field [integer] vim.bo',
+  'vim.bo = vim.bo',
+  '',
+  '---@class vim.wo',
+  '---@field [integer] vim.wo',
+  'vim.wo = vim.wo',
+}
+
+local LUA_KEYWORDS = {
+  ['and'] = true,
+  ['end'] = true,
+  ['function'] = true,
+  ['or'] = true,
+  ['if'] = true,
+  ['while'] = true,
+  ['repeat'] = true,
+}
+
+local OPTION_TYPES = {
+  bool = 'boolean',
+  number = 'integer',
+  string = 'string',
+}
+
+local API_TYPES = {
+  Window = 'integer',
+  Tabpage = 'integer',
+  Buffer = 'integer',
+  Boolean = 'boolean',
+  Object = 'any',
+  Integer = 'integer',
+  String = 'string',
+  Array = 'any[]',
+  LuaRef = 'function',
+  Dictionary = 'table<string,any>',
+  Float = 'number',
+  void = '',
+}
+
+--- @param x string
+--- @param sep? string
+--- @return string[]
+local function split(x, sep)
+  return vim.split(x, sep or '\n', { plain = true })
+end
+
+--- Convert an API type to Lua
+--- @param t string
+--- @return string
+local function api_type(t)
+  local as0 = t:match('^ArrayOf%((.*)%)')
+  if as0 then
+    local as = split(as0, ', ')
+    return api_type(as[1]) .. '[]'
+  end
+
+  local d = t:match('^Dict%((.*)%)')
+  if d then
+    return 'vim.api.keyset.' .. d
+  end
+
+  local d0 = t:match('^DictionaryOf%((.*)%)')
+  if d0 then
+    return 'table<string,' .. api_type(d0) .. '>'
+  end
+
+  return API_TYPES[t] or t
+end
+
+--- @param f string
+--- @param params {[1]:string,[2]:string}[]|true
+--- @return string
+local function render_fun_sig(f, params)
+  local param_str --- @type string
+  if params == true then
+    param_str = '...'
+  else
+    param_str = table.concat(
+      vim.tbl_map(
+        --- @param v {[1]:string,[2]:string}
+        --- @return string
+        function(v)
+          return v[1]
+        end,
+        params
+      ),
+      ', '
+    )
+  end
+
+  if LUA_KEYWORDS[f] then
+    return string.format("vim.fn['%s'] = function(%s) end", f, param_str)
+  else
+    return string.format('function vim.fn.%s(%s) end', f, param_str)
+  end
+end
+
+--- Uniquify names
+--- Fix any names that are lua keywords
+--- @param params {[1]:string,[2]:string,[3]:string}[]
+--- @return {[1]:string,[2]:string,[3]:string}[]
+local function process_params(params)
+  local seen = {} --- @type table<string,true>
+  local sfx = 1
+
+  for _, p in ipairs(params) do
+    if LUA_KEYWORDS[p[1]] then
+      p[1] = p[1] .. '_'
+    end
+    if seen[p[1]] then
+      p[1] = p[1] .. sfx
+      sfx = sfx + 1
+    else
+      seen[p[1]] = true
+    end
+  end
+
+  return params
+end
+
+--- @class vim.gen_vim_doc_fun
+--- @field signature string
+--- @field doc string[]
+--- @field parameters_doc table<string,string>
+--- @field return string[]
+--- @field seealso string[]
+--- @field annotations string[]
+
+--- @return table<string, vim.EvalFn>
+local function get_api_meta()
+  local mpack_f = assert(io.open(DEP_API_METADATA, 'rb'))
+  local metadata = vim.mpack.decode(mpack_f:read('*all')) --[[@as vim.api.metadata[] ]]
+  local ret = {} --- @type table<string, vim.EvalFn>
+
+  local doc_mpack_f = assert(io.open(DEP_API_DOC, 'rb'))
+  local doc_metadata = vim.mpack.decode(doc_mpack_f:read('*all')) --[[@as table<string,vim.gen_vim_doc_fun>]]
+
+  for _, fun in ipairs(metadata) do
+    if fun.lua then
+      local fdoc = doc_metadata[fun.name]
+
+      local params = {} --- @type {[1]:string,[2]:string}[]
+      for _, p in ipairs(fun.parameters) do
+        local ptype, pname = p[1], p[2]
+        params[#params + 1] = {
+          pname,
+          api_type(ptype),
+          fdoc and fdoc.parameters_doc[pname] or nil,
+        }
+      end
+
+      local r = {
+        signature = 'NA',
+        name = fun.name,
+        params = params,
+        returns = api_type(fun.return_type),
+        deprecated = fun.deprecated_since ~= nil,
+      }
+
+      if fdoc then
+        if #fdoc.doc > 0 then
+          r.desc = table.concat(fdoc.doc, '\n')
+        end
+        r.return_desc = (fdoc['return'] or {})[1]
+      end
+
+      ret[fun.name] = r
+    end
+  end
+  return ret
+end
+
+--- Convert vimdoc references to markdown literals
+--- Convert vimdoc codeblocks to markdown codeblocks
+---
+--- Ensure code blocks have one empty line before the start fence and after the closing fence.
+---
+--- @param x string
+--- @return string
+local function norm_text(x)
+  return (
+    x:gsub('|([^ ]+)|', '`%1`')
+      :gsub('\n*>lua', '\n\n```lua')
+      :gsub('\n*>vim', '\n\n```vim')
+      :gsub('\n+<$', '\n```')
+      :gsub('\n+<\n+', '\n```\n\n')
+      :gsub('%s+>\n+', '\n```\n')
+      :gsub('\n+<%s+\n?', '\n```\n')
+  )
+end
+
+--- @param _f string
+--- @param fun vim.EvalFn
+--- @param write fun(line: string)
+local function render_api_meta(_f, fun, write)
+  if not vim.startswith(fun.name, 'nvim_') then
+    return
+  end
+
+  write('')
+
+  if vim.startswith(fun.name, 'nvim__') then
+    write('--- @private')
+  end
+
+  if fun.deprecated then
+    write('--- @deprecated')
+  end
+
+  local desc = fun.desc
+  if desc then
+    for _, l in ipairs(split(norm_text(desc))) do
+      write('--- ' .. l)
+    end
+    write('---')
+  end
+
+  local param_names = {} --- @type string[]
+  local params = process_params(fun.params)
+  for _, p in ipairs(params) do
+    param_names[#param_names + 1] = p[1]
+    local pdesc = p[3]
+    if pdesc then
+      local pdesc_a = split(norm_text(pdesc))
+      write('--- @param ' .. p[1] .. ' ' .. p[2] .. ' ' .. pdesc_a[1])
+      for i = 2, #pdesc_a do
+        if not pdesc_a[i] then
+          break
+        end
+        write('--- ' .. pdesc_a[i])
+      end
+    else
+      write('--- @param ' .. p[1] .. ' ' .. p[2])
+    end
+  end
+  if fun.returns ~= '' then
+    if fun.returns_desc then
+      write('--- @return ' .. fun.returns .. ' : ' .. fun.returns_desc)
+    else
+      write('--- @return ' .. fun.returns)
+    end
+  end
+  local param_str = table.concat(param_names, ', ')
+
+  write(string.format('function vim.api.%s(%s) end', fun.name, param_str))
+end
+
+--- @return table<string, vim.EvalFn>
+local function get_api_keysets_meta()
+  local mpack_f = assert(io.open(DEP_API_METADATA, 'rb'))
+
+  --- @diagnostic disable-next-line:no-unknown
+  local metadata = assert(vim.mpack.decode(mpack_f:read('*all')))
+
+  local ret = {} --- @type table<string, vim.EvalFn>
+
+  --- @type {name: string, keys: string[], types: table<string,string>}[]
+  local keysets = metadata.keysets
+
+  for _, k in ipairs(keysets) do
+    local params = {}
+    for _, key in ipairs(k.keys) do
+      table.insert(params, {key..'?', api_type(k.types[key] or 'any')})
+    end
+    ret[k.name] = {
+      signature = 'NA',
+      name = k.name,
+      params = params,
+    }
+  end
+
+  return ret
+end
+
+--- @param _f string
+--- @param fun vim.EvalFn
+--- @param write fun(line: string)
+local function render_api_keyset_meta(_f, fun, write)
+  write('')
+  write('--- @class vim.api.keyset.' .. fun.name)
+  for _, p in ipairs(fun.params) do
+    write('--- @field ' .. p[1] .. ' ' .. p[2])
+  end
+end
+
+--- @return table<string, vim.EvalFn>
+local function get_eval_meta()
+  return require('src/nvim/eval').funcs
+end
+
+--- @param f string
+--- @param fun vim.EvalFn
+--- @param write fun(line: string)
+local function render_eval_meta(f, fun, write)
+  if fun.lua == false then
+    return
+  end
+
+  local funname = fun.name or f
+
+  local params = process_params(fun.params)
+
+  if fun.signature then
+    write('')
+    if fun.deprecated then
+      write('--- @deprecated')
+    end
+
+    local desc = fun.desc
+
+    if desc then
+      --- @type string
+      desc = desc:gsub('\n%s*\n%s*$', '\n')
+      for _, l in ipairs(split(desc)) do
+        l = l:gsub('^      ', ''):gsub('\t', '  '):gsub('@', '\\@')
+        write('--- ' .. l)
+      end
+    end
+
+    local req_args = type(fun.args) == 'table' and fun.args[1] or fun.args or 0
+
+    for i, param in ipairs(params) do
+      local pname, ptype = param[1], param[2]
+      local optional = (pname ~= '...' and i > req_args) and '?' or ''
+      write(string.format('--- @param %s%s %s', pname, optional, ptype))
+    end
+
+    if fun.returns ~= false then
+      write('--- @return ' .. (fun.returns or 'any'))
+    end
+
+    write(render_fun_sig(funname, params))
+
+    return
+  end
+
+  print('no doc for', funname)
+end
+
+--- @type table<string,true>
+local rendered_tags = {}
+
+--- @param name string
+--- @param fun vim.EvalFn
+--- @param write fun(line: string)
+local function render_sig_and_tag(name, fun, write)
+  local tags = { '*' .. name .. '()*' }
+
+  if fun.tags then
+    for _, t in ipairs(fun.tags) do
+      tags[#tags + 1] = '*' .. t .. '*'
+    end
+  end
+
+  local tag = table.concat(tags, ' ')
+  local siglen = #fun.signature
+  local conceal_offset = 2*(#tags - 1)
+  local tag_pad_len = math.max(1, 80 - #tag + conceal_offset)
+
+  if siglen + #tag > 80 then
+    write(string.rep(' ', tag_pad_len) .. tag)
+    write(fun.signature)
+  else
+    write(string.format('%s%s%s', fun.signature, string.rep(' ', tag_pad_len - siglen), tag))
+  end
+end
+
+--- @param f string
+--- @param fun vim.EvalFn
+--- @param write fun(line: string)
+local function render_eval_doc(f, fun, write)
+  if fun.deprecated then
+    return
+  end
+
+  if not fun.signature then
+    return
+  end
+
+  local desc = fun.desc
+
+  if not desc then
+    write(fun.signature)
+    return
+  end
+
+  local name = fun.name or f
+
+  if rendered_tags[name] then
+    write(fun.signature)
+  else
+    render_sig_and_tag(name, fun, write)
+    rendered_tags[name] = true
+  end
+
+  desc = vim.trim(desc)
+  local desc_l = split(desc)
+  for _, l in ipairs(desc_l) do
+    l = l:gsub('^      ', '')
+    if vim.startswith(l, '<') and not l:match('^<[^ \t]+>') then
+      write('<\t\t' .. l:sub(2))
+    elseif l:match('^>[a-z0-9]*$') then
+      write(l)
+    else
+      write('\t\t' .. l)
+    end
+  end
+
+  if #desc_l > 0 and not desc_l[#desc_l]:match('^<?$') then
+    write('')
+  end
+end
+
+--- @param d vim.option_defaults
+--- @param vimdoc? boolean
+--- @return string
+local function render_option_default(d, vimdoc)
+  local dt --- @type integer|boolean|string|fun(): string
+  if d.if_false ~= nil then
+    dt = d.if_false
+  else
+    dt = d.if_true
+  end
+
+  if vimdoc then
+    if d.doc then
+      return d.doc
+    end
+    if type(dt) == 'boolean' then
+      return dt and 'on' or 'off'
+    end
+  end
+
+  if dt == "" or dt == nil or type(dt) == 'function' then
+    dt = d.meta
+  end
+
+  local v --- @type string
+  if not vimdoc then
+    v = vim.inspect(dt) --[[@as string]]
+  else
+    v = type(dt) == 'string' and '"'..dt..'"' or tostring(dt)
+  end
+
+  --- @type table<string, string|false>
+  local envvars = {
+    TMPDIR = false,
+    VIMRUNTIME = false,
+    XDG_CONFIG_HOME = vim.env.HOME..'/.local/config',
+    XDG_DATA_HOME = vim.env.HOME..'/.local/share',
+    XDG_STATE_HOME = vim.env.HOME..'/.local/state',
+  }
+
+  for name, default in pairs(envvars) do
+    local value = vim.env[name] or default
+    if value then
+      v = v:gsub(vim.pesc(value), '$'..name)
+    end
+  end
+
+  return v
+end
+
+--- @param _f string
+--- @param opt vim.option_meta
+--- @param write fun(line: string)
+local function render_option_meta(_f, opt, write)
+  write('')
+  for _, l in ipairs(split(norm_text(opt.desc))) do
+    write('--- '..l)
+  end
+
+  write('--- @type '..OPTION_TYPES[opt.type])
+  write('vim.o.'..opt.full_name..' = '..render_option_default(opt.defaults))
+  if opt.abbreviation then
+    write('vim.o.'..opt.abbreviation..' = vim.o.'..opt.full_name)
+  end
+
+  for _, s in pairs {
+    {'wo', 'window'},
+    {'bo', 'buffer'},
+    {'go', 'global'},
+  } do
+    local id, scope = s[1], s[2]
+    if vim.list_contains(opt.scope, scope) or (id == 'go' and #opt.scope > 1) then
+      local pfx = 'vim.'..id..'.'
+      write(pfx..opt.full_name..' = vim.o.'..opt.full_name)
+      if opt.abbreviation then
+        write(pfx..opt.abbreviation..' = '..pfx..opt.full_name)
+      end
+    end
+  end
+end
+
+--- @param s string[]
+--- @return string
+local function scope_to_doc(s)
+  local m = {
+    global = 'global',
+    buffer = 'local to buffer',
+    window = 'local to window',
+    tab = 'local to tab page'
+  }
+
+  if #s == 1 then
+    return m[s[1]]
+  end
+  assert(s[1] == 'global')
+  return 'global or '..m[s[2]]..' |global-local|'
+end
+
+-- @param o vim.option_meta
+-- @return string
+local function scope_more_doc(o)
+  if
+    vim.list_contains({
+      'bufhidden',
+      'buftype',
+      'filetype',
+      'modified',
+      'previewwindow',
+      'readonly',
+      'scroll',
+      'syntax',
+      'winfixheight',
+      'winfixwidth',
+    }, o.full_name)
+  then
+    return '  |local-noglobal|'
+  end
+
+  return ''
+end
+
+--- @return table<string,vim.option_meta>
+local function get_option_meta()
+  local opts = require('src/nvim/options').options
+  local optinfo = vim.api.nvim_get_all_options_info()
+  local ret = {} --- @type table<string,vim.option_meta>
+  for _, o in ipairs(opts) do
+    if o.desc then
+      if o.full_name == 'cmdheight' then
+        table.insert(o.scope, 'tab')
+      end
+      local r = vim.deepcopy(o) --[[@as vim.option_meta]]
+      r.desc = o.desc:gsub('^        ', ''):gsub('\n        ', '\n')
+      r.defaults = r.defaults or {}
+      if r.defaults.meta == nil then
+        r.defaults.meta = optinfo[o.full_name].default
+      end
+      ret[o.full_name] = r
+    end
+  end
+  return ret
+end
+
+--- @param opt vim.option_meta
+--- @return string[]
+local function build_option_tags(opt)
+  --- @type string[]
+  local tags = { opt.full_name }
+
+  tags[#tags+1] = opt.abbreviation
+  if opt.type == 'bool' then
+    for i = 1, #tags do
+      tags[#tags+1] = 'no'..tags[i]
+    end
+  end
+
+  for i, t in ipairs(tags) do
+    tags[i] = "'"..t.."'"
+  end
+
+  for _, t in ipairs(opt.tags or {}) do
+    tags[#tags+1] = t
+  end
+
+  for i, t in ipairs(tags) do
+    tags[i] = "*"..t.."*"
+  end
+
+  return tags
+end
+
+--- @param _f string
+--- @param opt vim.option_meta
+--- @param write fun(line: string)
+local function render_option_doc(_f, opt, write)
+  local tags = build_option_tags(opt)
+  local tag_str = table.concat(tags, ' ')
+  local conceal_offset = 2*(#tags - 1)
+  local tag_pad = string.rep('\t', math.ceil((64 - #tag_str + conceal_offset) / 8))
+  -- local pad = string.rep(' ', 80 - #tag_str + conceal_offset)
+  write(tag_pad..tag_str)
+
+  local name_str --- @type string
+  if opt.abbreviation then
+    name_str = string.format("'%s' '%s'", opt.full_name, opt.abbreviation)
+  else
+    name_str = string.format("'%s'", opt.full_name)
+  end
+
+  local otype = opt.type == 'bool' and 'boolean' or opt.type
+  if opt.defaults.doc or opt.defaults.if_true ~= nil or opt.defaults.meta ~= nil then
+    local v = render_option_default(opt.defaults, true)
+    local pad = string.rep('\t', math.max(1, math.ceil((24 - #name_str) / 8)))
+    if opt.defaults.doc then
+      local deflen = #string.format('%s%s%s (', name_str, pad, otype)
+      --- @type string
+      v = v:gsub('\n', '\n'..string.rep(' ', deflen - 2))
+    end
+    write(string.format('%s%s%s\t(default %s)', name_str, pad, otype, v))
+  else
+    write(string.format('%s\t%s', name_str, otype))
+  end
+
+  write('\t\t\t'..scope_to_doc(opt.scope)..scope_more_doc(opt))
+  for _, l in ipairs(split(opt.desc)) do
+    if l == '<' or l:match('^<%s') then
+      write(l)
+    else
+      write('\t'..l:gsub('\\<', '<'))
+    end
+  end
+end
+
+--- @class nvim.gen_eval_files.elem
+--- @field path string
+--- @field from? string Skip lines in path until this pattern is reached.
+--- @field funcs fun(): table<string, table>
+--- @field render fun(f:string,obj:table,write:fun(line:string))
+--- @field header? string[]
+--- @field footer? string[]
+
+--- @type nvim.gen_eval_files.elem[]
+local CONFIG = {
+  {
+    path = 'runtime/lua/vim/_meta/vimfn.lua',
+    header = LUA_META_HEADER,
+    funcs = get_eval_meta,
+    render = render_eval_meta,
+  },
+  {
+    path = 'runtime/lua/vim/_meta/api.lua',
+    header = LUA_API_META_HEADER,
+    funcs = get_api_meta,
+    render = render_api_meta,
+  },
+  {
+    path = 'runtime/lua/vim/_meta/api_keysets.lua',
+    header = LUA_META_HEADER,
+    funcs = get_api_keysets_meta,
+    render = render_api_keyset_meta,
+  },
+  {
+    path = 'runtime/doc/builtin.txt',
+    funcs = get_eval_meta,
+    render = render_eval_doc,
+    header = {
+      '*builtin.txt*	Nvim',
+      '',
+      '',
+      '\t\t  NVIM REFERENCE MANUAL',
+      '',
+      '',
+      'Builtin functions\t\t*vimscript-functions* *builtin-functions*',
+      '',
+      'For functions grouped by what they are used for see |function-list|.',
+      '',
+      '\t\t\t\t      Type |gO| to see the table of contents.',
+      '==============================================================================',
+      '1. Details					*builtin-function-details*',
+      '',
+    },
+    footer = {
+      '==============================================================================',
+      '2. Matching a pattern in a String			*string-match*',
+      '',
+      'This is common between several functions. A regexp pattern as explained at',
+      '|pattern| is normally used to find a match in the buffer lines.  When a',
+      'pattern is used to find a match in a String, almost everything works in the',
+      'same way.  The difference is that a String is handled like it is one line.',
+      'When it contains a "\\n" character, this is not seen as a line break for the',
+      'pattern.  It can be matched with a "\\n" in the pattern, or with ".".  Example:',
+      '>vim',
+      '\tlet a = "aaaa\\nxxxx"',
+      '\techo matchstr(a, "..\\n..")',
+      '\t" aa',
+      '\t" xx',
+      '\techo matchstr(a, "a.x")',
+      '\t" a',
+      '\t" x',
+      '',
+      'Don\'t forget that "^" will only match at the first character of the String and',
+      '"$" at the last character of the string.  They don\'t match after or before a',
+      '"\\n".',
+      '',
+      ' vim:tw=78:ts=8:noet:ft=help:norl:',
+    },
+  },
+  {
+    path = 'runtime/lua/vim/_meta/options.lua',
+    header = LUA_OPTION_META_HEADER,
+    funcs = get_option_meta,
+    render = render_option_meta,
+  },
+  {
+    path = 'runtime/doc/options.txt',
+    header = { '' },
+    from = 'A jump table for the options with a short description can be found at |Q_op|.',
+    footer = {
+      ' vim:tw=78:ts=8:noet:ft=help:norl:'
+    },
+    funcs = get_option_meta,
+    render = render_option_doc,
+  }
+}
+
+--- @param elem nvim.gen_eval_files.elem
+local function render(elem)
+  print('Rendering '..elem.path)
+  local from_lines = {}  --- @type string[]
+  local from = elem.from
+  if from then
+    for line in io.lines(elem.path) do
+      from_lines[#from_lines+1] = line
+      if line:match(from) then
+        break
+      end
+    end
+  end
+
+  local o = assert(io.open(elem.path, 'w'))
+
+  --- @param l string
+  local function write(l)
+    local l1 = l:gsub('%s+$', '')
+    o:write(l1)
+    o:write('\n')
+  end
+
+  for _, l in ipairs(from_lines) do
+    write(l)
+  end
+
+  for _, l in ipairs(elem.header or {}) do
+    write(l)
+  end
+
+  local funcs = elem.funcs()
+
+  --- @type string[]
+  local fnames = vim.tbl_keys(funcs)
+  table.sort(fnames)
+
+  for _, f in ipairs(fnames) do
+    elem.render(f, funcs[f], write)
+  end
+
+  for _, l in ipairs(elem.footer or {}) do
+    write(l)
+  end
+
+  o:close()
+end
+
+local function main()
+  for _, c in ipairs(CONFIG) do
+    render(c)
+  end
+end
+
+main()
diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua
index 2563f2f410..633207e018 100644
--- a/scripts/gen_help_html.lua
+++ b/scripts/gen_help_html.lua
@@ -34,6 +34,8 @@ local spell_dict = {
   neovim = 'Nvim',
   lua = 'Lua',
   VimL = 'Vimscript',
+  vimL = 'Vimscript',
+  viml = 'Vimscript',
 }
 local language = nil
 
@@ -43,6 +45,7 @@ local M = {}
 -- All other files are "legacy" files which require fixed-width layout.
 local new_layout = {
   ['api.txt'] = true,
+  ['lsp.txt'] = true,
   ['channel.txt'] = true,
   ['deprecated.txt'] = true,
   ['develop.txt'] = true,
@@ -57,21 +60,10 @@ local new_layout = {
 
 -- TODO: These known invalid |links| require an update to the relevant docs.
 local exclude_invalid = {
-  ["'previewpopup'"] = "quickref.txt",
-  ["'pvp'"] = "quickref.txt",
   ["'string'"] = "eval.txt",
   Query = 'treesitter.txt',
-  ['eq?'] = 'treesitter.txt',
-  ['lsp-request'] = 'lsp.txt',
   matchit = 'vim_diff.txt',
-  ['matchit.txt'] = 'help.txt',
   ["set!"] = "treesitter.txt",
-  ['v:_null_blob'] = 'builtin.txt',
-  ['v:_null_dict'] = 'builtin.txt',
-  ['v:_null_list'] = 'builtin.txt',
-  ['v:_null_string'] = 'builtin.txt',
-  ['vim.lsp.buf_request()'] = 'lsp.txt',
-  ['vim.lsp.util.get_progress_messages()'] = 'lsp.txt',
 }
 
 -- False-positive "invalid URLs".
@@ -90,6 +82,11 @@ local exclude_invalid_urls = {
   ["http://www.jclark.com/"] = "quickfix.txt",
 }
 
+-- Deprecated, brain-damaged files that I don't care about.
+local ignore_errors = {
+  ['pi_netrw.txt'] = true,
+}
+
 local function tofile(fname, text)
   local f = io.open(fname, 'w')
   if not f then
@@ -145,11 +142,11 @@ local function trim(s, dir)
   return vim.fn.trim(s, '\r\t\n ', dir or 0)
 end
 
--- Remove common punctuation from URLs.
---
--- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc
---
--- @returns (fixed_url, removed_chars) where `removed_chars` is in the order found in the input.
+--- Removes common punctuation from URLs.
+---
+--- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc
+---
+--- @returns (fixed_url, removed_chars) where `removed_chars` is in the order found in the input.
 local function fix_url(url)
   local removed_chars = ''
   local fixed_url = url
@@ -163,7 +160,7 @@ local function fix_url(url)
   return fixed_url, removed_chars
 end
 
--- Checks if a given line is a "noise" line that doesn't look good in HTML form.
+--- Checks if a given line is a "noise" line that doesn't look good in HTML form.
 local function is_noise(line, noise_lines)
   if (
     -- First line is always noise.
@@ -188,7 +185,7 @@ local function is_noise(line, noise_lines)
   return false
 end
 
--- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content.
+--- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content.
 local function get_bug_url_vimdoc(fname, to_fname, sample_text)
   local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname))
   local bug_url = ('https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+'
@@ -201,7 +198,7 @@ local function get_bug_url_vimdoc(fname, to_fname, sample_text)
   return bug_url
 end
 
--- Creates a github issue URL at neovim/neovim with prefilled content.
+--- Creates a github issue URL at neovim/neovim with prefilled content.
 local function get_bug_url_nvim(fname, to_fname, sample_text, token_name)
   local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname))
   local bug_url = ('https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+'
@@ -216,7 +213,7 @@ local function get_bug_url_nvim(fname, to_fname, sample_text, token_name)
   return bug_url
 end
 
--- Gets a "foo.html" name from a "foo.txt" helpfile name.
+--- Gets a "foo.html" name from a "foo.txt" helpfile name.
 local function get_helppage(f)
   if not f then
     return nil
@@ -231,9 +228,9 @@ local function get_helppage(f)
   return (f:gsub('%.txt$', '.html'))
 end
 
--- Counts leading spaces (tab=8) to decide the indent size of multiline text.
---
--- Blank lines (empty or whitespace-only) are ignored.
+--- Counts leading spaces (tab=8) to decide the indent size of multiline text.
+---
+--- Blank lines (empty or whitespace-only) are ignored.
 local function get_indent(s)
   local min_indent = nil
   for line in vim.gsplit(s, '\n') do
@@ -245,7 +242,7 @@ local function get_indent(s)
   return min_indent or 0
 end
 
--- Removes the common indent level, after expanding tabs to 8 spaces.
+--- Removes the common indent level, after expanding tabs to 8 spaces.
 local function trim_indent(s)
   local indent_size = get_indent(s)
   local trimmed = ''
@@ -256,7 +253,7 @@ local function trim_indent(s)
   return trimmed:sub(1, -2)
 end
 
--- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string.
+--- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string.
 local function getbuflinestr(node, bufnr, offset)
   local line1, _, line2, _ = node:range()
   line1 = line1 - offset
@@ -265,8 +262,8 @@ local function getbuflinestr(node, bufnr, offset)
   return table.concat(lines, '\n')
 end
 
--- Gets the whitespace just before `node` from the raw buffer text.
--- Needed for preformatted `old` lines.
+--- Gets the whitespace just before `node` from the raw buffer text.
+--- Needed for preformatted `old` lines.
 local function getws(node, bufnr)
   local line1, c1, line2, _ = node:range()
   local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1]
@@ -283,18 +280,21 @@ local function get_tagname(node, bufnr)
   return helppage, tag
 end
 
--- Returns true if the given invalid tagname is a false positive.
+--- Returns true if the given invalid tagname is a false positive.
 local function ignore_invalid(s)
   return not not (
     exclude_invalid[s]
     -- Strings like |~/====| appear in various places and the parser thinks they are links, but they
     -- are just table borders.
     or s:find('===')
-    or s:find('---')
+    or s:find('%-%-%-')
   )
 end
 
-local function ignore_parse_error(s)
+local function ignore_parse_error(fname, s)
+  if ignore_errors[vim.fs.basename(fname)] then
+    return true
+  end
   return (
     -- Ignore parse errors for unclosed tag.
     -- This is common in vimdocs and is treated as plaintext by :help.
@@ -315,7 +315,7 @@ local function has_ancestor(node, ancestor_name)
   return false
 end
 
--- Gets the first matching child node matching `name`.
+--- Gets the first matching child node matching `name`.
 local function first(node, name)
   for c, _ in node:iter_children() do
     if c:named() and c:type() == name then
@@ -337,10 +337,10 @@ local function validate_link(node, bufnr, fname)
   return helppage, tagname, ignored
 end
 
--- TODO: port the logic from scripts/check_urls.vim
+--- TODO: port the logic from scripts/check_urls.vim
 local function validate_url(text, fname)
   local ignored = false
-  if vim.fs.basename(fname) == 'pi_netrw.txt' then
+  if ignore_errors[vim.fs.basename(fname)] then
     ignored = true
   elseif text:find('http%:') and not exclude_invalid_urls[text] then
     invalid_urls[text] = vim.fs.basename(fname)
@@ -348,10 +348,12 @@ local function validate_url(text, fname)
   return ignored
 end
 
--- Traverses the tree at `root` and checks that |tag| links point to valid helptags.
+--- Traverses the tree at `root` and checks that |tag| links point to valid helptags.
 local function visit_validate(root, level, lang_tree, opt, stats)
   level = level or 0
   local node_name = (root.named and root:named()) and root:type() or nil
+  -- Parent kind (string).
+  local parent = root:parent() and root:parent():type() or nil
   local toplevel = level < 1
   local function node_text(node)
     return vim.treesitter.get_node_text(node or root, opt.buf)
@@ -367,19 +369,21 @@ local function visit_validate(root, level, lang_tree, opt, stats)
   end
 
   if node_name == 'ERROR' then
-    if ignore_parse_error(text) then
+    if ignore_parse_error(opt.fname, text) then
       return
     end
     -- Store the raw text to give context to the error report.
-    local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]'
+    local sample_text = not toplevel and getbuflinestr(root, opt.buf, 0) or '[top level!]'
+    -- Flatten the sample text to a single, truncated line.
+    sample_text = vim.trim(sample_text):gsub('[\t\n]', ' '):sub(1, 80)
     table.insert(stats.parse_errors, sample_text)
-  elseif node_name == 'word' or node_name == 'uppercase_name' then
-    if spell_dict[text] then
-      if not invalid_spelling[text] then
-        invalid_spelling[text] = { vim.fs.basename(opt.fname) }
-      else
-        table.insert(invalid_spelling[text], vim.fs.basename(opt.fname))
-      end
+  elseif (node_name == 'word' or node_name == 'uppercase_name')
+    and (not vim.tbl_contains({'codespan', 'taglink', 'tag'}, parent))
+  then
+    local text_nopunct = vim.fn.trim(text, '.,', 0)  -- Ignore some punctuation.
+    if spell_dict[text_nopunct] then
+      invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {}
+      invalid_spelling[text_nopunct][vim.fs.basename(opt.fname)] = node_text(root:parent())
     end
   elseif node_name == 'url' then
     local fixed_url, _ = fix_url(trim(text))
@@ -461,6 +465,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', ''))
     -- Use the first *tag* node as the heading anchor, if any.
     local tagnode = first(root, 'tag')
+    -- Use the *tag* as the heading anchor id, if possible.
     local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname)
     if node_name == 'h1' or #headings == 0 then
       table.insert(headings, { name = hname, subheadings = {}, tag = tagname })
@@ -468,9 +473,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, tag = tagname })
     end
     local el = node_name == 'h1' and 'h2' or 'h3'
-    -- If we are re-using the *tag*, this heading anchor is redundant.
-    local a = tagnode and '' or ('<a name="%s"></a>'):format(tagname)
-    return ('%s<%s class="help-heading">%s</%s>\n'):format(a, el, text, el)
+    return ('<%s id="%s" class="help-heading">%s</%s>\n'):format(el, tagname, text, el)
   elseif node_name == 'column_heading' or node_name == 'column_name' then
     if root:has_error() then
       return text
@@ -491,7 +494,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       return ''  -- Discard common "noise" lines.
     end
     -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout.
-    local div = opt.old and root:child(0) and vim.tbl_contains({'column_heading', 'h1', 'h2', 'h3'}, root:child(0):type())
+    local div = opt.old and root:child(0) and vim.list_contains({'column_heading', 'h1', 'h2', 'h3'}, root:child(0):type())
     return string.format('%s%s', div and trim(text) or text, div and '' or '\n')
   elseif node_name == 'line_li' then
     local sib = root:prev_sibling()
@@ -522,7 +525,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       s = fix_tab_after_conceal(s, node_text(root:next_sibling()))
     end
     return s
-  elseif vim.tbl_contains({'codespan', 'keycode'}, node_name) then
+  elseif vim.list_contains({'codespan', 'keycode'}, node_name) then
     if root:has_error() then
       return text
     end
@@ -538,7 +541,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
   elseif node_name == 'language' then
     language = node_text(root)
     return ''
-  elseif node_name == 'code' then
+  elseif node_name == 'code' then  -- Highlighted codeblock (child).
     if is_blank(text) then
       return ''
     end
@@ -554,7 +557,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     if root:has_error() then
       return text
     end
-    local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent)
+    local in_heading = vim.list_contains({'h1', 'h2', 'h3'}, parent)
     local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag'
     local tagname = node_text(root:child(1), false)
     if vim.tbl_count(stats.first_tags) < 2 then
@@ -563,12 +566,15 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
       return ''
     end
     local el = in_heading and 'span' or 'code'
-    local s = ('%s<a name="%s"></a><%s class="%s">%s</%s>'):format(ws(), url_encode(tagname), el, cssclass, trimmed, el)
+    local encoded_tagname = url_encode(tagname)
+    local s = ('%s<%s id="%s" class="%s"><a href="#%s">%s</a></%s>'):format(ws(), el, encoded_tagname, cssclass, encoded_tagname, trimmed, el)
     if opt.old then
         s = fix_tab_after_conceal(s, node_text(root:next_sibling()))
     end
 
     if in_heading and prev ~= 'tag' then
+      -- Don't set "id", let the heading use the tag as its "id" (used by search engines).
+      s = ('%s<%s class="%s"><a href="#%s">%s</a></%s>'):format(ws(), el, cssclass, encoded_tagname, trimmed, el)
       -- Start the <span> container for tags in a heading.
       -- This makes "justify-content:space-between" right-align the tags.
       --    <h2>foo bar<span>tag1 tag2</span></h2>
@@ -579,7 +585,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats)
     end
     return s
   elseif node_name == 'ERROR' then
-    if ignore_parse_error(trimmed) then
+    if ignore_parse_error(opt.fname, trimmed) then
       return text
     end
 
@@ -601,7 +607,7 @@ local function get_helpfiles(include)
   for f, type in vim.fs.dir(dir) do
     if (vim.endswith(f, '.txt')
         and type == 'file'
-        and (not include or vim.tbl_contains(include, f))) then
+        and (not include or vim.list_contains(include, f))) then
       local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p')
       table.insert(rv, fullpath)
     end
@@ -609,7 +615,7 @@ local function get_helpfiles(include)
   return rv
 end
 
--- Populates the helptags map.
+--- Populates the helptags map.
 local function get_helptags(help_dir)
   local m = {}
   -- Load a random help file to convince taglist() to do its job.
@@ -624,17 +630,19 @@ local function get_helptags(help_dir)
   return m
 end
 
--- Use the help.so parser defined in the build, not whatever happens to be installed on the system.
+--- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system.
 local function ensure_runtimepath()
   if not vim.o.runtimepath:find('build/lib/nvim/') then
     vim.cmd[[set runtimepath^=./build/lib/nvim/]]
   end
 end
 
--- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents.
---
--- @returns lang_tree, bufnr
-local function parse_buf(fname)
+--- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents.
+---
+--- @param fname string help file to parse
+--- @param parser_path string? path to non-default vimdoc.so
+--- @returns lang_tree, bufnr
+local function parse_buf(fname, parser_path)
   local buf
   if type(fname) == 'string' then
     vim.cmd('split '..vim.fn.fnameescape(fname))  -- Filename.
@@ -643,21 +651,25 @@ local function parse_buf(fname)
     buf = fname
     vim.cmd('sbuffer '..tostring(fname))          -- Buffer number.
   end
-  -- vim.treesitter.require_language('help', './build/lib/nvim/parser/help.so')
-  local lang_tree = vim.treesitter.get_parser(buf, 'help')
+  if parser_path then
+    vim.treesitter.language.add('vimdoc', { path = parser_path })
+  end
+  local lang_tree = vim.treesitter.get_parser(buf)
   return lang_tree, buf
 end
 
--- Validates one :help file `fname`:
---  - checks that |tag| links point to valid helptags.
---  - recursively counts parse errors ("ERROR" nodes)
---
--- @returns { invalid_links: number, parse_errors: number }
-local function validate_one(fname)
+--- Validates one :help file `fname`:
+---  - checks that |tag| links point to valid helptags.
+---  - recursively counts parse errors ("ERROR" nodes)
+---
+--- @param fname string help file to validate
+--- @param parser_path string? path to non-default vimdoc.so
+--- @returns { invalid_links: number, parse_errors: string[] }
+local function validate_one(fname, parser_path)
   local stats = {
     parse_errors = {},
   }
-  local lang_tree, buf = parse_buf(fname)
+  local lang_tree, buf = parse_buf(fname, parser_path)
   for _, tree in ipairs(lang_tree:trees()) do
     visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname, }, stats)
   end
@@ -666,20 +678,21 @@ local function validate_one(fname)
   return stats
 end
 
--- Generates HTML from one :help file `fname` and writes the result to `to_fname`.
---
--- @param fname Source :help file
--- @param to_fname Destination .html file
--- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace)
---
--- @returns html, stats
-local function gen_one(fname, to_fname, old, commit)
+--- Generates HTML from one :help file `fname` and writes the result to `to_fname`.
+---
+--- @param fname string Source :help file
+--- @param to_fname string Destination .html file
+--- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace)
+--- @param parser_path string? path to non-default vimdoc.so
+---
+--- @returns html, stats
+local function gen_one(fname, to_fname, old, commit, parser_path)
   local stats = {
     noise_lines = {},
     parse_errors = {},
     first_tags = {},  -- Track the first few tags in doc.
   }
-  local lang_tree, buf = parse_buf(fname)
+  local lang_tree, buf = parse_buf(fname, parser_path)
   local headings = {}  -- Headings (for ToC). 2-dimensional: h1 contains h2/h3.
   local title = to_titlecase(basename_noext(fname))
 
@@ -691,11 +704,17 @@ local function gen_one(fname, to_fname, old, commit)
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta name="description" content="Neovim user documentation">
+
+    <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ -->
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@docsearch/css@3" />
+    <link rel="preconnect" href="https://X185E15FPG-dsn.algolia.net" crossorigin />
+
     <link href="/css/normalize.min.css" rel="stylesheet">
     <link href="/css/bootstrap.css" rel="stylesheet">
     <link href="/css/main.css" rel="stylesheet">
     <link href="help.css" rel="stylesheet">
     <link href="/highlight/styles/neovim.min.css" rel="stylesheet">
+
     <script src="/highlight/highlight.min.js"></script>
     <script>hljs.highlightAll();</script>
     <title>%s - Neovim docs</title>
@@ -766,19 +785,20 @@ local function gen_one(fname, to_fname, old, commit)
   main = ([[
   <header class="container">
     <nav class="navbar navbar-expand-lg">
-      <div>
+      <div class="container-fluid">
         <a href="/" class="navbar-brand" aria-label="logo">
           <!--TODO: use <img src="….svg"> here instead. Need one that has green lettering instead of gray. -->
           %s
           <!--<img src="https://neovim.io/logos/neovim-logo.svg" width="173" height="50" alt="Neovim" />-->
         </a>
+        <div id="docsearch"></div> <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ -->
       </div>
     </nav>
   </header>
 
   <div class="container golden-grid help-body">
   <div class="col-wide">
-  <a name="%s"></a><a name="%s"></a><h1>%s</h1>
+  <a name="%s"></a><h1 id="%s">%s</h1>
   <p>
     <i>
     Nvim <code>:help</code> pages, <a href="https://github.com/neovim/neovim/blob/master/scripts/gen_help_html.lua">generated</a>
@@ -789,7 +809,7 @@ local function gen_one(fname, to_fname, old, commit)
   <hr/>
   %s
   </div>
-  ]]):format(logo_svg, stats.first_tags[1] or '', stats.first_tags[2] or '', title, vim.fs.basename(fname), main)
+  ]]):format(logo_svg, stats.first_tags[2] or '', stats.first_tags[1] or '', title, vim.fs.basename(fname), main)
 
   local toc = [[
     <div class="col-narrow toc">
@@ -825,6 +845,18 @@ local function gen_one(fname, to_fname, old, commit)
       parse_errors: %d %s | <span title="%s">noise_lines: %d</span>
       </div>
     <div>
+
+    <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ -->
+    <script src="https://cdn.jsdelivr.net/npm/@docsearch/js@3"></script>
+    <script type="module">
+      docsearch({
+        container: '#docsearch',
+        appId: 'X185E15FPG',
+        apiKey: 'b5e6b2f9c636b2b471303205e59832ed',
+        indexName: 'nvim',
+      });
+    </script>
+
   </footer>
   ]]):format(
     os.date('%Y-%m-%d %H:%M'), commit, commit:sub(1, 7), #stats.parse_errors, bug_link,
@@ -908,6 +940,7 @@ local function gen_css(fname)
       padding-top: 10px;
       padding-bottom: 10px;
     }
+
     .old-help-para {
       padding-top: 10px;
       padding-bottom: 10px;
@@ -917,6 +950,13 @@ local function gen_css(fname)
       font-size: 16px;
       font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace;
     }
+    .old-help-para pre {
+      /* All text in .old-help-para is formatted as "white-space:pre" so text following <pre> is
+         already visually separated by the linebreak. */
+      margin-bottom: 0;
+    }
+
+    /* TODO: should this rule be deleted? help tags are rendered as <code> or <span>, not <a> */
     a.help-tag, a.help-tag:focus, a.help-tag:hover {
       color: inherit;
       text-decoration: none;
@@ -927,6 +967,17 @@ local function gen_css(fname)
     /* Tag pseudo-header common in :help docs. */
     .help-tag-right {
       color: var(--tag-color);
+      margin-left: auto;
+      margin-right: 0;
+      float: right;
+    }
+    .help-tag a,
+    .help-tag-right a {
+      color: inherit;
+    }
+    .help-tag a:not(:hover),
+    .help-tag-right a:not(:hover) {
+      text-decoration: none;
     }
     h1 .help-tag, h2 .help-tag, h3 .help-tag {
       font-size: smaller;
@@ -967,6 +1018,9 @@ local function gen_css(fname)
       font-size: 16px;
       margin-top: 10px;
     }
+    pre:last-child {
+      margin-bottom: 0;
+    }
     pre:hover,
     .help-heading:hover {
       overflow: visible;
@@ -1038,18 +1092,21 @@ end
 --- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'}
 ---
 --- @returns info dict
-function M.gen(help_dir, to_dir, include, commit)
+function M.gen(help_dir, to_dir, include, commit, parser_path)
   vim.validate{
-    help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'},
+    help_dir={help_dir, function(d) return vim.fn.isdirectory(vim.fn.expand(d)) == 1 end, 'valid directory'},
     to_dir={to_dir, 's'},
     include={include, 't', true},
     commit={commit, 's', true},
+    parser_path={parser_path, function(f) return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 end, 'valid vimdoc.{so,dll} filepath'},
   }
 
   local err_count = 0
   ensure_runtimepath()
-  tagmap = get_helptags(help_dir)
+  tagmap = get_helptags(vim.fn.expand(help_dir))
   helpfiles = get_helpfiles(include)
+  to_dir = vim.fn.expand(to_dir)
+  parser_path = parser_path and vim.fn.expand(parser_path) or nil
 
   print(('output dir: %s'):format(to_dir))
   vim.fn.mkdir(to_dir, 'p')
@@ -1058,7 +1115,7 @@ function M.gen(help_dir, to_dir, include, commit)
   for _, f in ipairs(helpfiles) do
     local helpfile = vim.fs.basename(f)
     local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile))
-    local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?')
+    local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?', parser_path)
     tofile(to_fname, html)
     print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname)))
     err_count = err_count + #stats.parse_errors
@@ -1081,20 +1138,27 @@ end
 -- This is 10x faster than gen(), for use in CI.
 --
 -- @returns results dict
-function M.validate(help_dir, include)
+function M.validate(help_dir, include, parser_path)
   vim.validate{
-    help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'},
+    help_dir={help_dir, function(d) return vim.fn.isdirectory(vim.fn.expand(d)) == 1 end, 'valid directory'},
     include={include, 't', true},
+    parser_path={parser_path, function(f) return f == nil or vim.fn.filereadable(vim.fn.expand(f)) == 1 end, 'valid vimdoc.{so,dll} filepath'},
   }
   local err_count = 0
+  local files_to_errors = {}
   ensure_runtimepath()
-  tagmap = get_helptags(help_dir)
+  tagmap = get_helptags(vim.fn.expand(help_dir))
   helpfiles = get_helpfiles(include)
+  parser_path = parser_path and vim.fn.expand(parser_path) or nil
 
   for _, f in ipairs(helpfiles) do
     local helpfile = vim.fs.basename(f)
-    local rv = validate_one(f)
+    local rv = validate_one(f, parser_path)
     print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile))
+    if #rv.parse_errors > 0 then
+      files_to_errors[helpfile] = rv.parse_errors
+      vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v) return s..'\n    '..v end)))
+    end
     err_count = err_count + #rv.parse_errors
   end
 
@@ -1104,6 +1168,7 @@ function M.validate(help_dir, include)
     invalid_links = invalid_links,
     invalid_urls = invalid_urls,
     invalid_spelling = invalid_spelling,
+    parse_errors = files_to_errors,
   }
 end
 
diff --git a/scripts/gen_lsp.lua b/scripts/gen_lsp.lua
new file mode 100644
index 0000000000..6ff8dcb3f4
--- /dev/null
+++ b/scripts/gen_lsp.lua
@@ -0,0 +1,290 @@
+--[[
+Generates lua-ls annotations for lsp
+USAGE:
+nvim -l scripts/gen_lsp.lua gen  # this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua
+nvim -l scripts/gen_lsp.lua gen --version 3.18 --build/new_lsp_types.lua
+nvim -l scripts/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua
+nvim -l scripts/gen_lsp.lua gen --version 3.18 --methods
+--]]
+
+local M = {}
+
+local function tofile(fname, text)
+  local f = io.open(fname, 'w')
+  if not f then
+    error(('failed to write: %s'):format(f))
+  else
+    f:write(text)
+    f:close()
+  end
+end
+
+local function read_json(opt)
+  local uri = 'https://raw.githubusercontent.com/microsoft/language-server-protocol/gh-pages/_specifications/lsp/'
+    .. opt.version
+    .. '/metaModel/metaModel.json'
+
+  local res = vim.system({ 'curl', '--no-progress-meter', uri, '-o', '-' }):wait()
+  if res.code ~= 0 or (res.stdout or ''):len() < 999 then
+    print(('URL failed: %s'):format(uri))
+    vim.print(res)
+    error(res.stdout)
+  end
+  return vim.json.decode(res.stdout)
+end
+
+-- Gets the Lua symbol for a given fully-qualified LSP method name.
+local function name(s)
+  -- "$/" prefix is special: https://microsoft.github.io/language-server-protocol/specification/#dollarRequests
+  return s:gsub('^%$', 'dollar'):gsub('/', '_')
+end
+
+local function gen_methods(protocol)
+  local output = {
+    '-- Generated by gen_lsp.lua, keep at end of file.',
+    '--- LSP method names.',
+    '---',
+    '---@see https://microsoft.github.io/language-server-protocol/specification/#metaModel',
+    'protocol.Methods = {',
+  }
+  local indent = (' '):rep(2)
+
+  local all = vim.list_extend(protocol.requests, protocol.notifications)
+  table.sort(all, function(a, b)
+    return name(a.method) < name(b.method)
+  end)
+  for _, item in ipairs(all) do
+    if item.method then
+      if item.documentation then
+        local document = vim.split(item.documentation, '\n?\n', { trimempty = true })
+        for _, docstring in ipairs(document) do
+          output[#output + 1] = indent .. '--- ' .. docstring
+        end
+      end
+      output[#output + 1] = ("%s%s = '%s',"):format(indent, name(item.method), item.method)
+    end
+  end
+  output[#output + 1] = '}'
+  output = vim.list_extend(
+    output,
+    vim.split(
+      [[
+local function freeze(t)
+  return setmetatable({}, {
+    __index = t,
+    __newindex = function()
+      error('cannot modify immutable table')
+    end,
+  })
+end
+protocol.Methods = freeze(protocol.Methods)
+
+return protocol
+]],
+      '\n',
+      { trimempty = true }
+    )
+  )
+
+  local fname = './runtime/lua/vim/lsp/protocol.lua'
+  local bufnr = vim.fn.bufadd(fname)
+  vim.fn.bufload(bufnr)
+  vim.api.nvim_set_current_buf(bufnr)
+  local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false)
+  local index = vim.iter(ipairs(lines)):find(function(key, item)
+    return vim.startswith(item, '-- Generated by') and key or nil
+  end)
+  index = index and index - 1 or vim.api.nvim_buf_line_count(bufnr) - 1
+  vim.api.nvim_buf_set_lines(bufnr, index, -1, true, output)
+  vim.cmd.write()
+end
+
+function M.gen(opt)
+  local protocol = read_json(opt)
+
+  if opt.methods then
+    gen_methods(protocol)
+  end
+
+  local output = {
+    '--[[',
+    'This file is autogenerated from scripts/gen_lsp.lua',
+    'Regenerate:',
+    [=[nvim -l scripts/gen_lsp.lua gen --version 3.18 --runtime/lua/vim/lsp/_meta/protocol.lua]=],
+    '--]]',
+    '',
+    '---@alias lsp.null nil',
+    '---@alias uinteger integer',
+    '---@alias lsp.decimal number',
+    '---@alias lsp.DocumentUri string',
+    '---@alias lsp.URI string',
+    '---@alias lsp.LSPObject table<string, lsp.LSPAny>',
+    '---@alias lsp.LSPArray lsp.LSPAny[]',
+    '---@alias lsp.LSPAny lsp.LSPObject|lsp.LSPArray|string|number|boolean|nil',
+    '',
+  }
+
+  local anonymous_num = 0
+
+  local anonym_classes = {}
+
+  local simple_types = {
+    'string',
+    'boolean',
+    'integer',
+    'uinteger',
+    'decimal',
+  }
+
+  local function parse_type(type)
+    if type.kind == 'reference' or type.kind == 'base' then
+      if vim.tbl_contains(simple_types, type.name) then
+        return type.name
+      end
+      return 'lsp.' .. type.name
+    elseif type.kind == 'array' then
+      return parse_type(type.element) .. '[]'
+    elseif type.kind == 'or' then
+      local val = ''
+      for _, item in ipairs(type.items) do
+        val = val .. parse_type(item) .. '|'
+      end
+      val = val:sub(0, -2)
+      return val
+    elseif type.kind == 'stringLiteral' then
+      return '"' .. type.value .. '"'
+    elseif type.kind == 'map' then
+      return 'table<' .. parse_type(type.key) .. ', ' .. parse_type(type.value) .. '>'
+    elseif type.kind == 'literal' then
+      -- can I use ---@param disabled? {reason: string}
+      -- use | to continue the inline class to be able to add docs
+      -- https://github.com/LuaLS/lua-language-server/issues/2128
+      anonymous_num = anonymous_num + 1
+      local anonym = { '---@class anonym' .. anonymous_num }
+      for _, field in ipairs(type.value.properties) do
+        if field.documentation then
+          field.documentation = field.documentation:gsub('\n', '\n---')
+          anonym[#anonym + 1] = '---' .. field.documentation
+        end
+        anonym[#anonym + 1] = '---@field '
+          .. field.name
+          .. (field.optional and '?' or '')
+          .. ' '
+          .. parse_type(field.type)
+      end
+      anonym[#anonym + 1] = ''
+      for _, line in ipairs(anonym) do
+        anonym_classes[#anonym_classes + 1] = line
+      end
+      return 'anonym' .. anonymous_num
+    elseif type.kind == 'tuple' then
+      local tuple = '{ '
+      for i, value in ipairs(type.items) do
+        tuple = tuple .. '[' .. i .. ']: ' .. parse_type(value) .. ', '
+      end
+      -- remove , at the end
+      tuple = tuple:sub(0, -3)
+      return tuple .. ' }'
+    end
+    vim.print(type)
+    return ''
+  end
+
+  for _, structure in ipairs(protocol.structures) do
+    if structure.documentation then
+      structure.documentation = structure.documentation:gsub('\n', '\n---')
+      output[#output + 1] = '---' .. structure.documentation
+    end
+    if structure.extends then
+      local class_string = '---@class lsp.'
+        .. structure.name
+        .. ': '
+        .. parse_type(structure.extends[1])
+      for _, mixin in ipairs(structure.mixins or {}) do
+        class_string = class_string .. ', ' .. parse_type(mixin)
+      end
+      output[#output + 1] = class_string
+    else
+      output[#output + 1] = '---@class lsp.' .. structure.name
+    end
+    for _, field in ipairs(structure.properties or {}) do
+      if field.documentation then
+        field.documentation = field.documentation:gsub('\n', '\n---')
+        output[#output + 1] = '---' .. field.documentation
+      end
+      output[#output + 1] = '---@field '
+        .. field.name
+        .. (field.optional and '?' or '')
+        .. ' '
+        .. parse_type(field.type)
+    end
+    output[#output + 1] = ''
+  end
+
+  for _, enum in ipairs(protocol.enumerations) do
+    if enum.documentation then
+      enum.documentation = enum.documentation:gsub('\n', '\n---')
+      output[#output + 1] = '---' .. enum.documentation
+    end
+    local enum_type = '---@alias lsp.' .. enum.name
+    for _, value in ipairs(enum.values) do
+      enum_type = enum_type
+        .. '\n---| '
+        .. (type(value.value) == 'string' and '"' .. value.value .. '"' or value.value)
+        .. ' # '
+        .. value.name
+    end
+    output[#output + 1] = enum_type
+    output[#output + 1] = ''
+  end
+
+  for _, alias in ipairs(protocol.typeAliases) do
+    if alias.documentation then
+      alias.documentation = alias.documentation:gsub('\n', '\n---')
+      output[#output + 1] = '---' .. alias.documentation
+    end
+    if alias.type.kind == 'or' then
+      local alias_type = '---@alias lsp.' .. alias.name .. ' '
+      for _, item in ipairs(alias.type.items) do
+        alias_type = alias_type .. parse_type(item) .. '|'
+      end
+      alias_type = alias_type:sub(0, -2)
+      output[#output + 1] = alias_type
+    else
+      output[#output + 1] = '---@alias lsp.' .. alias.name .. ' ' .. parse_type(alias.type)
+    end
+    output[#output + 1] = ''
+  end
+
+  for _, line in ipairs(anonym_classes) do
+    output[#output + 1] = line
+  end
+
+  tofile(opt.output_file, table.concat(output, '\n'))
+end
+
+local opt = {
+  output_file = 'runtime/lua/vim/lsp/_meta/protocol.lua',
+  version = nil,
+  methods = nil,
+}
+
+for i = 1, #_G.arg do
+  if _G.arg[i] == '--out' then
+    opt.output_file = _G.arg[i + 1]
+  elseif _G.arg[i] == '--version' then
+    opt.version = _G.arg[i + 1]
+  elseif _G.arg[i] == '--methods' then
+    opt.methods = true
+  elseif vim.startswith(_G.arg[i], '--') then
+    opt.output_file = _G.arg[i]:sub(3)
+  end
+end
+
+for _, a in ipairs(arg) do
+  if M[a] then
+    M[a](opt)
+  end
+end
+
+return M
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 8e1d6ef80a..8ed88cb8f5 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -2,7 +2,7 @@
 """Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
 
 Also generates *.mpack files. To inspect the *.mpack structure:
-    :new | put=v:lua.vim.inspect(v:lua.vim.mpack.unpack(readfile('runtime/doc/api.mpack','B')))
+    :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B')))
 
 Flow:
     main
@@ -42,8 +42,12 @@ import subprocess
 import collections
 import msgpack
 import logging
+from typing import Tuple
+from pathlib import Path
 
 from xml.dom import minidom
+Element = minidom.Element
+Document = minidom.Document
 
 MIN_PYTHON_VERSION = (3, 6)
 MIN_DOXYGEN_VERSION = (1, 9, 0)
@@ -55,18 +59,28 @@ if sys.version_info < MIN_PYTHON_VERSION:
 doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"],
                         universal_newlines=True).split()[0].split('.')))
 
-# Until 0.9 is released, need this hacky way to check that "nvim -l foo.lua" works.
-nvim_version = list(line for line in subprocess.check_output(['nvim', '-h'], universal_newlines=True).split('\n')
-                     if '-l ' in line)
-
 if doxygen_version < MIN_DOXYGEN_VERSION:
     print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION))
     print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version))
     sys.exit(1)
 
-if len(nvim_version) == 0:
-    print("\nRequires 'nvim -l' feature, see https://github.com/neovim/neovim/pull/18706")
-    sys.exit(1)
+
+# Need a `nvim` that supports `-l`, try the local build
+nvim_path = Path(__file__).parent / "../build/bin/nvim"
+if nvim_path.exists():
+    nvim = str(nvim_path)
+else:
+    # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works.
+    nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True)
+    nvim_version = [line for line in nvim_out.split('\n')
+                    if '-l ' in line]
+    if len(nvim_version) == 0:
+        print((
+            "\nYou need to have a local Neovim build or a `nvim` version 0.9 for `-l` "
+            "support to build the documentation."))
+        sys.exit(1)
+    nvim = 'nvim'
+
 
 # DEBUG = ('DEBUG' in os.environ)
 INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ)
@@ -122,7 +136,7 @@ CONFIG = {
         # Section helptag.
         'helptag_fmt': lambda name: f'*api-{name.lower()}*',
         # Per-function helptag.
-        'fn_helptag_fmt': lambda fstem, name: f'*{name}()*',
+        'fn_helptag_fmt': lambda fstem, name, istbl: f'*{name}()*',
         # Module name overrides (for Lua).
         'module_override': {},
         # Append the docs for these modules, do not start a new section.
@@ -132,55 +146,111 @@ CONFIG = {
         'mode': 'lua',
         'filename': 'lua.txt',
         'section_order': [
+            'highlight.lua',
+            'regex.lua',
+            'diff.lua',
+            'mpack.lua',
+            'json.lua',
+            'base64.lua',
+            'spell.lua',
+            'builtin.lua',
+            '_options.lua',
             '_editor.lua',
             '_inspector.lua',
             'shared.lua',
+            'loader.lua',
             'uri.lua',
             'ui.lua',
             'filetype.lua',
             'keymap.lua',
             'fs.lua',
             'secure.lua',
+            'version.lua',
+            'iter.lua',
+            'snippet.lua',
+            'text.lua',
         ],
         'files': [
+            'runtime/lua/vim/iter.lua',
             'runtime/lua/vim/_editor.lua',
+            'runtime/lua/vim/_options.lua',
             'runtime/lua/vim/shared.lua',
+            'runtime/lua/vim/loader.lua',
             'runtime/lua/vim/uri.lua',
             'runtime/lua/vim/ui.lua',
             'runtime/lua/vim/filetype.lua',
             'runtime/lua/vim/keymap.lua',
             'runtime/lua/vim/fs.lua',
+            'runtime/lua/vim/highlight.lua',
             'runtime/lua/vim/secure.lua',
+            'runtime/lua/vim/version.lua',
             'runtime/lua/vim/_inspector.lua',
+            'runtime/lua/vim/snippet.lua',
+            'runtime/lua/vim/text.lua',
+            'runtime/lua/vim/_meta/builtin.lua',
+            'runtime/lua/vim/_meta/diff.lua',
+            'runtime/lua/vim/_meta/mpack.lua',
+            'runtime/lua/vim/_meta/json.lua',
+            'runtime/lua/vim/_meta/base64.lua',
+            'runtime/lua/vim/_meta/regex.lua',
+            'runtime/lua/vim/_meta/spell.lua',
         ],
         'file_patterns': '*.lua',
         'fn_name_prefix': '',
+        'fn_name_fmt': lambda fstem, name: (
+            name if fstem in [ 'vim.iter' ] else
+            f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else
+            f'vim.{name}' if fstem == '_options' and not name[0].isupper() else
+            f'{fstem}.{name}' if fstem.startswith('vim') else
+            name
+        ),
         'section_name': {
             'lsp.lua': 'core',
             '_inspector.lua': 'inspector',
         },
         'section_fmt': lambda name: (
-            'Lua module: vim'
-            if name.lower() == '_editor'
-            else f'Lua module: {name.lower()}'),
+            'Lua module: vim' if name.lower() == '_editor' else
+            'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else
+            f'VIM.{name.upper()}' if name.lower() in [ 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', 'regex' ] else
+            'VIM' if name.lower() == 'builtin' else
+            f'Lua module: vim.{name.lower()}'),
         'helptag_fmt': lambda name: (
-            '*lua-vim*'
-            if name.lower() == '_editor'
-            else f'*lua-{name.lower()}*'),
-        'fn_helptag_fmt': lambda fstem, name: (
-            f'*vim.{name}()*'
-            if fstem.lower() == '_editor'
-            else f'*{fstem}.{name}()*'),
+            '*lua-vim*' if name.lower() == '_editor' else
+            '*lua-vimscript*' if name.lower() == '_options' else
+            f'*vim.{name.lower()}*'),
+        'fn_helptag_fmt': lambda fstem, name, istbl: (
+            f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else
+            # Exclude fstem for methods
+            f'*{name}()*' if ':' in name else
+            f'*vim.{name}()*' if fstem.lower() == '_editor' else
+            f'*vim.{name}*' if fstem.lower() == '_options' and istbl else
+            # Prevents vim.regex.regex
+            f'*{fstem}()*' if fstem.endswith('.' + name) else
+            f'*{fstem}.{name}{"" if istbl else "()"}*'
+            ),
         'module_override': {
             # `shared` functions are exposed on the `vim` module.
             'shared': 'vim',
             '_inspector': 'vim',
             'uri': 'vim',
             'ui': 'vim.ui',
+            'loader': 'vim.loader',
             'filetype': 'vim.filetype',
             'keymap': 'vim.keymap',
             'fs': 'vim.fs',
+            'highlight': 'vim.highlight',
             'secure': 'vim.secure',
+            'version': 'vim.version',
+            'iter': 'vim.iter',
+            'diff': 'vim',
+            'builtin': 'vim',
+            'mpack': 'vim.mpack',
+            'json': 'vim.json',
+            'base64': 'vim.base64',
+            'regex': 'vim.regex',
+            'spell': 'vim.spell',
+            'snippet': 'vim.snippet',
+            'text': 'vim.text',
         },
         'append_only': [
             'shared.lua',
@@ -194,13 +264,13 @@ CONFIG = {
             'buf.lua',
             'diagnostic.lua',
             'codelens.lua',
+            'inlay_hint.lua',
             'tagfunc.lua',
             'semantic_tokens.lua',
             'handlers.lua',
             'util.lua',
             'log.lua',
             'rpc.lua',
-            'sync.lua',
             'protocol.lua',
         ],
         'files': [
@@ -218,14 +288,11 @@ CONFIG = {
             '*lsp-core*'
             if name.lower() == 'lsp'
             else f'*lsp-{name.lower()}*'),
-        'fn_helptag_fmt': lambda fstem, name: (
-            f'*vim.lsp.{name}()*'
-            if fstem == 'lsp' and name != 'client'
-            else (
-                '*vim.lsp.client*'
-                # HACK. TODO(justinmk): class/structure support in lua2dox
-                if 'lsp.client' == f'{fstem}.{name}'
-                else f'*vim.lsp.{fstem}.{name}()*')),
+        'fn_helptag_fmt': lambda fstem, name, istbl: (
+            f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else
+            # HACK. TODO(justinmk): class/structure support in lua2dox
+            '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else
+            f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'),
         'module_override': {},
         'append_only': [],
     },
@@ -238,10 +305,11 @@ CONFIG = {
         'files': ['runtime/lua/vim/diagnostic.lua'],
         'file_patterns': '*.lua',
         'fn_name_prefix': '',
+        'include_tables': False,
         'section_name': {'diagnostic.lua': 'diagnostic'},
         'section_fmt': lambda _: 'Lua module: vim.diagnostic',
         'helptag_fmt': lambda _: '*diagnostic-api*',
-        'fn_helptag_fmt': lambda fstem, name: f'*vim.{fstem}.{name}()*',
+        'fn_helptag_fmt': lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*',
         'module_override': {},
         'append_only': [],
     },
@@ -254,6 +322,7 @@ CONFIG = {
             'query.lua',
             'highlighter.lua',
             'languagetree.lua',
+            'dev.lua',
         ],
         'files': [
             'runtime/lua/vim/treesitter.lua',
@@ -270,7 +339,7 @@ CONFIG = {
             '*lua-treesitter-core*'
             if name.lower() == 'treesitter'
             else f'*lua-treesitter-{name.lower()}*'),
-        'fn_helptag_fmt': lambda fstem, name: (
+        'fn_helptag_fmt': lambda fstem, name, istbl: (
             f'*vim.{fstem}.{name}()*'
             if fstem == 'treesitter'
             else f'*{name}()*'
@@ -288,12 +357,37 @@ param_exclude = (
 # Annotations are displayed as line items after API function descriptions.
 annotation_map = {
     'FUNC_API_FAST': '|api-fast|',
-    'FUNC_API_CHECK_TEXTLOCK': 'not allowed when |textlock| is active',
+    'FUNC_API_TEXTLOCK': 'not allowed when |textlock| is active or in the |cmdwin|',
+    'FUNC_API_TEXTLOCK_ALLOW_CMDWIN': 'not allowed when |textlock| is active',
     'FUNC_API_REMOTE_ONLY': '|RPC| only',
     'FUNC_API_LUA_ONLY': 'Lua |vim.api| only',
 }
 
 
+def nvim_api_info() -> Tuple[int, bool]:
+    """Returns NVIM_API_LEVEL, NVIM_API_PRERELEASE from CMakeLists.txt"""
+    if not hasattr(nvim_api_info, 'LEVEL'):
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        cmake_file_path = os.path.join(script_dir, '..', 'CMakeLists.txt')
+        with open(cmake_file_path, 'r') as cmake_file:
+            cmake_content = cmake_file.read()
+
+        api_level_match = re.search(r'set\(NVIM_API_LEVEL (\d+)\)', cmake_content)
+        api_prerelease_match = re.search(
+            r'set\(NVIM_API_PRERELEASE (\w+)\)', cmake_content
+        )
+
+        if not api_level_match or not api_prerelease_match:
+            raise RuntimeError(
+                'Could not find NVIM_API_LEVEL or NVIM_API_PRERELEASE in CMakeLists.txt'
+            )
+
+        nvim_api_info.LEVEL = int(api_level_match.group(1))
+        nvim_api_info.PRERELEASE = api_prerelease_match.group(1).lower() == 'true'
+
+    return nvim_api_info.LEVEL, nvim_api_info.PRERELEASE
+
+
 # Raises an error with details about `o`, if `cond` is in object `o`,
 # or if `cond()` is callable and returns True.
 def debug_this(o, cond=True):
@@ -381,7 +475,7 @@ def is_blank(text):
     return '' == clean_lines(text)
 
 
-def get_text(n, preformatted=False):
+def get_text(n):
     """Recursively concatenates all text in a node tree."""
     text = ''
     if n.nodeType == n.TEXT_NODE:
@@ -390,11 +484,13 @@ def get_text(n, preformatted=False):
         for node in n.childNodes:
             text += get_text(node)
         return '`{}`'.format(text)
+    if n.nodeName == 'sp': # space, used in "programlisting" nodes
+        return ' '
     for node in n.childNodes:
         if node.nodeType == node.TEXT_NODE:
             text += node.data
         elif node.nodeType == node.ELEMENT_NODE:
-            text += get_text(node, preformatted)
+            text += get_text(node)
     return text
 
 
@@ -512,17 +608,26 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation,
     # text += (int(not space_preceding) * ' ')
 
     if n.nodeName == 'preformatted':
-        o = get_text(n, preformatted=True)
+        o = get_text(n)
         ensure_nl = '' if o[-1] == '\n' else '\n'
         if o[0:4] == 'lua\n':
             text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1])
         elif o[0:4] == 'vim\n':
             text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1])
+        elif o[0:5] == 'help\n':
+            text += o[4:-1]
         else:
             text += '>{}{}\n<'.format(ensure_nl, o)
-
+    elif n.nodeName == 'programlisting': # codeblock (```)
+        o = get_text(n)
+        text += '>'
+        if 'filename' in n.attributes:
+            filename = n.attributes['filename'].value
+            text += filename.lstrip('.')
+
+        text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4))
     elif is_inline(n):
-        text = doc_wrap(get_text(n), indent=indent, width=width)
+        text = doc_wrap(get_text(n), prefix=prefix, indent=indent, width=width)
     elif n.nodeName == 'verbatim':
         # TODO: currently we don't use this. The "[verbatim]" hint is there as
         # a reminder that we must decide how to format this if we do use it.
@@ -535,19 +640,19 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation,
                 indent=indent + (' ' * len(prefix)),
                 width=width
             )
-
             if is_blank(result):
                 continue
-
             text += indent + prefix + result
     elif n.nodeName in ('para', 'heading'):
+        did_prefix = False
         for c in n.childNodes:
             if (is_inline(c)
                     and '' != get_text(c).strip()
                     and text
                     and ' ' != text[-1]):
                 text += ' '
-            text += render_node(c, text, indent=indent, width=width)
+            text += render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width)
+            did_prefix = True
     elif n.nodeName == 'itemizedlist':
         for c in n.childNodes:
             text += '{}\n'.format(render_node(c, text, prefix='• ',
@@ -562,17 +667,26 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation,
                                               indent=indent, width=width))
             i = i + 1
     elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'):
-        text += '\nNote:\n    '
+        text += ind('  ')
         for c in n.childNodes:
-            text += render_node(c, text, indent='    ', width=width)
-        text += '\n'
+            if is_blank(render_node(c, text, prefix='• ', indent='    ', width=width)):
+                continue
+            text += render_node(c, text, prefix='• ', indent='    ', width=width)
+        # text += '\n'
     elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'):
         text += 'Warning:\n    '
         for c in n.childNodes:
             text += render_node(c, text, indent='    ', width=width)
         text += '\n'
-    elif (n.nodeName == 'simplesect'
-            and n.getAttribute('kind') in ('return', 'see')):
+    elif n.nodeName == 'simplesect' and 'see' == n.getAttribute('kind'):
+        text += ind('  ')
+        # Example:
+        #   <simplesect kind="see">
+        #     <para>|autocommand|</para>
+        #   </simplesect>
+        for c in n.childNodes:
+            text += render_node(c, text, prefix='• ', indent='    ', width=width)
+    elif n.nodeName == 'simplesect' and 'return' == n.getAttribute('kind'):
         text += ind('    ')
         for c in n.childNodes:
             text += render_node(c, text, indent='    ', width=width)
@@ -590,6 +704,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
 
     Keys:
         'text': Text from this <para> element
+        'note': List of @note strings
         'params': <parameterlist> map
         'return': List of @return strings
         'seealso': List of @see strings
@@ -597,14 +712,17 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
     """
     chunks = {
         'text': '',
+        'note': [],
         'params': collections.OrderedDict(),
         'return': [],
         'seealso': [],
+        'prerelease': False,
         'xrefs': []
     }
 
     # Ordered dict of ordered lists.
     groups = collections.OrderedDict([
+        ('note', []),
         ('params', []),
         ('return', []),
         ('seealso', []),
@@ -615,7 +733,6 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
     # nodes to appear together.
     text = ''
     kind = ''
-    last = ''
     if is_inline(parent):
         # Flatten inline text from a tree of non-block nodes.
         text = doc_wrap(render_node(parent, "", fmt_vimhelp=fmt_vimhelp),
@@ -628,15 +745,24 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
             elif child.nodeName == 'xrefsect':
                 groups['xrefs'].append(child)
             elif child.nodeName == 'simplesect':
-                last = kind
                 kind = child.getAttribute('kind')
-                if kind == 'return' or (kind == 'note' and last == 'return'):
+                if kind == 'note':
+                    groups['note'].append(child)
+                elif kind == 'return':
                     groups['return'].append(child)
                 elif kind == 'see':
                     groups['seealso'].append(child)
-                elif kind in ('note', 'warning'):
+                elif kind == 'warning':
                     text += render_node(child, text, indent=indent,
                                         width=width, fmt_vimhelp=fmt_vimhelp)
+                elif kind == 'since':
+                    since_match = re.match(r'^(\d+)', get_text(child))
+                    since = int(since_match.group(1)) if since_match else 0
+                    NVIM_API_LEVEL, NVIM_API_PRERELEASE = nvim_api_info()
+                    if since > NVIM_API_LEVEL or (
+                        since == NVIM_API_LEVEL and NVIM_API_PRERELEASE
+                    ):
+                        chunks['prerelease'] = True
                 else:
                     raise RuntimeError('unhandled simplesect: {}\n{}'.format(
                         child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
@@ -659,10 +785,17 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
     if len(groups['params']) > 0:
         for child in groups['params']:
             update_params_map(child, ret_map=chunks['params'], width=width)
+    for child in groups['note']:
+        chunks['note'].append(render_node(
+            child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp).rstrip())
     for child in groups['return']:
         chunks['return'].append(render_node(
             child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp))
     for child in groups['seealso']:
+        # Example:
+        #   <simplesect kind="see">
+        #     <para>|autocommand|</para>
+        #   </simplesect>
         chunks['seealso'].append(render_node(
             child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp))
 
@@ -678,8 +811,28 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F
 
     return chunks, xrefs
 
+def is_program_listing(para):
+    """
+    Return True if `para` contains a "programlisting" (i.e. a Markdown code
+    block ```).
+
+    Sometimes a <para> element will have only a single "programlisting" child
+    node, but othertimes it will have extra whitespace around the
+    "programlisting" node.
 
-def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='',
+    @param para XML <para> node
+    @return True if <para> is a programlisting
+    """
+
+    # Remove any child text nodes that are only whitespace
+    children = [
+        n for n in para.childNodes
+        if n.nodeType != n.TEXT_NODE or n.data.strip() != ''
+    ]
+
+    return len(children) == 1 and children[0].nodeName == 'programlisting'
+
+def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='',
                         fmt_vimhelp=False):
     """Renders (nested) Doxygen <para> nodes as Vim :help text.
 
@@ -692,6 +845,8 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='',
         max_name_len = max_name(m.keys()) + 4
         out = ''
         for name, desc in m.items():
+            if name == 'self':
+                continue
             name = '  • {}'.format('{{{}}}'.format(name).ljust(max_name_len))
             out += '{}{}\n'.format(name, desc)
         return out.rstrip()
@@ -705,13 +860,28 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='',
     for child in parent.childNodes:
         para, _ = para_as_map(child, indent, width, fmt_vimhelp)
 
+        # 'programlisting' blocks are Markdown code blocks. Do not include
+        # these as a separate paragraph, but append to the last non-empty line
+        # in the text
+        if is_program_listing(child):
+            while rendered_blocks and rendered_blocks[-1] == '':
+                rendered_blocks.pop()
+            rendered_blocks[-1] += ' ' + para['text']
+            continue
+
         # Generate text from the gathered items.
         chunks = [para['text']]
+        notes = ["    This API is pre-release (unstable)."] if para['prerelease'] else []
+        notes += para['note']
+        if len(notes) > 0:
+            chunks.append('\nNote: ~')
+            for s in notes:
+                chunks.append(s)
         if len(para['params']) > 0 and has_nonexcluded_params(para['params']):
             chunks.append('\nParameters: ~')
             chunks.append(fmt_param_doc(para['params']))
         if len(para['return']) > 0:
-            chunks.append('\nReturn: ~')
+            chunks.append('\nReturn (multiple): ~' if len(para['return']) > 1 else '\nReturn: ~')
             for s in para['return']:
                 chunks.append(s)
         if len(para['seealso']) > 0:
@@ -757,6 +927,13 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
         if return_type == '':
             continue
 
+        if 'local_function' in return_type:  # Special from lua2dox.lua.
+            continue
+
+        istbl = return_type.startswith('table')  # Special from lua2dox.lua.
+        if istbl and not CONFIG[target].get('include_tables', True):
+            continue
+
         if return_type.startswith(('ArrayOf', 'DictionaryOf')):
             parts = return_type.strip('_').split('_')
             return_type = '{}({})'.format(parts[0], ', '.join(parts[1:]))
@@ -805,6 +982,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
                 and any(x[1] == 'self' for x in params):
             split_return = return_type.split(' ')
             name = f'{split_return[1]}:{name}'
+            params = [x for x in params if x[1] != 'self']
 
         c_args = []
         for param_type, param_name in params:
@@ -818,12 +996,20 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
             if '.' in compoundname:
                 fstem = compoundname.split('.')[0]
                 fstem = CONFIG[target]['module_override'].get(fstem, fstem)
-            vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name)
+            vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name, istbl)
+
+            if 'fn_name_fmt' in CONFIG[target]:
+                name = CONFIG[target]['fn_name_fmt'](fstem, name)
+
+        if istbl:
+            aopen, aclose = '', ''
+        else:
+            aopen, aclose = '(', ')'
 
-        prefix = '%s(' % name
-        suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params
-                                   if a[0] not in ('void', 'Error', 'Arena',
-                                                   'lua_State'))
+        prefix = name + aopen
+        suffix = ', '.join('{%s}' % a[1] for a in params
+                           if a[0] not in ('void', 'Error', 'Arena',
+                                           'lua_State')) + aclose
 
         if not fmt_vimhelp:
             c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
@@ -1001,6 +1187,42 @@ def delete_lines_below(filename, tokenstr):
         fp.writelines(lines[0:i])
 
 
+def extract_defgroups(base: str, dom: Document):
+    '''Generate module-level (section) docs (@defgroup).'''
+    section_docs = {}
+
+    for compound in dom.getElementsByTagName('compound'):
+        if compound.getAttribute('kind') != 'group':
+            continue
+
+        # Doxygen "@defgroup" directive.
+        groupname = get_text(find_first(compound, 'name'))
+        groupxml = os.path.join(base, '%s.xml' %
+                                compound.getAttribute('refid'))
+
+        group_parsed = minidom.parse(groupxml)
+        doc_list = []
+        brief_desc = find_first(group_parsed, 'briefdescription')
+        if brief_desc:
+            for child in brief_desc.childNodes:
+                doc_list.append(fmt_node_as_vimhelp(child))
+
+        desc = find_first(group_parsed, 'detaileddescription')
+        if desc:
+            doc = fmt_node_as_vimhelp(desc)
+
+            if doc:
+                doc_list.append(doc)
+
+        # Can't use '.' in @defgroup, so convert to '--'
+        # "vim.json" => "vim-dot-json"
+        groupname = groupname.replace('-dot-', '.')
+
+        section_docs[groupname] = "\n".join(doc_list)
+
+    return section_docs
+
+
 def main(doxygen_config, args):
     """Generates:
 
@@ -1042,62 +1264,44 @@ def main(doxygen_config, args):
 
         fn_map_full = {}  # Collects all functions as each module is processed.
         sections = {}
-        intros = {}
         sep = '=' * text_width
 
         base = os.path.join(output_dir, 'xml')
         dom = minidom.parse(os.path.join(base, 'index.xml'))
 
-        # generate docs for section intros
-        for compound in dom.getElementsByTagName('compound'):
-            if compound.getAttribute('kind') != 'group':
-                continue
-
-            groupname = get_text(find_first(compound, 'name'))
-            groupxml = os.path.join(base, '%s.xml' %
-                                    compound.getAttribute('refid'))
-
-            group_parsed = minidom.parse(groupxml)
-            doc_list = []
-            brief_desc = find_first(group_parsed, 'briefdescription')
-            if brief_desc:
-                for child in brief_desc.childNodes:
-                    doc_list.append(fmt_node_as_vimhelp(child))
-
-            desc = find_first(group_parsed, 'detaileddescription')
-            if desc:
-                doc = fmt_node_as_vimhelp(desc)
-
-                if doc:
-                    doc_list.append(doc)
-
-            intros[groupname] = "\n".join(doc_list)
+        section_docs = extract_defgroups(base, dom)
 
+        # Generate docs for all functions in the current module.
         for compound in dom.getElementsByTagName('compound'):
             if compound.getAttribute('kind') != 'file':
                 continue
 
             filename = get_text(find_first(compound, 'name'))
             if filename.endswith('.c') or filename.endswith('.lua'):
-                xmlfile = os.path.join(base,
-                                       '{}.xml'.format(compound.getAttribute('refid')))
+                xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid')))
                 # Extract unformatted (*.mpack).
                 fn_map, _ = extract_from_xml(xmlfile, target, 9999, False)
                 # Extract formatted (:help).
                 functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
-                    os.path.join(base, '{}.xml'.format(
-                                 compound.getAttribute('refid'))), target)
+                    os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))), target)
 
                 if not functions_text and not deprecated_text:
                     continue
                 else:
-                    name = os.path.splitext(
-                            os.path.basename(filename))[0].lower()
+                    filename = os.path.basename(filename)
+                    name = os.path.splitext(filename)[0].lower()
                     sectname = name.upper() if name == 'ui' else name.title()
+                    sectname = CONFIG[target]['section_name'].get(filename, sectname)
+                    title = CONFIG[target]['section_fmt'](sectname)
+                    section_tag = CONFIG[target]['helptag_fmt'](sectname)
+                    # Module/Section id matched against @defgroup.
+                    #   "*api-buffer*" => "api-buffer"
+                    section_id = section_tag.strip('*')
+
                     doc = ''
-                    intro = intros.get(f'api-{name}')
-                    if intro:
-                        doc += '\n\n' + intro
+                    section_doc = section_docs.get(section_id)
+                    if section_doc:
+                        doc += '\n\n' + section_doc
 
                     if functions_text:
                         doc += '\n\n' + functions_text
@@ -1107,12 +1311,7 @@ def main(doxygen_config, args):
                         doc += deprecated_text
 
                     if doc:
-                        filename = os.path.basename(filename)
-                        sectname = CONFIG[target]['section_name'].get(
-                                filename, sectname)
-                        title = CONFIG[target]['section_fmt'](sectname)
-                        helptag = CONFIG[target]['helptag_fmt'](sectname)
-                        sections[filename] = (title, helptag, doc)
+                        sections[filename] = (title, section_tag, doc)
                         fn_map_full.update(fn_map)
 
         if len(sections) == 0:
@@ -1127,15 +1326,14 @@ def main(doxygen_config, args):
 
         for filename in CONFIG[target]['section_order']:
             try:
-                title, helptag, section_doc = sections.pop(filename)
+                title, section_tag, section_doc = sections.pop(filename)
             except KeyError:
                 msg(f'warning: empty docs, skipping (target={target}): {filename}')
                 msg(f'    existing docs: {sections.keys()}')
                 continue
             if filename not in CONFIG[target]['append_only']:
                 docs += sep
-                docs += '\n%s%s' % (title,
-                                    helptag.rjust(text_width - len(title)))
+                docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title)))
             docs += section_doc
             docs += '\n\n\n'
 
@@ -1160,10 +1358,12 @@ def main(doxygen_config, args):
     msg_report()
 
 
-def filter_source(filename):
+def filter_source(filename, keep_tmpfiles):
+    output_dir = out_dir.format(target='lua2dox')
     name, extension = os.path.splitext(filename)
     if extension == '.lua':
-        p = subprocess.run(['nvim', '-l', lua2dox, filename], stdout=subprocess.PIPE)
+        args = [str(nvim), '-l', lua2dox, filename] + (['--outdir', output_dir] if keep_tmpfiles else [])
+        p = subprocess.run(args, stdout=subprocess.PIPE)
         op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8'))
         print(op)
     else:
@@ -1186,7 +1386,7 @@ def parse_args():
     ap.add_argument('source_filter', nargs='*',
                     help="Filter source file(s)")
     ap.add_argument('-k', '--keep-tmpfiles', action='store_true',
-                    help="Keep temporary files")
+                    help="Keep temporary files (tmp-xx-doc/ directories, including tmp-lua2dox-doc/ for lua2dox.lua quasi-C output)")
     ap.add_argument('-t', '--target',
                     help=f'One of ({targets}), defaults to "all"')
     return ap.parse_args()
@@ -1234,8 +1434,13 @@ if __name__ == "__main__":
     log.setLevel(args.log_level)
     log.addHandler(logging.StreamHandler())
 
+    # When invoked as a filter, args won't be passed, so use an env var.
+    if args.keep_tmpfiles:
+        os.environ['NVIM_KEEP_TMPFILES'] = '1'
+    keep_tmpfiles = ('NVIM_KEEP_TMPFILES' in os.environ)
+
     if len(args.source_filter) > 0:
-        filter_source(args.source_filter[0])
+        filter_source(args.source_filter[0], keep_tmpfiles)
     else:
         main(Doxyfile, args)
 
diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh
index 9944b5eb31..b0bf186f85 100755
--- a/scripts/genappimage.sh
+++ b/scripts/genappimage.sh
@@ -26,7 +26,7 @@ APP_DIR="$APP.AppDir"
 ########################################################################
 
 # Build and install nvim into the AppImage
-make CMAKE_BUILD_TYPE=RelWithDebInfo CMAKE_EXTRA_FLAGS="-DCMAKE_INSTALL_PREFIX=${APP_DIR}/usr -DCMAKE_INSTALL_MANDIR=man"
+make CMAKE_BUILD_TYPE="${NVIM_BUILD_TYPE}" CMAKE_EXTRA_FLAGS="-DCMAKE_INSTALL_PREFIX=${APP_DIR}/usr -DCMAKE_INSTALL_MANDIR=man"
 make install
 
 ########################################################################
diff --git a/scripts/genvimvim.lua b/scripts/genvimvim.lua
deleted file mode 100644
index 3e9e7077be..0000000000
--- a/scripts/genvimvim.lua
+++ /dev/null
@@ -1,141 +0,0 @@
-local mpack = require('mpack')
-
-if arg[1] == '--help' then
-  print('Usage: lua genvimvim.lua src/nvim runtime/syntax/vim/generated.vim')
-  os.exit(0)
-end
-
-local nvimsrcdir = arg[1]
-local syntax_file = arg[2]
-local funcs_file = arg[3]
-
-package.path = nvimsrcdir .. '/?.lua;' .. package.path
-
-_G.vim = loadfile(nvimsrcdir..'/../../runtime/lua/vim/shared.lua')()
-
-local lld = {}
-local syn_fd = io.open(syntax_file, 'w')
-lld.line_length = 0
-local function w(s)
-  syn_fd:write(s)
-  if s:find('\n') then
-    lld.line_length = #(s:gsub('.*\n', ''))
-  else
-    lld.line_length = lld.line_length + #s
-  end
-end
-
-local options = require('options')
-local auevents = require('auevents')
-local ex_cmds = require('ex_cmds')
-
-local function cmd_kw(prev_cmd, cmd)
-  if not prev_cmd then
-    return cmd:sub(1, 1) .. '[' .. cmd:sub(2) .. ']'
-  else
-    local shift = 1
-    while cmd:sub(shift, shift) == prev_cmd:sub(shift, shift) do
-      shift = shift + 1
-    end
-    if shift >= #cmd then
-      return cmd
-    else
-      return cmd:sub(1, shift) .. '[' .. cmd:sub(shift + 1) .. ']'
-    end
-  end
-end
-
--- Exclude these from the vimCommand keyword list, they are handled specially
--- in syntax/vim.vim (vimAugroupKey, vimAutoCmd, vimGlobal, vimSubst). #9327
-local function is_special_cased_cmd(cmd)
-  return (cmd == 'augroup'
-          or cmd == 'autocmd'
-          or cmd == 'doautocmd'
-          or cmd == 'doautoall'
-          or cmd == 'global'
-          or cmd == 'substitute')
-end
-
-local vimcmd_start = 'syn keyword vimCommand contained '
-w(vimcmd_start)
-local prev_cmd = nil
-for _, cmd_desc in ipairs(ex_cmds.cmds) do
-  if lld.line_length > 850 then
-    w('\n' .. vimcmd_start)
-  end
-  local cmd = cmd_desc.command
-  if cmd:match('%w') and cmd ~= 'z' and not is_special_cased_cmd(cmd) then
-    w(' ' .. cmd_kw(prev_cmd, cmd))
-  end
-  prev_cmd = cmd
-end
-
-local vimopt_start = 'syn keyword vimOption contained '
-w('\n\n' .. vimopt_start)
-
-for _, opt_desc in ipairs(options.options) do
-  if not opt_desc.varname or opt_desc.varname:sub(1, 7) ~= 'p_force' then
-    if lld.line_length > 850 then
-      w('\n' .. vimopt_start)
-    end
-    w(' ' .. opt_desc.full_name)
-    if opt_desc.abbreviation then
-      w(' ' .. opt_desc.abbreviation)
-    end
-    if opt_desc.type == 'bool' then
-      w(' inv' .. opt_desc.full_name)
-      w(' no' .. opt_desc.full_name)
-      if opt_desc.abbreviation then
-        w(' inv' .. opt_desc.abbreviation)
-        w(' no' .. opt_desc.abbreviation)
-      end
-    end
-  end
-end
-
-w('\n\nsyn case ignore')
-local vimau_start = 'syn keyword vimAutoEvent contained '
-w('\n\n' .. vimau_start)
-
-for _, au in ipairs(auevents.events) do
-  if not auevents.nvim_specific[au] then
-    if lld.line_length > 850 then
-      w('\n' .. vimau_start)
-    end
-    w(' ' .. au)
-  end
-end
-for au, _ in pairs(auevents.aliases) do
-  if not auevents.nvim_specific[au] then
-    if lld.line_length > 850 then
-      w('\n' .. vimau_start)
-    end
-    w(' ' .. au)
-  end
-end
-
-local nvimau_start = 'syn keyword nvimAutoEvent contained '
-w('\n\n' .. nvimau_start)
-
-for au, _ in vim.spairs(auevents.nvim_specific) do
-  if lld.line_length > 850 then
-    w('\n' .. nvimau_start)
-  end
-  w(' ' .. au)
-end
-
-w('\n\nsyn case match')
-local vimfun_start = 'syn keyword vimFuncName contained '
-w('\n\n' .. vimfun_start)
-local funcs = mpack.unpack(io.open(funcs_file, 'rb'):read("*all"))
-for _, name in ipairs(funcs) do
-  if name then
-    if lld.line_length > 850 then
-      w('\n' .. vimfun_start)
-    end
-    w(' ' .. name)
-  end
-end
-
-w('\n')
-syn_fd:close()
diff --git a/scripts/lintcommit.lua b/scripts/lintcommit.lua
index 87a8e62503..d2c8601c25 100644
--- a/scripts/lintcommit.lua
+++ b/scripts/lintcommit.lua
@@ -1,13 +1,14 @@
 -- Usage:
 --    # verbose
---    nvim -es +"lua require('scripts.lintcommit').main()"
+--    nvim -l scripts/lintcommit.lua main --trace
 --
 --    # silent
---    nvim -es +"lua require('scripts.lintcommit').main({trace=false})"
+--    nvim -l scripts/lintcommit.lua main
 --
 --    # self-test
---    nvim -es +"lua require('scripts.lintcommit')._test()"
+--    nvim -l scripts/lintcommit.lua _test
 
+--- @type table<string,fun(opt: LintcommitOptions)>
 local M = {}
 
 local _trace = false
@@ -19,11 +20,6 @@ local function p(s)
   vim.cmd('set verbose=0')
 end
 
-local function die()
-  p('')
-  vim.cmd("cquit 1")
-end
-
 -- Executes and returns the output of `cmd`, or nil on failure.
 --
 -- Prints `cmd` if `trace` is enabled.
@@ -35,7 +31,7 @@ local function run(cmd, or_die)
   if vim.v.shell_error ~= 0 then
     if or_die then
       p(rv)
-      die()
+      os.exit(1)
     end
     return nil
   end
@@ -51,7 +47,7 @@ local function validate_commit(commit_message)
     return nil
   end
 
-  local commit_split = vim.split(commit_message, ":")
+  local commit_split = vim.split(commit_message, ":", {plain = true})
   -- Return nil if the type is vim-patch since most of the normal rules don't
   -- apply.
   if commit_split[1] == "vim-patch" then
@@ -63,13 +59,22 @@ local function validate_commit(commit_message)
     return [[Commit message is too long, a maximum of 80 characters is allowed.]]
   end
 
+  local before_colon = commit_split[1]
 
-  if vim.tbl_count(commit_split) < 2 then
+  local after_idx = 2
+  if before_colon:match('^[^%(]*%([^%)]*$') then
+    -- Need to find the end of commit scope when commit scope contains colons.
+    while after_idx <= vim.tbl_count(commit_split) do
+      after_idx = after_idx + 1
+      if commit_split[after_idx - 1]:find(')') then
+        break
+      end
+    end
+  end
+  if after_idx > vim.tbl_count(commit_split) then
     return [[Commit message does not include colons.]]
   end
-
-  local before_colon = commit_split[1]
-  local after_colon = commit_split[2]
+  local after_colon = commit_split[after_idx]
 
   -- Check if commit introduces a breaking change.
   if vim.endswith(before_colon, "!") then
@@ -77,18 +82,20 @@ local function validate_commit(commit_message)
   end
 
   -- Check if type is correct
-  local type = vim.split(before_colon, "%(")[1]
-  local allowed_types = {'build', 'ci', 'docs', 'feat', 'fix', 'perf', 'refactor', 'revert', 'test', 'dist', 'vim-patch'}
+  local type = vim.split(before_colon, "(", {plain = true})[1]
+  local allowed_types = {'build', 'ci', 'docs', 'feat', 'fix', 'perf', 'refactor', 'revert', 'test', 'vim-patch'}
   if not vim.tbl_contains(allowed_types, type) then
     return string.format(
-      'Invalid commit type "%s". Allowed types are:\n    %s',
+      [[Invalid commit type "%s". Allowed types are:
+      %s.
+    If none of these seem appropriate then use "fix"]],
       type,
       vim.inspect(allowed_types))
   end
 
   -- Check if scope is appropriate
   if before_colon:match("%(") then
-    local scope = vim.trim(before_colon:match("%((.*)%)"))
+    local scope = vim.trim(commit_message:match("%((.-)%)"))
 
     if scope == '' then
       return [[Scope can't be empty]]
@@ -126,10 +133,10 @@ local function validate_commit(commit_message)
     return [[There should only be one whitespace after the colon.]]
   end
 
-  -- Check that first character after space isn't uppercase.
-  if string.match(after_colon:sub(2,2), '%u') then
-    return [[First character should not be uppercase.]]
-   end
+  -- Allow lowercase or ALL_UPPER but not Titlecase.
+  if after_colon:match('^ *%u%l') then
+    return [[Description first word should not be Capitalized.]]
+  end
 
   -- Check that description isn't just whitespaces
   if vim.trim(after_colon) == "" then
@@ -139,6 +146,7 @@ local function validate_commit(commit_message)
   return nil
 end
 
+--- @param opt? LintcommitOptions
 function M.main(opt)
   _trace = not opt or not not opt.trace
 
@@ -149,10 +157,11 @@ function M.main(opt)
     ancestor = run({'git', 'merge-base', 'upstream/master', branch})
   end
   local commits_str = run({'git', 'rev-list', ancestor..'..'..branch}, true)
+  assert(commits_str)
 
-  local commits = {}
+  local commits = {} --- @type string[]
   for substring in commits_str:gmatch("%S+") do
-     table.insert(commits, substring)
+   table.insert(commits, substring)
   end
 
   local failed = 0
@@ -164,13 +173,16 @@ function M.main(opt)
       local invalid_msg = validate_commit(msg)
       if invalid_msg then
         failed = failed + 1
+
+        -- Some breathing room
+        if failed == 1 then
+          p('\n')
+        end
+
         p(string.format([[
 Invalid commit message: "%s"
     Commit: %s
     %s
-    See also:
-        https://github.com/neovim/neovim/blob/master/CONTRIBUTING.md#commit-messages
-        https://www.conventionalcommits.org/en/v1.0.0/
 ]],
           msg,
           commit_id,
@@ -180,7 +192,12 @@ Invalid commit message: "%s"
   end
 
   if failed > 0 then
-    die()  -- Exit with error.
+        p([[
+See also:
+    https://github.com/neovim/neovim/blob/master/CONTRIBUTING.md#commit-messages
+
+]])
+    os.exit(1)
   else
     p('')
   end
@@ -198,7 +215,6 @@ function M._test()
     ['refactor: normal message'] = true,
     ['revert: normal message'] = true,
     ['test: normal message'] = true,
-    ['dist: normal message'] = true,
     ['ci(window): message with scope'] = true,
     ['ci!: message with breaking change'] = true,
     ['ci(tui)!: message with scope and breaking change'] = true,
@@ -223,8 +239,19 @@ function M._test()
     ['refactor(): empty scope'] = false,
     ['ci( ): whitespace as scope'] = false,
     ['ci: period at end of sentence.'] = false,
-    ['ci: Starting sentence capitalized'] = false,
+    ['ci: Capitalized first word'] = false,
+    ['ci: UPPER_CASE First Word'] = true,
     ['unknown: using unknown type'] = false,
+    ['feat: foo:bar'] = true,
+    ['feat(something): foo:bar'] = true,
+    ['feat(:grep): read from pipe'] = true,
+    ['feat(:grep/:make): read from pipe'] = true,
+    ['feat(:grep): foo:bar'] = true,
+    ['feat(:grep/:make): foo:bar'] = true,
+    ['feat(:grep)'] = false,
+    ['feat(:grep/:make)'] = false,
+    ['feat(:grep'] = false,
+    ['feat(:grep/:make'] = false,
     ['ci: you\'re saying this commit message just goes on and on and on and on and on and on for way too long?'] = false,
   }
 
@@ -238,9 +265,30 @@ function M._test()
   end
 
   if failed > 0 then
-    die()  -- Exit with error.
+    os.exit(1)
   end
 
 end
 
-return M
+--- @class LintcommitOptions
+--- @field trace? boolean
+local opt = {}
+
+for _, a in ipairs(arg) do
+  if vim.startswith(a, '--') then
+    local nm, val = a:sub(3), true
+    if vim.startswith(a, '--no') then
+      nm, val = a:sub(5), false
+    end
+
+    if nm == 'trace' then
+      opt.trace = val
+    end
+  end
+end
+
+for _, a in ipairs(arg) do
+  if M[a] then
+    M[a](opt)
+  end
+end
diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua
index 19f8f8141d..1c8bc5a3cb 100644
--- a/scripts/lua2dox.lua
+++ b/scripts/lua2dox.lua
@@ -1,6 +1,6 @@
---[[--------------------------------------------------------------------------
---   Copyright (C) 2012 by Simon Dales   --
---   simon@purrsoft.co.uk   --
+-----------------------------------------------------------------------------
+--   Copyright (C) 2012 by Simon Dales                                     --
+--   simon@purrsoft.co.uk                                                  --
 --                                                                         --
 --   This program is free software; you can redistribute it and/or modify  --
 --   it under the terms of the GNU General Public License as published by  --
@@ -16,7 +16,7 @@
 --   along with this program; if not, write to the                         --
 --   Free Software Foundation, Inc.,                                       --
 --   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             --
-----------------------------------------------------------------------------]]
+-----------------------------------------------------------------------------
 
 --[[!
 Lua-to-Doxygen converter
@@ -24,11 +24,18 @@ Lua-to-Doxygen converter
 Partially from lua2dox
 http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm
 
-Running
+RUNNING
 -------
 
 This script "lua2dox.lua" gets called by "gen_vimdoc.py".
 
+DEBUGGING/DEVELOPING
+---------------------
+
+1. To debug, run gen_vimdoc.py with --keep-tmpfiles:
+   python3 scripts/gen_vimdoc.py -t treesitter --keep-tmpfiles
+2. The filtered result will be written to ./tmp-lua2dox-doc/….lua.c
+
 Doxygen must be on your system. You can experiment like so:
 
 - Run "doxygen -g" to create a default Doxyfile.
@@ -41,498 +48,395 @@ It only has to be good enough for doxygen to see it as legal.
 
 One limitation is that each line is treated separately (except for long comments).
 The implication is that class and function declarations must be on the same line.
-Some functions can have their parameter lists extended over multiple lines to make it look neat.
-Managing this where there are also some comments is a bit more coding than I want to do at this stage,
-so it will probably not document accurately if we do do this.
 
-However I have put in a hack that will insert the "missing" close paren.
+There is hack that will insert the "missing" close paren.
 The effect is that you will get the function documented, but not with the parameter list you might expect.
 ]]
 
-local function class()
-  local newClass = {} -- a new class newClass
-  -- the class will be the metatable for all its newInstanceects,
-  -- and they will look up their methods in it.
-  newClass.__index = newClass
-
-  -- expose a constructor which can be called by <classname>(<args>)
-  setmetatable(newClass, {
-    __call = function(class_tbl, ...)
-      local newInstance = {}
-      setmetatable(newInstance, newClass)
-      --if init then
-      --  init(newInstance,...)
-      if class_tbl.init then
-        class_tbl.init(newInstance, ...)
-      end
-      return newInstance
-    end
-  })
-  return newClass
-end
+local TYPES = { 'integer', 'number', 'string', 'table', 'list', 'boolean', 'function' }
+
+local TAGGED_TYPES = { 'TSNode', 'LanguageTree' }
 
---! \brief write to stdout
-local function TCore_IO_write(Str)
-  if Str then
-    io.write(Str)
+-- Document these as 'table'
+local ALIAS_TYPES = {
+  'Range', 'Range4', 'Range6', 'TSMetadata',
+  'vim.filetype.add.filetypes',
+  'vim.filetype.match.args'
+}
+
+local debug_outfile = nil --- @type string?
+local debug_output = {}
+
+--- write to stdout
+--- @param str? string
+local function write(str)
+  if not str then
+    return
   end
-end
 
---! \brief write to stdout
-local function TCore_IO_writeln(Str)
-  if Str then
-    io.write(Str)
+  io.write(str)
+  if debug_outfile then
+    table.insert(debug_output, str)
   end
-  io.write('\n')
 end
 
---! \brief trims a string
-local function string_trim(Str)
-  return Str:match('^%s*(.-)%s*$')
+--- write to stdout
+--- @param str? string
+local function writeln(str)
+  write(str)
+  write('\n')
 end
 
---! \brief split a string
---!
---! \param Str
---! \param Pattern
---! \returns table of string fragments
----@return string[]
-local function string_split(Str, Pattern)
-  local splitStr = {}
-  local fpat = '(.-)' .. Pattern
-  local last_end = 1
-  local str, e, cap = string.find(Str, fpat, 1)
-  while str do
-    if str ~= 1 or cap ~= '' then
-      table.insert(splitStr, cap)
-    end
-    last_end = e + 1
-    str, e, cap = string.find(Str, fpat, last_end)
-  end
-  if last_end <= #Str then
-    cap = string.sub(Str, last_end)
-    table.insert(splitStr, cap)
+--- an input file buffer
+--- @class StreamRead
+--- @field currentLine string?
+--- @field contentsLen integer
+--- @field currentLineNo integer
+--- @field filecontents string[]
+local StreamRead = {}
+
+--- @return StreamRead
+--- @param filename string
+function StreamRead.new(filename)
+  assert(filename, ('invalid file: %s'):format(filename))
+  -- get lines from file
+  -- syphon lines to our table
+  local filecontents = {} --- @type string[]
+  for line in io.lines(filename) do
+    filecontents[#filecontents+1] = line
   end
-  return splitStr
+
+  return setmetatable({
+    filecontents = filecontents,
+    contentsLen = #filecontents,
+    currentLineNo = 1,
+  }, { __index = StreamRead })
 end
 
--------------------------------
---! \brief file buffer
---!
---! an input file buffer
-local TStream_Read = class()
-
---! \brief get contents of file
---!
---! \param Filename name of file to read (or nil == stdin)
-function TStream_Read.getContents(this, Filename)
-  assert(Filename, ('invalid file: %s'):format(Filename))
-  -- get lines from file
-  -- syphon lines to our table
-  local filecontents = {}
-  for line in io.lines(Filename) do
-    table.insert(filecontents, line)
+-- get a line
+function StreamRead:getLine()
+  if self.currentLine then
+    self.currentLine = nil
+    return self.currentLine
   end
 
-  if filecontents then
-    this.filecontents = filecontents
-    this.contentsLen = #filecontents
-    this.currentLineNo = 1
+  -- get line
+  if self.currentLineNo <= self.contentsLen then
+    local line = self.filecontents[self.currentLineNo]
+    self.currentLineNo = self.currentLineNo + 1
+    return line
   end
 
-  return filecontents
+  return ''
 end
 
---! \brief get lineno
-function TStream_Read.getLineNo(this)
-  return this.currentLineNo
+-- save line fragment
+--- @param line_fragment string
+function StreamRead:ungetLine(line_fragment)
+  self.currentLine = line_fragment
 end
 
---! \brief get a line
-function TStream_Read.getLine(this)
-  local line
-  if this.currentLine then
-    line = this.currentLine
-    this.currentLine = nil
-  else
-    -- get line
-    if this.currentLineNo <= this.contentsLen then
-      line = this.filecontents[this.currentLineNo]
-      this.currentLineNo = this.currentLineNo + 1
-    else
-      line = ''
-    end
-  end
-  return line
+-- is it eof?
+function StreamRead:eof()
+  return not self.currentLine and self.currentLineNo > self.contentsLen
 end
 
---! \brief save line fragment
-function TStream_Read.ungetLine(this, LineFrag)
-  this.currentLine = LineFrag
+-- input filter
+--- @class Lua2DoxFilter
+local Lua2DoxFilter = {}
+setmetatable(Lua2DoxFilter, { __index = Lua2DoxFilter })
+
+--- trim comment off end of string
+---
+--- @param line string
+--- @return string, string?
+local function removeCommentFromLine(line)
+  local pos_comment = line:find('%-%-')
+  if not pos_comment then
+    return line
+  end
+  return line:sub(1, pos_comment - 1), line:sub(pos_comment)
 end
 
---! \brief is it eof?
-function TStream_Read.eof(this)
-  if this.currentLine or this.currentLineNo <= this.contentsLen then
-    return false
+--- Processes "@…" directives in a docstring line.
+---
+--- @param line string
+--- @param generics table<string,string>
+--- @return string?
+local function process_magic(line, generics)
+  line = line:gsub('^%s+@', '@')
+  line = line:gsub('@package', '@private')
+  line = line:gsub('@nodoc', '@private')
+
+  if not vim.startswith(line, '@') then -- it's a magic comment
+    return '/// ' .. line
   end
-  return true
-end
 
---! \brief output stream
-local TStream_Write = class()
+  local magic = line:sub(2)
+  local magic_split = vim.split(magic, ' ', { plain = true })
+  local directive = magic_split[1]
 
---! \brief constructor
-function TStream_Write.init(this)
-  this.tailLine = {}
-end
+  if vim.list_contains({
+    'cast', 'diagnostic', 'overload', 'meta', 'type'
+  }, directive) then
+    -- Ignore LSP directives
+    return '// gg:"' .. line .. '"'
+  end
 
---! \brief write immediately
-function TStream_Write.write(_, Str)
-  TCore_IO_write(Str)
-end
+  if directive == 'defgroup' or directive == 'addtogroup' then
+    -- Can't use '.' in defgroup, so convert to '--'
+    return '/// @' .. magic:gsub('%.', '-dot-')
+  end
 
---! \brief write immediately
-function TStream_Write.writeln(_, Str)
-  TCore_IO_writeln(Str)
-end
+  if directive == 'generic' then
+    local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)')
+    if generic_type == '' then
+      generic_type = 'any'
+    end
+    generics[generic_name] = generic_type
+    return
+  end
 
---! \brief write immediately
-function TStream_Write.writelnComment(_, Str)
-  TCore_IO_write('// ZZ: ')
-  TCore_IO_writeln(Str)
-end
+  local type_index = 2
 
---! \brief write to tail
-function TStream_Write.writelnTail(this, Line)
-  if not Line then
-    Line = ''
+  if directive == 'param' then
+    for _, type in ipairs(TYPES) do
+      magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2')
+      magic =
+        magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2')
+    end
+    magic_split = vim.split(magic, ' ', { plain = true })
+    type_index = 3
+  elseif directive == 'return' then
+    for _, type in ipairs(TYPES) do
+      magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1')
+      magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1')
+    end
+    -- Remove first "#" comment char, if any. https://github.com/LuaLS/lua-language-server/wiki/Annotations#return
+    magic = magic:gsub('# ', '', 1)
+    -- handle the return of vim.spell.check
+    magic = magic:gsub('({.*}%[%])', '`%1`')
+    magic_split = vim.split(magic, ' ', { plain = true })
   end
-  table.insert(this.tailLine, Line)
-end
 
---! \brief output tail lines
-function TStream_Write.write_tailLines(this)
-  for _, line in ipairs(this.tailLine) do
-    TCore_IO_writeln(line)
-  end
-  TCore_IO_write('// Lua2DoX new eof')
-end
+  local ty = magic_split[type_index]
+
+  if ty then
+    -- fix optional parameters
+    if magic_split[2]:find('%?$') then
+      if not ty:find('nil') then
+        ty = ty  .. '|nil'
+      end
+      magic_split[2] = magic_split[2]:sub(1, -2)
+    end
 
---! \brief input filter
-local TLua2DoX_filter = class()
+    -- replace generic types
+    for k, v in pairs(generics) do
+      ty = ty:gsub(k, v) --- @type string
+    end
 
---! \brief allow us to do errormessages
-function TLua2DoX_filter.warning(this, Line, LineNo, Legend)
-  this.outStream:writelnTail(
-    '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"'
-  )
-end
+    for _, type in ipairs(TAGGED_TYPES) do
+      ty = ty:gsub(type, '|%1|')
+    end
+
+    for _, type in ipairs(ALIAS_TYPES) do
+      ty = ty:gsub('^'..type..'$', 'table') --- @type string
+    end
+
+    -- surround some types by ()
+    for _, type in ipairs(TYPES) do
+      ty = ty
+        :gsub('^(' .. type .. '|nil):?$', '(%1)')
+        :gsub('^(' .. type .. '):?$', '(%1)')
+    end
+
+    magic_split[type_index] = ty
 
---! \brief trim comment off end of string
---!
---! If the string has a comment on the end, this trims it off.
---!
-local function TString_removeCommentFromLine(Line)
-  local pos_comment = string.find(Line, '%-%-')
-  local tailComment
-  if pos_comment then
-    Line = string.sub(Line, 1, pos_comment - 1)
-    tailComment = string.sub(Line, pos_comment)
   end
-  return Line, tailComment
+
+  magic = table.concat(magic_split, ' ')
+
+  return '/// @' .. magic
 end
 
---! \brief get directive from magic
-local function getMagicDirective(Line)
-  local macro, tail
-  local macroStr = '[\\@]'
-  local pos_macro = string.find(Line, macroStr)
-  if pos_macro then
-    --! ....\\ macro...stuff
-    --! ....\@ macro...stuff
-    local line = string.sub(Line, pos_macro + 1)
-    local space = string.find(line, '%s+')
-    if space then
-      macro = string.sub(line, 1, space - 1)
-      tail = string_trim(string.sub(line, space + 1))
+--- @param line string
+--- @param in_stream StreamRead
+--- @return string
+local function process_block_comment(line, in_stream)
+  local comment_parts = {} --- @type string[]
+  local done --- @type boolean?
+
+  while not done and not in_stream:eof() do
+    local thisComment --- @type string?
+    local closeSquare = line:find(']]')
+    if not closeSquare then -- need to look on another line
+      thisComment = line .. '\n'
+      line = in_stream:getLine()
     else
-      macro = line
-      tail = ''
+      thisComment = line:sub(1, closeSquare - 1)
+      done = true
+
+      -- unget the tail of the line
+      -- in most cases it's empty. This may make us less efficient but
+      -- easier to program
+      in_stream:ungetLine(vim.trim(line:sub(closeSquare + 2)))
     end
+    comment_parts[#comment_parts+1] = thisComment
   end
-  return macro, tail
+
+  local comment = table.concat(comment_parts)
+
+  if comment:sub(1, 1) == '@' then -- it's a long magic comment
+    return '/*' .. comment .. '*/  '
+  end
+
+  -- discard
+  return '/* zz:' .. comment .. '*/  '
 end
 
---! \brief check comment for fn
-local function checkComment4fn(Fn_magic, MagicLines)
-  local fn_magic = Fn_magic
-  --    TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"')
+--- @param line string
+--- @return string
+local function process_function_header(line)
+  local pos_fn = assert(line:find('function'))
+  -- we've got a function
+  local fn = removeCommentFromLine(vim.trim(line:sub(pos_fn + 8)))
+
+  if fn:sub(1, 1) == '(' then
+    -- it's an anonymous function
+    return '// ZZ: '..line
+  end
+  -- fn has a name, so is interesting
+
+  -- want to fix for iffy declarations
+  if fn:find('[%({]') then
+    -- we might have a missing close paren
+    if not fn:find('%)') then
+      fn = fn .. ' ___MissingCloseParenHere___)'
+    end
+  end
 
-  local magicLines = string_split(MagicLines, '\n')
+  -- Big hax
+  if fn:find(':') then
+    fn = fn:gsub(':', '.', 1)
 
-  local macro, tail
+    local paren_start = fn:find('(', 1, true)
+    local paren_finish = fn:find(')', 1, true)
 
-  for _, line in ipairs(magicLines) do
-    macro, tail = getMagicDirective(line)
-    if macro == 'fn' then
-      fn_magic = tail
-      --    TCore_IO_writeln('// found fn "' .. fn_magic .. '"')
-      --else
-      --TCore_IO_writeln('// not found fn "' .. line .. '"')
+    -- Nothing in between the parens
+    local comma --- @type string
+    if paren_finish == paren_start + 1 then
+      comma = ''
+    else
+      comma = ', '
     end
+
+    fn = fn:sub(1, paren_start)
+      .. 'self'
+      .. comma
+      .. fn:sub(paren_start + 1)
+  end
+
+  if line:match('local') then
+    -- Special: tell gen_vimdoc.py this is a local function.
+    return 'local_function ' .. fn .. '{}'
   end
 
-  return fn_magic
+  -- add vanilla function
+  return 'function ' .. fn .. '{}'
 end
 
-local types = { 'number', 'string', 'table', 'list', 'boolean', 'function' }
-
---! \brief run the filter
-function TLua2DoX_filter.readfile(this, AppStamp, Filename)
-  local inStream = TStream_Read()
-  local outStream = TStream_Write()
-  this.outStream = outStream -- save to this obj
-
-  if inStream:getContents(Filename) then
-    -- output the file
-    local line
-    local fn_magic -- function name/def from  magic comment
-
-    outStream:writelnTail('// #######################')
-    outStream:writelnTail('// app run:' .. AppStamp)
-    outStream:writelnTail('// #######################')
-    outStream:writelnTail()
-
-    local state = '' -- luacheck: ignore 231 variable is set but never accessed.
-    local offset = 0
-    local generic = {}
-    local l = 0
-    while not (inStream:eof()) do
-      line = string_trim(inStream:getLine())
-      l = l + 1
-      if string.sub(line, 1, 2) == '--' then -- it's a comment
-        -- Allow people to write style similar to EmmyLua (since they are basically the same)
-        -- instead of silently skipping things that start with ---
-        if string.sub(line, 3, 3) == '@' then -- it's a magic comment
-          offset = 0
-        elseif string.sub(line, 1, 4) == '---@' then -- it's a magic comment
-          offset = 1
-        end
-
-        if string.sub(line, 3, 3) == '@' or string.sub(line, 1, 4) == '---@' then -- it's a magic comment
-          state = 'in_magic_comment'
-          local magic = string.sub(line, 4 + offset)
-
-          local magic_split = string_split(magic, ' ')
-          if magic_split[1] == 'param' then
-            for _, type in ipairs(types) do
-              magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2')
-              magic =
-                magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2')
-            end
-            magic_split = string_split(magic, ' ')
-          elseif magic_split[1] == 'return' then
-            for _, type in ipairs(types) do
-              magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1')
-              magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1')
-            end
-            magic_split = string_split(magic, ' ')
-          end
-
-          if magic_split[1] == 'generic' then
-            local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)')
-            if generic_type == '' then
-              generic_type = 'any'
-            end
-            generic[generic_name] = generic_type
-          else
-            local type_index = 2
-            if magic_split[1] == 'param' then
-              type_index = type_index + 1
-            end
-
-            if magic_split[type_index] then
-              -- fix optional parameters
-              if magic_split[type_index] and magic_split[2]:find('%?$') then
-                if not magic_split[type_index]:find('nil') then
-                  magic_split[type_index] = magic_split[type_index] .. '|nil'
-                end
-                magic_split[2] = magic_split[2]:sub(1, -2)
-              end
-              -- replace generic types
-              if magic_split[type_index] then
-                for k, v in pairs(generic) do
-                  magic_split[type_index] = magic_split[type_index]:gsub(k, v)
-                end
-              end
-              -- surround some types by ()
-              for _, type in ipairs(types) do
-                magic_split[type_index] =
-                  magic_split[type_index]:gsub('^(' .. type .. '|nil):?$', '(%1)')
-                magic_split[type_index] =
-                  magic_split[type_index]:gsub('^(' .. type .. '):?$', '(%1)')
-              end
-            end
-
-            magic = table.concat(magic_split, ' ')
-
-            outStream:writeln('/// @' .. magic)
-            fn_magic = checkComment4fn(fn_magic, magic)
-          end
-        elseif string.sub(line, 3, 3) == '-' then -- it's a nonmagic doc comment
-          local comment = string.sub(line, 4)
-          outStream:writeln('/// ' .. comment)
-        elseif string.sub(line, 3, 4) == '[[' then -- it's a long comment
-          line = string.sub(line, 5) -- nibble head
-          local comment = ''
-          local closeSquare, hitend, thisComment
-          while not hitend and (not inStream:eof()) do
-            closeSquare = string.find(line, ']]')
-            if not closeSquare then -- need to look on another line
-              thisComment = line .. '\n'
-              line = inStream:getLine()
-            else
-              thisComment = string.sub(line, 1, closeSquare - 1)
-              hitend = true
-
-              -- unget the tail of the line
-              -- in most cases it's empty. This may make us less efficient but
-              -- easier to program
-              inStream:ungetLine(string_trim(string.sub(line, closeSquare + 2)))
-            end
-            comment = comment .. thisComment
-          end
-          if string.sub(comment, 1, 1) == '@' then -- it's a long magic comment
-            outStream:write('/*' .. comment .. '*/  ')
-            fn_magic = checkComment4fn(fn_magic, comment)
-          else -- discard
-            outStream:write('/* zz:' .. comment .. '*/  ')
-            fn_magic = nil
-          end
-        -- TODO(justinmk): Uncomment this if we want "--" lines to continue the
-        --                 preceding magic ("---", "--@", …) lines.
-        -- elseif state == 'in_magic_comment' then  -- next line of magic comment
-        --   outStream:writeln('/// '.. line:sub(3))
-        else -- discard
-          outStream:writeln('// zz:"' .. line .. '"')
-          fn_magic = nil
-        end
-      elseif string.find(line, '^function') or string.find(line, '^local%s+function') then
-        generic = {}
-        state = 'in_function' -- it's a function
-        local pos_fn = string.find(line, 'function')
-        -- function
-        -- ....v...
-        if pos_fn then
-          -- we've got a function
-          local fn = TString_removeCommentFromLine(string_trim(string.sub(line, pos_fn + 8)))
-          if fn_magic then
-            fn = fn_magic
-          end
-
-          if string.sub(fn, 1, 1) == '(' then
-            -- it's an anonymous function
-            outStream:writelnComment(line)
-          else
-            -- fn has a name, so is interesting
-
-            -- want to fix for iffy declarations
-            local open_paren = string.find(fn, '[%({]')
-            if open_paren then
-              -- we might have a missing close paren
-              if not string.find(fn, '%)') then
-                fn = fn .. ' ___MissingCloseParenHere___)'
-              end
-            end
-
-            -- Big hax
-            if string.find(fn, ':') then
-              -- TODO: We need to add a first parameter of "SELF" here
-              -- local colon_place = string.find(fn, ":")
-              -- local name = string.sub(fn, 1, colon_place)
-              fn = fn:gsub(':', '.', 1)
-              outStream:writeln('/// @param self')
-
-              local paren_start = string.find(fn, '(', 1, true)
-              local paren_finish = string.find(fn, ')', 1, true)
-
-              -- Nothing in between the parens
-              local comma
-              if paren_finish == paren_start + 1 then
-                comma = ''
-              else
-                comma = ', '
-              end
-              fn = string.sub(fn, 1, paren_start)
-                .. 'self'
-                .. comma
-                .. string.sub(fn, paren_start + 1)
-            end
-
-            -- add vanilla function
-            outStream:writeln('function ' .. fn .. '{}')
-          end
-        else
-          this:warning(inStream:getLineNo(), 'something weird here')
-        end
-        fn_magic = nil -- mustn't inadvertently use it again
-
-      -- TODO: If we can make this learn how to generate these, that would be helpful.
-      -- elseif string.find(line, "^M%['.*'%] = function") then
-      --   state = 'in_function'  -- it's a function
-      --   outStream:writeln("function textDocument/publishDiagnostics(...){}")
-
-      --   fn_magic = nil -- mustn't inadvertently use it again
-      else
-        state = '' -- unknown
-        if #line > 0 then -- we don't know what this line means, so just comment it out
-          outStream:writeln('// zz: ' .. line)
-        else
-          outStream:writeln() -- keep this line blank
-        end
-      end
+--- @param line string
+--- @param in_stream StreamRead
+--- @param generics table<string,string>>
+--- @return string?
+local function process_line(line, in_stream, generics)
+  local line_raw = line
+  line = vim.trim(line)
+
+  if vim.startswith(line, '---') then
+    return process_magic(line:sub(4), generics)
+  end
+
+  if vim.startswith(line, '--'..'[[') then -- it's a long comment
+    return process_block_comment(line:sub(5), in_stream)
+  end
+
+  -- Hax... I'm sorry
+  -- M.fun = vim.memoize(function(...)
+  --   ->
+  -- function M.fun(...)
+  line = line:gsub('^(.+) = .*_memoize%([^,]+, function%((.*)%)$', 'function %1(%2)')
+
+  if line:find('^function') or line:find('^local%s+function') then
+    return process_function_header(line)
+  end
+
+  if not line:match('^local') then
+    local v = line_raw:match('^([A-Za-z][.a-zA-Z_]*)%s+%=')
+    if v and v:match('%.') then
+      -- Special: this lets gen_vimdoc.py handle tables.
+      return 'table '..v..'() {}'
     end
+  end
 
-    -- output the tail
-    outStream:write_tailLines()
-  else
-    outStream:writeln('!empty file')
+  if #line > 0 then -- we don't know what this line means, so just comment it out
+    return '// zz: ' .. line
   end
+
+  return ''
 end
 
---! \brief this application
-local TApp = class()
+-- Processes the file and writes filtered output to stdout.
+---@param filename string
+function Lua2DoxFilter:filter(filename)
+  local in_stream = StreamRead.new(filename)
 
---! \brief constructor
-function TApp.init(this)
-  this.timestamp = os.date('%c %Z', os.time())
-  this.name = 'Lua2DoX'
-  this.version = '0.2 20130128'
-  this.copyright = 'Copyright (c) Simon Dales 2012-13'
-end
+  local generics = {} --- @type table<string,string>
 
-function TApp.getRunStamp(this)
-  return this.name .. ' (' .. this.version .. ') ' .. this.timestamp
-end
+  while not in_stream:eof() do
+    local line = in_stream:getLine()
+
+    local out_line = process_line(line, in_stream, generics)
+
+    if not vim.startswith(vim.trim(line), '---') then
+      generics = {}
+    end
 
-function TApp.getVersion(this)
-  return this.name .. ' (' .. this.version .. ') '
+    if out_line then
+      writeln(out_line)
+    end
+  end
 end
 
-function TApp.getCopyright(this)
-  return this.copyright
+--- @class TApp
+--- @field timestamp string|osdate
+--- @field name string
+--- @field version string
+--- @field copyright string
+--- this application
+local TApp = {
+  timestamp = os.date('%c %Z', os.time()),
+  name = 'Lua2DoX',
+  version = '0.2 20130128',
+  copyright = 'Copyright (c) Simon Dales 2012-13'
+}
+
+setmetatable(TApp, { __index = TApp })
+
+function TApp:getRunStamp()
+  return self.name .. ' (' .. self.version .. ') ' .. self.timestamp
 end
 
-local This_app = TApp()
+function TApp:getVersion()
+  return self.name .. ' (' .. self.version .. ') '
+end
 
 --main
 
-local argv1 = arg[1]
-if argv1 == '--help' then
-  TCore_IO_writeln(This_app:getVersion())
-  TCore_IO_writeln(This_app:getCopyright())
-  TCore_IO_writeln([[
+if arg[1] == '--help' then
+  writeln(TApp:getVersion())
+  writeln(TApp.copyright)
+  writeln([[
   run as:
   nvim -l scripts/lua2dox.lua <param>
   --------------
@@ -540,16 +444,32 @@ if argv1 == '--help' then
   <filename> : interprets filename
   --version  : show version/copyright info
   --help     : this help text]])
-elseif argv1 == '--version' then
-  TCore_IO_writeln(This_app:getVersion())
-  TCore_IO_writeln(This_app:getCopyright())
-else
-  -- it's a filter
-  local appStamp = This_app:getRunStamp()
-  local filename = argv1
-
-  local filter = TLua2DoX_filter()
-  filter:readfile(appStamp, filename)
-end
+elseif arg[1] == '--version' then
+  writeln(TApp:getVersion())
+  writeln(TApp.copyright)
+else  -- It's a filter.
+  local filename = arg[1]
+
+  if arg[2] == '--outdir' then
+    local outdir = arg[3]
+    if type(outdir) ~= 'string' or (0 ~= vim.fn.filereadable(outdir) and 0 == vim.fn.isdirectory(outdir)) then
+      error(('invalid --outdir: "%s"'):format(tostring(outdir)))
+    end
+    vim.fn.mkdir(outdir, 'p')
+    debug_outfile = string.format('%s/%s.c', outdir, vim.fs.basename(filename))
+  end
+
+  Lua2DoxFilter:filter(filename)
+
+  -- output the tail
+  writeln('// #######################')
+  writeln('// app run:' .. TApp:getRunStamp())
+  writeln('// #######################')
+  writeln()
 
---eof
+  if debug_outfile then
+    local f = assert(io.open(debug_outfile, 'w'))
+    f:write(table.concat(debug_output))
+    f:close()
+  end
+end
diff --git a/scripts/pvscheck.sh b/scripts/pvscheck.sh
deleted file mode 100755
index 97757c0848..0000000000
--- a/scripts/pvscheck.sh
+++ /dev/null
@@ -1,495 +0,0 @@
-#!/bin/sh
-
-# Assume that "local" is available.
-# shellcheck disable=SC2039
-
-set -e
-# Note: -u causes problems with posh, it barks at “undefined” $@ when no
-# arguments provided.
-test -z "$POSH_VERSION" && set -u
-
-log_info() {
-  >&2 printf "pvscheck.sh: %s\n" "$@"
-}
-
-get_jobs_num() {
-  if [ -n "${TRAVIS:-}" ] ; then
-    # HACK: /proc/cpuinfo on Travis CI is misleading, so hardcode 1.
-    echo 1
-  else
-    echo $(( $(grep -c "^processor" /proc/cpuinfo) + 1 ))
-  fi
-}
-
-help() {
-  echo 'Usage:'
-  echo '  pvscheck.sh [--pvs URL] [--deps] [--environment-cc]'
-  echo '              [target-directory [branch]]'
-  echo '  pvscheck.sh [--pvs URL] [--recheck] [--environment-cc] [--update]'
-  echo '              [target-directory]'
-  echo '  pvscheck.sh [--pvs URL] --only-analyse [target-directory]'
-  echo '  pvscheck.sh [--pvs URL] --pvs-install {target-directory}'
-  echo '  pvscheck.sh --patch [--only-build]'
-  echo
-  echo '    --pvs: Fetch pvs-studio from URL.'
-  echo
-  echo '    --pvs detect: Auto-detect latest version (by scraping viva64.com).'
-  echo
-  echo '    --deps: (for regular run) Use top-level Makefile and build deps.'
-  echo '            Without this it assumes all dependencies are already'
-  echo '            installed.'
-  echo
-  echo '    --environment-cc: (for regular run and --recheck) Do not export'
-  echo '                      CC=clang. Build is still run with CFLAGS=-O0.'
-  echo
-  echo '    --only-build: (for --patch) Only patch files in ./build directory.'
-  echo
-  echo '    --pvs-install: Only install PVS-studio to the specified location.'
-  echo
-  echo '    --patch: patch sources in the current directory.'
-  echo '             Does not patch already patched files.'
-  echo '             Does not run analysis.'
-  echo
-  echo '    --recheck: run analysis on a prepared target directory.'
-  echo
-  echo '    --update: when rechecking first do a pull.'
-  echo
-  echo '    --only-analyse: run analysis on a prepared target directory '
-  echo '                    without building Neovim.'
-  echo
-  echo '    target-directory: Directory where build should occur.'
-  echo '                      Default: ../neovim-pvs'
-  echo
-  echo '    branch: Branch to check.'
-  echo '            Default: master.'
-}
-
-getopts_error() {
-  local msg="$1" ; shift
-  local do_help=
-  if test "$msg" = "--help" ; then
-    msg="$1" ; shift
-    do_help=1
-  fi
-  printf '%s\n' "$msg" >&2
-  if test -n "$do_help" ; then
-    printf '\n' >&2
-    help >&2
-  fi
-  echo 'return 1'
-  return 1
-}
-
-# Usage `eval "$(getopts_long long_defs -- positionals_defs -- "$@")"`
-#
-# long_defs: list of pairs of arguments like `longopt action`.
-# positionals_defs: list of arguments like `action`.
-#
-# `action` is a space-separated commands:
-#
-#   store_const [const] [varname] [default]
-#     Store constant [const] (default 1) (note: eval’ed) if argument is present
-#     (long options only). Assumes long option accepts no arguments.
-#   store [varname] [default]
-#     Store value. Assumes long option needs an argument.
-#   run {func} [varname] [default]
-#     Run function {func} and store its output to the [varname]. Assumes no
-#     arguments accepted (long options only).
-#   modify {func} [varname] [default]
-#     Like run, but assumes a single argument, passed to function {func} as $1.
-#
-#   All actions stores empty value if neither [varname] nor [default] are
-#   present. [default] is evaled by top-level `eval`, so be careful. Also note
-#   that no arguments may contain spaces, including [default] and [const].
-getopts_long() {
-  local positional=
-  local opt_bases=""
-  while test $# -gt 0 ; do
-    local arg="$1" ; shift
-    local opt_base=
-    local act=
-    local opt_name=
-    if test -z "$positional" ; then
-      if test "$arg" = "--" ; then
-        positional=0
-        continue
-      fi
-      act="$1" ; shift
-      opt_name="$(echo "$arg" | tr '-' '_')"
-      opt_base="longopt_$opt_name"
-    else
-      if test "$arg" = "--" ; then
-        break
-      fi
-      : $(( positional+=1 ))
-      act="$arg"
-      opt_name="arg_$positional"
-      opt_base="positional_$positional"
-    fi
-    opt_bases="$opt_bases $opt_base"
-    eval "local varname_$opt_base=$opt_name"
-    local i=0
-    for act_subarg in $act ; do
-      eval "local act_$(( i+=1 ))_$opt_base=\"\$act_subarg\""
-    done
-  done
-  # Process options
-  local positional=0
-  local force_positional=
-  while test $# -gt 0 ; do
-    local argument="$1" ; shift
-    local opt_base=
-    local has_equal=
-    local equal_arg=
-    local is_positional=
-    if test "$argument" = "--" ; then
-      force_positional=1
-      continue
-    elif test -z "$force_positional" && test "${argument#--}" != "$argument"
-    then
-      local opt_name="${argument#--}"
-      local opt_name_striparg="${opt_name%%=*}"
-      if test "$opt_name" = "$opt_name_striparg" ; then
-        has_equal=0
-      else
-        has_equal=1
-        equal_arg="${argument#*=}"
-        opt_name="$opt_name_striparg"
-      fi
-      # Use trailing x to prevent stripping newlines
-      opt_name="$(printf '%sx' "$opt_name" | tr '-' '_')"
-      opt_name="${opt_name%x}"
-      if test -n "$(printf '%sx' "$opt_name" | tr -d 'a-z_')" ; then
-        getopts_error "Option contains invalid characters: $opt_name"
-      fi
-      opt_base="longopt_$opt_name"
-    else
-      : $(( positional+=1 ))
-      opt_base="positional_$positional"
-      is_positional=1
-    fi
-    if test -n "$opt_base" ; then
-      eval "local occurred_$opt_base=1"
-
-      eval "local act_1=\"\${act_1_$opt_base:-}\""
-      eval "local varname=\"\${varname_$opt_base:-}\""
-      local need_val=
-      local func=
-      case "$act_1" in
-        (store_const)
-          eval "local const=\"\${act_2_${opt_base}:-1}\""
-          eval "local varname=\"\${act_3_${opt_base}:-$varname}\""
-          printf 'local %s=%s\n' "$varname" "$const"
-          ;;
-        (store)
-          eval "varname=\"\${act_2_${opt_base}:-$varname}\""
-          need_val=1
-          ;;
-        (run)
-          eval "func=\"\${act_2_${opt_base}}\""
-          eval "varname=\"\${act_3_${opt_base}:-$varname}\""
-          printf 'local %s="$(%s)"\n' "$varname" "$func"
-          ;;
-        (modify)
-          eval "func=\"\${act_2_${opt_base}}\""
-          eval "varname=\"\${act_3_${opt_base}:-$varname}\""
-          need_val=1
-          ;;
-        ("")
-          getopts_error --help "Wrong argument: $argument"
-          ;;
-      esac
-      if test -n "$need_val" ; then
-        local val=
-        if test -z "$is_positional" ; then
-          if test $has_equal = 1 ; then
-            val="$equal_arg"
-          else
-            if test $# -eq 0 ; then
-              getopts_error "Missing argument for $opt_name"
-            fi
-            val="$1" ; shift
-          fi
-        else
-          val="$argument"
-        fi
-        local escaped_val="'$(printf "%s" "$val" | sed "s/'/'\\\\''/g")'"
-        case "$act_1" in
-          (store)
-            printf 'local %s=%s\n' "$varname" "$escaped_val"
-            ;;
-          (modify)
-            printf 'local %s="$(%s %s)"\n' "$varname" "$func" "$escaped_val"
-            ;;
-        esac
-      fi
-    fi
-  done
-  # Print default values when no values were provided
-  local opt_base=
-  for opt_base in $opt_bases ; do
-    eval "local occurred=\"\${occurred_$opt_base:-}\""
-    if test -n "$occurred" ; then
-      continue
-    fi
-    eval "local act_1=\"\$act_1_$opt_base\""
-    eval "local varname=\"\$varname_$opt_base\""
-    case "$act_1" in
-      (store)
-        eval "local varname=\"\${act_2_${opt_base}:-$varname}\""
-        eval "local default=\"\${act_3_${opt_base}:-}\""
-        printf 'local %s=%s\n' "$varname" "$default"
-        ;;
-      (store_const|run|modify)
-        eval "local varname=\"\${act_3_${opt_base}:-$varname}\""
-        eval "local default=\"\${act_4_${opt_base}:-}\""
-        printf 'local %s=%s\n' "$varname" "$default"
-        ;;
-    esac
-  done
-}
-
-get_pvs_comment() {
-  local tgt="$1" ; shift
-
-  cat > "$tgt/pvs-comment" << EOF
-// This is an open source non-commercial project. Dear PVS-Studio, please check
-// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
-
-EOF
-}
-
-install_pvs() {(
-  local tgt="$1" ; shift
-  local pvs_url="$1" ; shift
-
-  cd "$tgt"
-
-  if test -d pvs-studio ; then
-    log_info 'install_pvs: "pvs-studio" directory already exists, skipping install'
-    return 0
-  fi
-
-  mkdir pvs-studio
-  cd pvs-studio
-
-  curl -L -o pvs-studio.tar.gz "$pvs_url"
-  tar xzf pvs-studio.tar.gz
-  rm pvs-studio.tar.gz
-  local pvsdir="$(find . -maxdepth 1 -mindepth 1)"
-  find "$pvsdir" -maxdepth 1 -mindepth 1 -exec mv '{}' . \;
-  rmdir "$pvsdir"
-)}
-
-create_compile_commands() {(
-  local tgt="$1" ; shift
-  local deps="$1" ; shift
-  local environment_cc="$1" ; shift
-
-  if test -z "$environment_cc" ; then
-    export CC=clang
-  fi
-  export CFLAGS=' -O0 '
-
-  if test -z "$deps" ; then
-    mkdir -p "$tgt/build"
-    (
-      cd "$tgt/build"
-
-      cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$PWD/root"
-      make -j"$(get_jobs_num)"
-    )
-  else
-    (
-      cd "$tgt"
-
-      make -j"$(get_jobs_num)" CMAKE_EXTRA_FLAGS=" -DCMAKE_INSTALL_PREFIX=$PWD/root -DCMAKE_BUILD_TYPE=Debug "
-    )
-  fi
-  find "$tgt/build/src/nvim/auto" -name '*.test-include.c' -delete
-)}
-
-# Warning: realdir below only cares about directories unlike realpath.
-#
-# realpath is not available in Ubuntu trusty yet.
-realdir() {(
-  local dir="$1"
-  local add=""
-  while ! cd "$dir" 2>/dev/null ; do
-    add="${dir##*/}/$add"
-    local new_dir="${dir%/*}"
-    if test "$new_dir" = "$dir" ; then
-      return 1
-    fi
-    dir="$new_dir"
-  done
-  printf '%s\n' "$PWD/$add"
-)}
-
-patch_sources() {(
-  local tgt="$1" ; shift
-  local only_build="${1}" ; shift
-
-  get_pvs_comment "$tgt"
-
-  local sh_script='
-    pvs_comment="$(cat pvs-comment ; echo -n EOS)"
-    filehead="$(head -c $(( ${#pvs_comment} - 3 )) "$1" ; echo -n EOS)"
-    if test "x$filehead" != "x$pvs_comment" ; then
-      cat pvs-comment "$1" > "$1.tmp"
-      mv "$1.tmp" "$1"
-    fi
-  '
-
-  cd "$tgt"
-
-  if test "$only_build" != "--only-build" ; then
-    find \
-      src/nvim test/functional/fixtures test/unit/fixtures \
-      \( -name '*.c' -a '!' -path '*xdiff*' \) \
-      -exec /bin/sh -c "$sh_script" - '{}' \;
-  fi
-
-  find \
-    build/src/nvim/auto build/config \
-    -name '*.c' -not -name '*.test-include.c' \
-    -exec /bin/sh -c "$sh_script" - '{}' \;
-
-  rm pvs-comment
-)}
-
-run_analysis() {(
-  local tgt="$1" ; shift
-
-  cd "$tgt"
-
-  if [ ! -r PVS-Studio.lic ]; then
-    pvs-studio-analyzer credentials -o PVS-Studio.lic 'PVS-Studio Free' 'FREE-FREE-FREE-FREE'
-  fi
-
-  # pvs-studio-analyzer exits with a non-zero exit code when there are detected
-  # errors, so ignore its return
-  pvs-studio-analyzer \
-    analyze \
-      --lic-file PVS-Studio.lic \
-      --threads "$(get_jobs_num)" \
-      --exclude-path src/cjson \
-      --exclude-path src/xdiff \
-      --output-file PVS-studio.log \
-      --file build/compile_commands.json \
-      --sourcetree-root . || true
-
-  rm -rf PVS-studio.{xml,err,tsk,html.d}
-  local plog_args="PVS-studio.log --srcRoot . --excludedCodes V002,V011,V1028,V1042,V1051,V1074"
-  plog-converter $plog_args --renderTypes xml       --output PVS-studio.xml
-  plog-converter $plog_args --renderTypes errorfile --output PVS-studio.err
-  plog-converter $plog_args --renderTypes tasklist  --output PVS-studio.tsk
-  plog-converter $plog_args --renderTypes fullhtml  --output PVS-studio.html.d
-)}
-
-detect_url() {
-  local url="${1:-detect}"
-  if test "$url" = detect ; then
-    curl --silent -L 'https://pvs-studio.com/en/pvs-studio/download-all/' \
-    | grep -o 'https\{0,1\}://[^"<>]\{1,\}/pvs-studio[^/"<>]*-x86_64\.tgz' \
-    || echo FAILED
-  else
-    printf '%s' "$url"
-  fi
-}
-
-do_check() {
-  local tgt="$1" ; shift
-  local branch="$1" ; shift
-  local pvs_url="$1" ; shift
-  local deps="$1" ; shift
-  local environment_cc="$1" ; shift
-
-  if test -z "$pvs_url" || test "$pvs_url" = FAILED ; then
-    pvs_url="$(detect_url detect)"
-    if test -z "$pvs_url" || test "$pvs_url" = FAILED ; then
-      echo "failed to auto-detect PVS URL"
-      exit 1
-    fi
-    echo "Auto-detected PVS URL: ${pvs_url}"
-  fi
-
-  git clone --branch="$branch" . "$tgt"
-
-  install_pvs "$tgt" "$pvs_url"
-
-  do_recheck "$tgt" "$deps" "$environment_cc" ""
-}
-
-do_recheck() {
-  local tgt="$1" ; shift
-  local deps="$1" ; shift
-  local environment_cc="$1" ; shift
-  local update="$1" ; shift
-
-  if test -n "$update" ; then
-    (
-      cd "$tgt"
-      local branch="$(git rev-parse --abbrev-ref HEAD)"
-      git checkout --detach
-      git fetch -f origin "${branch}:${branch}"
-      git checkout -f "$branch"
-    )
-  fi
-
-  create_compile_commands "$tgt" "$deps" "$environment_cc"
-
-  do_analysis "$tgt"
-}
-
-do_analysis() {
-  local tgt="$1" ; shift
-
-  if test -d "$tgt/pvs-studio" ; then
-    local saved_pwd="$PWD"
-    cd "$tgt/pvs-studio"
-    export PATH="$PWD/bin${PATH+:}${PATH}"
-    cd "$saved_pwd"
-  fi
-
-  run_analysis "$tgt"
-}
-
-main() {
-  eval "$(
-    getopts_long \
-      help store_const \
-      pvs 'modify detect_url pvs_url' \
-      patch store_const \
-      only-build 'store_const --only-build' \
-      recheck store_const \
-      only-analyse store_const \
-      pvs-install store_const \
-      deps store_const \
-      environment-cc store_const \
-      update store_const \
-      -- \
-      'modify realdir tgt "$PWD/../neovim-pvs"' \
-      'store branch master' \
-      -- "$@"
-  )"
-
-  if test -n "$help" ; then
-    help
-    return 0
-  fi
-
-  if test -n "$patch" ; then
-    patch_sources "$tgt" "$only_build"
-  elif test -n "$pvs_install" ; then
-    install_pvs "$tgt" "$pvs_url"
-  elif test -n "$recheck" ; then
-    do_recheck "$tgt" "$deps" "$environment_cc" "$update"
-  elif test -n "$only_analyse" ; then
-    do_analysis "$tgt"
-  else
-    do_check "$tgt" "$branch" "$pvs_url" "$deps" "$environment_cc"
-  fi
-}
-
-main "$@"
diff --git a/scripts/update_terminfo.sh b/scripts/update_terminfo.sh
index 775048f246..e12365ba8f 100755
--- a/scripts/update_terminfo.sh
+++ b/scripts/update_terminfo.sh
@@ -61,17 +61,11 @@ print_bold "[*] Writing $target... "
 sorted_terms="$(echo "${!entries[@]}" | tr ' ' '\n' | sort | xargs)"
 
 cat > "$target" <<EOF
-// This is an open source non-commercial project. Dear PVS-Studio, please check
-// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
-
 // uncrustify:off
 
-//
 // Generated by scripts/update_terminfo.sh and $(tic -V)
-//
 
-#ifndef NVIM_TUI_TERMINFO_DEFS_H
-#define NVIM_TUI_TERMINFO_DEFS_H
+#pragma once
 
 #include <stdint.h>
 EOF
@@ -90,7 +84,4 @@ for term in $sorted_terms; do
   printf '};\n'
 done >> "$target"
 
-cat >> "$target" <<EOF
-#endif  // NVIM_TUI_TERMINFO_DEFS_H
-EOF
 print_bold 'done\n'
diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh
index f9f7330097..47c6d293bc 100755
--- a/scripts/vim-patch.sh
+++ b/scripts/vim-patch.sh
@@ -119,10 +119,10 @@ get_vim_sources() {
 }
 
 commit_message() {
-  if [[ -n "$vim_tag" ]]; then
-    printf '%s\n\n%s\n\n%s' "${vim_message}" "${vim_commit_url}" "${vim_coauthor}"
+  if [[ "${vim_message}" == "vim-patch:${vim_version}:"* ]]; then
+    printf '%s\n\n%s\n\n%s' "${vim_message}" "${vim_commit_url}" "${vim_coauthors}"
   else
-    printf 'vim-patch:%s\n\n%s\n\n%s\n\n%s' "$vim_version" "$vim_message" "$vim_commit_url" "$vim_coauthor"
+    printf 'vim-patch:%s\n\n%s\n\n%s\n\n%s' "$vim_version" "$vim_message" "$vim_commit_url" "$vim_coauthors"
   fi
 }
 
@@ -174,11 +174,17 @@ assign_commit_details() {
 
   vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}"
   vim_message="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:%B' "${vim_commit}" \
-      | sed -e 's/\(#[0-9]\{1,\}\)/vim\/vim\1/g')"
-  vim_coauthor="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:Co-authored-by: %an <%ae>' "${vim_commit}")"
+      | sed -Ee 's/(#[0-9]{1,})/vim\/vim\1/g')"
+  local vim_coauthor0
+  vim_coauthor0="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:Co-authored-by: %an <%ae>' "${vim_commit}")"
+  # Extract co-authors from the commit message.
+  vim_coauthors="$(echo "${vim_message}" | (grep -E '^Co-authored-by: ' || true) | (grep -Fxv "${vim_coauthor0}" || true))"
+  vim_coauthors="$(echo "${vim_coauthor0}"; echo "${vim_coauthors}")"
+  # Remove Co-authored-by and Signed-off-by lines from the commit message.
+  vim_message="$(echo "${vim_message}" | grep -Ev '^(Co-authored|Signed-off)-by: ')"
   if [[ ${munge_commit_line} == "true" ]]; then
     # Remove first line of commit message.
-    vim_message="$(echo "${vim_message}" | sed -e '1s/^patch /vim-patch:/')"
+    vim_message="$(echo "${vim_message}" | sed -Ee '1s/^patch /vim-patch:/')"
   fi
   patch_file="vim-${vim_version}.patch"
 }
@@ -190,100 +196,113 @@ preprocess_patch() {
 
   # Remove Filelist, README
   local na_files='Filelist\|README.*'
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/\<\%('"${na_files}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/\<\%('"${na_files}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove *.proto, Make*, INSTALL*, gui_*, beval.*, some if_*, gvim, libvterm, tee, VisVim, xpm, xxd
   local na_src='auto\|configure.*\|GvimExt\|hardcopy.*\|libvterm\|proto\|tee\|VisVim\|xpm\|xxd\|Make.*\|INSTALL.*\|beval.*\|gui.*\|if_cscop\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv'
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%(testdir/\)\@<!\%('"${na_src}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%(testdir/\)\@<!\%('"${na_src}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
-  # Remove runtime files ported to Lua.
-  local na_rt='filetype\.vim\|scripts\.vim\|autoload\/ft\/dist\.vim\|print\/.*'
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('"${na_rt}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  # Remove runtime/print/
+  local na_rt='print\/.*'
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('"${na_rt}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove unwanted Vim doc files.
   local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|print\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags'
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('"${na_doc}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('"${na_doc}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove "Last change ..." changes in doc files.
   2>/dev/null $nvim --cmd 'set dir=/tmp' +'%s/^@@.*\n.*For Vim version.*Last change.*\n.*For Vim version.*Last change.*//' +w +q "$file"
 
   # Remove gui, option, setup, screen dumps, testdir/Make_*.mak files
   local na_src_testdir='gen_opt_test\.vim\|gui_.*\|Make_amiga\.mak\|Make_dos\.mak\|Make_ming\.mak\|Make_vms\.mms\|dumps/.*\.dump\|setup_gui\.vim'
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<\%('"${na_src_testdir}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<\%('"${na_src_testdir}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove testdir/test_*.vim files
-  local na_src_testdir='balloon.*\|channel.*\|crypt\.vim\|cscope\.vim\|gui.*\|hardcopy\.vim\|job_fails\.vim\|json\.vim\|mzscheme\.vim\|netbeans.*\|paste\.vim\|popupwin.*\|restricted\.vim\|shortpathname\.vim\|tcl\.vim\|terminal.*\|xxd\.vim'
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<test_\%('"${na_src_testdir}"'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  local na_src_testdir='balloon.*\|behave\.vim\|channel.*\|crypt\.vim\|cscope\.vim\|gui.*\|hardcopy\.vim\|job_fails\.vim\|json\.vim\|listener\.vim\|mzscheme\.vim\|netbeans.*\|paste\.vim\|popupwin.*\|python2\.vim\|pyx2\.vim\|restricted\.vim\|shortpathname\.vim\|sound\.vim\|tcl\.vim\|terminal.*\|xxd\.vim'
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<test_\%('"${na_src_testdir}"'\)\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
-  # Remove version.c #7555
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\<\%(version\.c\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  # Remove N/A src/*.[ch] files: sound.c, version.c
+  local na_src_c='sound\|version'
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\<\%('"${na_src_c}"'\)\.[ch]\>@exe "norm! d/\\v(^diff)|%$\r"' +w +q "$file"
 
   # Remove some *.po files. #5622
   local na_po='sjiscorr\.c\|ja\.sjis\.po\|ko\.po\|pl\.cp1250\.po\|pl\.po\|ru\.cp1251\.po\|uk\.cp1251\.po\|zh_CN\.cp936\.po\|zh_CN\.po\|zh_TW\.po'
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/po/\<\%('${na_po}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/po/\<\%('${na_po}'\)\>@exe "norm! d/\\v(^diff)|%$\r"+' +w +q "$file"
 
   # Remove vimrc_example.vim
   local na_vimrcexample='vimrc_example\.vim'
-  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('${na_vimrcexample}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file"
+  2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('${na_vimrcexample}'\)\>@exe "norm! d/\\v(^diff)|%$\r"+' +w +q "$file"
+
+  # Rename src/testdir/ paths to test/old/testdir/
+  LC_ALL=C sed -Ee 's/( [ab])\/src\/testdir/\1\/test\/old\/testdir/g' \
+    "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename src/ paths to src/nvim/
-  LC_ALL=C sed -e 's/\( [ab]\/src\)/\1\/nvim/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src)/\1\/nvim/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename evalfunc.c to eval/funcs.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/evalfunc\.c/\1\/eval\/funcs\.c/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/evalfunc\.c/\1\/eval\/funcs.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename evalvars.c to eval/vars.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/evalvars\.c/\1\/eval\/vars\.c/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/evalvars\.c/\1\/eval\/vars.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename userfunc.c to eval/userfunc.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/userfunc\.c/\1\/eval\/userfunc\.c/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/userfunc\.c/\1\/eval\/userfunc.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename evalbuffer.c to eval/buffer.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/evalbuffer\.c/\1\/eval\/buffer\.c/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/evalbuffer\.c/\1\/eval\/buffer.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename evalwindow.c to eval/window.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/evalwindow\.c/\1\/eval\/window\.c/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/evalwindow\.c/\1\/eval\/window.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename map.c to mapping.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/map\(\.[ch]\)/\1\/mapping\2/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/map\.c/\1\/mapping.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename profiler.c to profile.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/profiler\(\.[ch]\)/\1\/profile\2/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/profiler\.c/\1\/profile.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename scriptfile.c to runtime.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/scriptfile\(\.[ch]\)/\1\/runtime\2/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/scriptfile\.c/\1\/runtime.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename session.c to ex_session.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/session\(\.[ch]\)/\1\/ex_session\2/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/session\.c/\1\/ex_session.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename highlight.c to highlight_group.c
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/highlight\(\.[ch]\)/\1\/highlight_group\2/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/highlight\.c/\1\/highlight_group.c/g' \
+    "$file" > "$file".tmp && mv "$file".tmp "$file"
+
+  # Rename locale.c to os/lang.c
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/locale\.c/\1\/os\/lang.c/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename keymap.h to keycodes.h
-  LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/keymap\.h/\1\/keycodes.h/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/keymap\.h/\1\/keycodes.h/g' \
+    "$file" > "$file".tmp && mv "$file".tmp "$file"
+
+  # Rename option.h to option_vars.h
+  LC_ALL=C sed -Ee 's/( [ab]\/src\/nvim)\/option\.h/\1\/option_vars.h/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename terminal.txt to nvim_terminal_emulator.txt
-  LC_ALL=C sed -e 's/\( [ab]\/runtime\/doc\)\/terminal\.txt/\1\/nvim_terminal_emulator.txt/g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/runtime\/doc)\/terminal\.txt/\1\/nvim_terminal_emulator.txt/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename test_urls.vim to check_urls.vim
-  LC_ALL=C sed -e 's@\( [ab]\)/runtime/doc/test\(_urls\.vim\)@\1/scripts/check\2@g' \
+  LC_ALL=C sed -Ee 's/( [ab])\/runtime\/doc\/test(_urls\.vim)/\1\/scripts\/check\2/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 
   # Rename path to check_colors.vim
-  LC_ALL=C sed -e 's@\( [ab]/runtime\)/colors/\(tools/check_colors\.vim\)@\1/\2@g' \
+  LC_ALL=C sed -Ee 's/( [ab]\/runtime)\/colors\/(tools\/check_colors\.vim)/\1\/\2/g' \
     "$file" > "$file".tmp && mv "$file".tmp "$file"
 }
 
@@ -293,7 +312,7 @@ uncrustify_patch() {
     exit 1
   }
 
-  local patch_path=$NVIM_SOURCE_DIR/build/vim_patch
+  local patch_path="$NVIM_SOURCE_DIR"/build/vim_patch
   rm -rf "$patch_path"
   mkdir -p "$patch_path"/{a,b}
 
@@ -310,7 +329,7 @@ uncrustify_patch() {
   # than once. This is obviously a bug that needs to be fixed on uncrustify's
   # end, but in the meantime this workaround is sufficient.
   for _ in {1..2}; do
-    uncrustify -c "$NVIM_SOURCE_DIR"/src/uncrustify.cfg -q --replace --no-backup "$patch_path"/{a,b}/src/*.[ch]
+    "$NVIM_SOURCE_DIR"/build/usr/bin/uncrustify -c "$NVIM_SOURCE_DIR"/src/uncrustify.cfg -q --replace --no-backup "$patch_path"/{a,b}/src/*.[ch]
   done
 
   (cd "$patch_path" && (git --no-pager diff --no-index --no-prefix --patch --unified=5 --color=never a/ b/ || true))
@@ -324,7 +343,7 @@ get_vimpatch() {
   msg_ok "Found Vim revision '${vim_commit}'."
 
   local patch_content
-  if check_executable uncrustify; then
+  if check_executable "$NVIM_SOURCE_DIR"/build/usr/bin/uncrustify; then
     patch_content="$(uncrustify_patch "${vim_commit}")"
   else
     patch_content="$(git --no-pager show --unified=5 --color=never -1 --pretty=medium "${vim_commit}")"
@@ -522,7 +541,7 @@ list_vimpatch_tokens() {
     | grep -oE 'vim-patch:[^ ,{:]{7,}' \
     | sort \
     | uniq \
-    | sed -nE 's/^(vim-patch:([0-9]+\.[^ ]+|[0-9a-z]{7,7})).*/\1/p'
+    | sed -nEe 's/^(vim-patch:([0-9]+\.[^ ]+|[0-9a-z]{7,7})).*/\1/p'
 }
 
 # Prints all patch-numbers (for the current v:version) for which there is
@@ -530,7 +549,7 @@ list_vimpatch_tokens() {
 list_vimpatch_numbers() {
   # Transform "vim-patch:X.Y.ZZZZ" to "ZZZZ".
   list_vimpatch_tokens | while read -r vimpatch_token; do
-    echo "$vimpatch_token" | grep '8\.1\.' | sed -E 's/.*vim-patch:8\.1\.([0-9a-z]+).*/\1/'
+    echo "$vimpatch_token" | grep -F '8.1.' | sed -Ee 's/.*vim-patch:8\.1\.([0-9a-z]+).*/\1/'
   done
 }
 
@@ -551,10 +570,8 @@ _set_tokens_and_tags() {
 
   # Create an associative array mapping Vim commits to tags.
   eval "vim_commit_tags=(
-    $(git -C "${VIM_SOURCE_DIR}" for-each-ref refs/tags \
-      --format '[%(objectname)]=%(refname:strip=2)' \
-      --sort='-*authordate' \
-      --shell)
+    $(git -C "${VIM_SOURCE_DIR}" show-ref --tags --dereference \
+      | sed -nEe 's/^([0-9a-f]+) refs\/tags\/(v[0-9.]+)(\^\{\})?$/["\1"]="\2"/p')
   )"
   # Exit in case of errors from the above eval (empty vim_commit_tags).
   if ! (( "${#vim_commit_tags[@]}" )); then
@@ -594,6 +611,7 @@ _set_missing_vimpatches() {
   # Massage arguments for git-log.
   declare -A git_log_replacements=(
     [^\(.*/\)?src/nvim/\(.*\)]="\${BASH_REMATCH[1]}src/\${BASH_REMATCH[2]}"
+    [^\(.*/\)?test/old/\(.*\)]="\${BASH_REMATCH[1]}src/\${BASH_REMATCH[2]}"
     [^\(.*/\)?\.vim-src/\(.*\)]="\${BASH_REMATCH[2]}"
   )
   local i j
@@ -654,7 +672,7 @@ show_vimpatches() {
   printf "Vim patches missing from Neovim:\n"
 
   local -A runtime_commits
-  for commit in $(git -C "${VIM_SOURCE_DIR}" log --format="%H %D" -- runtime | sed 's/,\? tag: / /g'); do
+  for commit in $(git -C "${VIM_SOURCE_DIR}" log --format="%H %D" -- runtime | sed -Ee 's/,\? tag: / /g'); do
     runtime_commits[$commit]=1
   done
 
@@ -748,7 +766,7 @@ review_commit() {
   local nvim_patch
   nvim_patch="$(curl -Ssf "${nvim_patch_url}")"
   local vim_version
-  vim_version="$(head -n 4 <<< "${nvim_patch}" | sed -n 's/'"${git_patch_prefix}"'vim-patch:\([a-z0-9.]*\)\(:.*\)\{0,1\}$/\1/p')"
+  vim_version="$(head -n 4 <<< "${nvim_patch}" | sed -nEe 's/'"${git_patch_prefix}"'vim-patch:([a-z0-9.]*)(:.*){0,1}$/\1/p')"
 
   echo
   if [[ -n "${vim_version}" ]]; then