Merge remote-tracking branch 'upstream/master' into floattitlefloattitle

7 files changed, 1267 insertions, 700 deletions

diff --git a/scripts/bump-deps.sh b/scripts/bump-deps.sh
index 85c7f72700..e725608b39 100755
--- a/scripts/bump-deps.sh
+++ b/scripts/bump-deps.sh
@@ -17,9 +17,13 @@ BASENAME="$(basename "${0}")"
 readonly BASENAME
 
 usage() {
-  echo "Bump Neovim dependencies"
+  echo "Bump Nvim dependencies"
   echo
   echo "Usage:  ${BASENAME} [ -h | --pr | --branch=<dep> | --dep=<dependency> ]"
+  echo "    Update a dependency:"
+  echo "        ./scripts/bump-deps.sh --dep Luv --version 1.43.0-0"
+  echo "    Create a PR:"
+  echo "        ./scripts/bump-deps.sh --pr"
   echo
   echo "Options:"
   echo "    -h                    show this message and exit."
diff --git a/scripts/check-includes.py b/scripts/check-includes.py
deleted file mode 100755
index ed1fe407c5..0000000000
--- a/scripts/check-includes.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import re
-import os
-
-from subprocess import Popen, PIPE
-from argparse import ArgumentParser
-
-
-GENERATED_INCLUDE_RE = re.compile(
-    r'^\s*#\s*include\s*"([/a-z_0-9.]+\.generated\.h)"(\s+//.*)?$')
-
-
-def main(argv):
-    argparser = ArgumentParser()
-    argparser.add_argument('--generated-includes-dir', action='append',
-                           help='Directory where generated includes are located.')
-    argparser.add_argument('--file', type=open, help='File to check.')
-    argparser.add_argument('iwyu_args', nargs='*',
-                           help='IWYU arguments, must go after --.')
-    args = argparser.parse_args(argv)
-
-    with args.file:
-        iwyu = Popen(['include-what-you-use', '-xc'] + args.iwyu_args + ['/dev/stdin'],
-                     stdin=PIPE, stdout=PIPE, stderr=PIPE)
-
-        for line in args.file:
-            match = GENERATED_INCLUDE_RE.match(line)
-            if match:
-                for d in args.generated_includes_dir:
-                    try:
-                        f = open(os.path.join(d, match.group(1)))
-                    except IOError:
-                        continue
-                    else:
-                        with f:
-                            for generated_line in f:
-                                iwyu.stdin.write(generated_line)
-                            break
-                else:
-                    raise IOError('Failed to find {0}'.format(match.group(1)))
-            else:
-                iwyu.stdin.write(line)
-
-    iwyu.stdin.close()
-
-    out = iwyu.stdout.read()
-    err = iwyu.stderr.read()
-
-    ret = iwyu.wait()
-
-    if ret != 2:
-        print('IWYU failed with exit code {0}:'.format(ret))
-        print('{0} stdout {0}'.format('=' * ((80 - len(' stdout ')) // 2)))
-        print(out)
-        print('{0} stderr {0}'.format('=' * ((80 - len(' stderr ')) // 2)))
-        print(err)
-        return 1
-    return 0
-
-
-if __name__ == '__main__':
-    raise SystemExit(main(sys.argv[1:]))
diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua
new file mode 100644
index 0000000000..06ea1831b0
--- /dev/null
+++ b/scripts/gen_help_html.lua
@@ -0,0 +1,1042 @@
+-- Converts Vim :help files to HTML.  Validates |tag| links and document syntax (parser errors).
+--
+-- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags).
+--
+-- USAGE (GENERATE HTML):
+--   1. Run `make helptags` first; this script depends on vim.fn.taglist().
+--   2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')"
+--      - Read the docstring at gen().
+--   3. cd target/dir/ && jekyll serve --host 0.0.0.0
+--   4. Visit http://localhost:4000/…/help.txt.html
+--
+-- USAGE (VALIDATE):
+--   1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate()"
+--      - validate() is 10x faster than gen(), so it is used in CI.
+--
+-- SELF-TEST MODE:
+--   1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()"
+--
+-- NOTES:
+--   * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too
+--     slow (~1 min) to run in per-commit CI.
+--   * visit_node() is the core function used by gen() to traverse the document tree and produce HTML.
+--   * visit_validate() is the core function used by validate().
+--   * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout.
+
+local tagmap = nil
+local helpfiles = nil
+local invalid_links = {}
+local invalid_urls = {}
+local invalid_spelling = {}
+local spell_dict = {
+  Neovim = 'Nvim',
+  NeoVim = 'Nvim',
+  neovim = 'Nvim',
+  lua = 'Lua',
+  VimL = 'Vimscript',
+}
+
+local M = {}
+
+-- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs).
+-- All other files are "legacy" files which require fixed-width layout.
+local new_layout = {
+  ['api.txt'] = true,
+  ['channel.txt'] = true,
+  ['deprecated.txt'] = true,
+  ['develop.txt'] = true,
+  ['lua.txt'] = true,
+  ['luaref.txt'] = true,
+  ['news.txt'] = true,
+  ['nvim.txt'] = true,
+  ['pi_health.txt'] = true,
+  ['provider.txt'] = true,
+  ['ui.txt'] = true,
+}
+
+-- TODO: These known invalid |links| require an update to the relevant docs.
+local exclude_invalid = {
+  ["'previewpopup'"] = "quickref.txt",
+  ["'pvp'"] = "quickref.txt",
+  ["'string'"] = "eval.txt",
+  Query = "treesitter.txt",
+  ["eq?"] = "treesitter.txt",
+  ["lsp-request"] = "lsp.txt",
+  matchit = "vim_diff.txt",
+  ["matchit.txt"] = "help.txt",
+  ["set!"] = "treesitter.txt",
+  ["v:_null_blob"] = "builtin.txt",
+  ["v:_null_dict"] = "builtin.txt",
+  ["v:_null_list"] = "builtin.txt",
+  ["v:_null_string"] = "builtin.txt",
+  ["vim.lsp.buf_request()"] = "lsp.txt",
+  ["vim.lsp.util.get_progress_messages()"] = "lsp.txt",
+  ["vim.treesitter.start()"] = "treesitter.txt",
+}
+
+local function tofile(fname, text)
+  local f = io.open(fname, 'w')
+  if not f then
+    error(('failed to write: %s'):format(f))
+  else
+    f:write(text)
+    f:close()
+  end
+end
+
+local function html_esc(s)
+  return s:gsub(
+    '&', '&amp;'):gsub(
+    '<', '&lt;'):gsub(
+    '>', '&gt;')
+end
+
+local function url_encode(s)
+  -- Credit: tpope / vim-unimpaired
+  -- NOTE: these chars intentionally *not* escaped: ' ( )
+  return vim.fn.substitute(vim.fn.iconv(s, 'latin1', 'utf-8'),
+    [=[[^A-Za-z0-9()'_.~-]]=],
+    [=[\="%".printf("%02X",char2nr(submatch(0)))]=],
+    'g')
+end
+
+local function expandtabs(s)
+  return s:gsub('\t', (' '):rep(8))
+end
+
+local function to_titlecase(s)
+  local text = ''
+  for w in vim.gsplit(s, '[ \t]+') do
+    text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2))
+  end
+  return text
+end
+
+local function to_heading_tag(text)
+  -- Prepend "_" to avoid conflicts with actual :help tags.
+  return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown'
+end
+
+local function basename_noext(f)
+ return vim.fs.basename(f:gsub('%.txt', ''))
+end
+
+local function is_blank(s)
+  return not not s:find([[^[\t ]*$]])
+end
+
+local function trim(s, dir)
+  return vim.fn.trim(s, '\r\t\n ', dir or 0)
+end
+
+-- Remove common punctuation from URLs.
+--
+-- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc
+--
+-- @returns (fixed_url, removed_chars) where `removed_chars` is in the order found in the input.
+local function fix_url(url)
+  local removed_chars = ''
+  local fixed_url = url
+  -- Remove up to one of each char from end of the URL, in this order.
+  for _, c in ipairs({ '.', ')', }) do
+    if fixed_url:sub(-1) == c then
+      removed_chars = c .. removed_chars
+      fixed_url = fixed_url:sub(1, -2)
+    end
+  end
+  return fixed_url, removed_chars
+end
+
+-- Checks if a given line is a "noise" line that doesn't look good in HTML form.
+local function is_noise(line, noise_lines)
+  if (
+    -- First line is always noise.
+    (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0)
+    or line:find('Type .*gO.* to see the table of contents')
+    -- Title line of traditional :help pages.
+    -- Example: "NVIM REFERENCE MANUAL    by ..."
+    or line:find([[^%s*N?VIM[ \t]*REFERENCE[ \t]*MANUAL]])
+    -- First line of traditional :help pages.
+    -- Example: "*api.txt*    Nvim"
+    or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$')
+    -- modeline
+    -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:"
+    or line:find('^%s*vim?%:.*ft=help')
+    or line:find('^%s*vim?%:.*filetype=help')
+    or line:find('[*>]local%-additions[*<]')
+  ) then
+    -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0))
+    table.insert(noise_lines or {}, line)
+    return true
+  end
+  return false
+end
+
+-- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content.
+local function get_bug_url_vimdoc(fname, to_fname, sample_text)
+  local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname))
+  local bug_url = ('https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+'
+    ..vim.fs.basename(fname)
+    ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+'
+    ..this_url
+    ..'%0D%0DContext%3A%0D%0D%60%60%60%0D'
+    ..url_encode(sample_text)
+    ..'%0D%60%60%60')
+  return bug_url
+end
+
+-- Creates a github issue URL at neovim/neovim with prefilled content.
+local function get_bug_url_nvim(fname, to_fname, sample_text, token_name)
+  local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname))
+  local bug_url = ('https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+'
+    ..vim.fs.basename(fname)
+    ..'+&body=%60gen_help_html.lua%60+problem+at%3A+'
+    ..this_url
+    ..'%0D'
+    ..(token_name and '+unhandled+token%3A+%60'..token_name..'%60' or '')
+    ..'%0DContext%3A%0D%0D%60%60%60%0D'
+    ..url_encode(sample_text)
+    ..'%0D%60%60%60')
+  return bug_url
+end
+
+-- Gets a "foo.html" name from a "foo.txt" helpfile name.
+local function get_helppage(f)
+  if not f then
+    return nil
+  end
+  -- Special case: help.txt is the "main landing page" of :help files, not index.txt.
+  if f == 'index.txt' then
+    return 'vimindex.html'
+  elseif f == 'help.txt' then
+    return 'index.html'
+  end
+
+  return (f:gsub('%.txt$', '.html'))
+end
+
+-- Counts leading spaces (tab=8) to decide the indent size of multiline text.
+--
+-- Blank lines (empty or whitespace-only) are ignored.
+local function get_indent(s)
+  local min_indent = nil
+  for line in vim.gsplit(s, '\n') do
+    if line and not is_blank(line) then
+      local ws = expandtabs(line:match('^%s+') or '')
+      min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent
+    end
+  end
+  return min_indent or 0
+end
+
+-- Removes the common indent level, after expanding tabs to 8 spaces.
+local function trim_indent(s)
+  local indent_size = get_indent(s)
+  local trimmed = ''
+  for line in vim.gsplit(s, '\n') do
+    line = expandtabs(line)
+    trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1))
+  end
+  return trimmed:sub(1, -2)
+end
+
+-- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string.
+local function getbuflinestr(node, bufnr, offset)
+  local line1, _, line2, _ = node:range()
+  line1 = line1 - offset
+  line2 = line2 + offset
+  local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)
+  return table.concat(lines, '\n')
+end
+
+-- Gets the whitespace just before `node` from the raw buffer text.
+-- Needed for preformatted `old` lines.
+local function getws(node, bufnr)
+  local line1, c1, line2, _ = node:range()
+  local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1]
+  local text_before = raw:sub(1, c1)
+  local leading_ws = text_before:match('%s+$') or ''
+  return leading_ws
+end
+
+local function get_tagname(node, bufnr)
+  local text = vim.treesitter.get_node_text(node, bufnr)
+  local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(text) or text
+  local helpfile = vim.fs.basename(tagmap[tag]) or nil  -- "api.txt"
+  local helppage = get_helppage(helpfile)               -- "api.html"
+  return helppage, tag
+end
+
+-- Returns true if the given invalid tagname is a false positive.
+local function ignore_invalid(s)
+  return not not (
+    exclude_invalid[s]
+    -- Strings like |~/====| appear in various places and the parser thinks they are links, but they
+    -- are just table borders.
+    or s:find('===')
+    or s:find('---')
+  )
+end
+
+local function ignore_parse_error(s)
+  -- Ignore parse errors for unclosed codespan/optionlink/tag.
+  -- This is common in vimdocs and is treated as plaintext by :help.
+  return s:find("^[`'|*]")
+end
+
+local function has_ancestor(node, ancestor_name)
+  local p = node
+  while true do
+    p = p:parent()
+    if not p or p:type() == 'help_file' then
+      break
+    elseif p:type() == ancestor_name then
+      return true
+    end
+  end
+  return false
+end
+
+-- Gets the first matching child node matching `name`.
+local function first(node, name)
+  for c, _ in node:iter_children() do
+    if c:named() and c:type() == name then
+      return c
+    end
+  end
+  return nil
+end
+
+local function validate_link(node, bufnr, fname)
+  local helppage, tagname = get_tagname(node:child(1), bufnr)
+  local ignored = false
+  if not tagmap[tagname] then
+    ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname)
+    if not ignored then
+      invalid_links[tagname] = vim.fs.basename(fname)
+    end
+  end
+  return helppage, tagname, ignored
+end
+
+-- Traverses the tree at `root` and checks that |tag| links point to valid helptags.
+local function visit_validate(root, level, lang_tree, opt, stats)
+  level = level or 0
+  local node_name = (root.named and root:named()) and root:type() or nil
+  local toplevel = level < 1
+  local function node_text(node)
+    return vim.treesitter.get_node_text(node or root, opt.buf)
+  end
+  local text = trim(node_text())
+
+  if root:child_count() > 0 then
+    for node, _ in root:iter_children() do
+      if node:named() then
+        visit_validate(node, level + 1, lang_tree, opt, stats)
+      end
+    end
+  end
+
+  if node_name == 'ERROR' then
+    if ignore_parse_error(text) then
+      return
+    end
+    -- Store the raw text to give context to the error report.
+    local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]'
+    table.insert(stats.parse_errors, sample_text)
+  elseif node_name == 'word' or node_name == 'uppercase_name' then
+    if spell_dict[text] then
+      if not invalid_spelling[text] then
+        invalid_spelling[text] = { vim.fs.basename(opt.fname) }
+      else
+        table.insert(invalid_spelling[text], vim.fs.basename(opt.fname))
+      end
+    end
+  elseif node_name == 'url' then
+    if text:find('http%:') then
+      invalid_urls[text] = vim.fs.basename(opt.fname)
+    end
+  elseif node_name == 'taglink' or node_name == 'optionlink' then
+    local _, _, _ = validate_link(root, opt.buf, opt.fname)
+  end
+end
+
+-- Generates HTML from node `root` recursively.
+local function visit_node(root, level, lang_tree, headings, opt, stats)
+  level = level or 0
+
+  local node_name = (root.named and root:named()) and root:type() or nil
+  -- Previous sibling kind (string).
+  local prev = root:prev_sibling() and (root:prev_sibling().named and root:prev_sibling():named()) and root:prev_sibling():type() or nil
+  -- Next sibling kind (string).
+  local next_ = root:next_sibling() and (root:next_sibling().named and root:next_sibling():named()) and root:next_sibling():type() or nil
+  -- Parent kind (string).
+  local parent = root:parent() and root:parent():type() or nil
+  local text = ''
+  local trimmed
+  -- Gets leading whitespace of `node`.
+  local function ws(node)
+    node = node or root
+    local ws_ = getws(node, opt.buf)
+    -- XXX: first node of a (line) includes whitespace, even after
+    -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ?
+    if ws_ == '' then
+      ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or ''
+    end
+    return ws_
+  end
+  local function node_text(node, ws_)
+    node = node or root
+    ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or ''
+    return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf))
+  end
+
+  if root:named_child_count() == 0 or node_name == 'ERROR' then
+    text = node_text()
+    trimmed = html_esc(trim(text))
+    text = html_esc(text)
+  else
+    -- Process children and join them with whitespace.
+    for node, _ in root:iter_children() do
+      if node:named() then
+        local r = visit_node(node, level + 1, lang_tree, headings, opt, stats)
+        text = string.format('%s%s', text, r)
+      end
+    end
+    trimmed = trim(text)
+  end
+
+  if node_name == 'help_file' then  -- root node
+    return text
+  elseif node_name == 'url' then
+    local fixed_url, removed_chars = fix_url(trimmed)
+    return ('%s<a href="%s">%s</a>%s'):format(ws(), fixed_url, fixed_url, removed_chars)
+  elseif node_name == 'word' or node_name == 'uppercase_name' then
+    return text
+  elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then
+    if is_noise(text, stats.noise_lines) then
+      return ''  -- Discard common "noise" lines.
+    end
+    -- Remove "===" and tags from ToC text.
+    local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', ''))
+    -- Use the first *tag* node as the heading anchor, if any.
+    local tagnode = first(root, 'tag')
+    local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname)
+    if node_name == 'h1' or #headings == 0 then
+      table.insert(headings, { name = hname, subheadings = {}, tag = tagname })
+    else
+      table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, tag = tagname })
+    end
+    local el = node_name == 'h1' and 'h2' or 'h3'
+    -- If we are re-using the *tag*, this heading anchor is redundant.
+    local a = tagnode and '' or ('<a name="%s"></a>'):format(tagname)
+    return ('%s<%s class="help-heading">%s</%s>\n'):format(a, el, text, el)
+  elseif node_name == 'column_heading' or node_name == 'column_name' then
+    if root:has_error() then
+      return text
+    end
+    return ('<div class="help-column_heading">%s</div>'):format(text)
+  elseif node_name == 'block' then
+    if is_blank(text) then
+      return ''
+    end
+    if opt.old then
+      -- XXX: Treat "old" docs as preformatted: they use indentation for layout.
+      --      Trim trailing newlines to avoid too much whitespace between divs.
+      return ('<div class="old-help-para">%s</div>\n'):format(trim(text, 2))
+    end
+    return string.format('<div class="help-para">\n%s\n</div>\n', text)
+  elseif node_name == 'line' then
+    if parent ~= 'codeblock' and (is_blank(text) or is_noise(text, stats.noise_lines)) then
+      return ''  -- Discard common "noise" lines.
+    end
+    -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout.
+    local div = opt.old and root:child(0) and vim.tbl_contains({'column_heading', 'h1', 'h2', 'h3'}, root:child(0):type())
+    return string.format('%s%s', div and trim(text) or text, div and '' or '\n')
+  elseif node_name == 'line_li' then
+    local sib = root:prev_sibling()
+    local prev_li = sib and sib:type() == 'line_li'
+
+    if not prev_li then
+      opt.indent = 1
+    else
+      -- The previous listitem _sibling_ is _logically_ the _parent_ if it is indented less.
+      local parent_indent = get_indent(node_text(sib))
+      local this_indent = get_indent(node_text())
+      if this_indent > parent_indent then
+        opt.indent = opt.indent + 1
+      elseif this_indent < parent_indent then
+        opt.indent = math.max(1, opt.indent - 1)
+      end
+    end
+    local margin = opt.indent == 1 and '' or ('margin-left: %drem;'):format((1.5 * opt.indent))
+
+    return string.format('<div class="help-li" style="%s">%s</div>', margin, text)
+  elseif node_name == 'taglink' or node_name == 'optionlink' then
+    local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname)
+    if ignored then
+      return text
+    end
+    return ('%s<a href="%s#%s">%s</a>'):format(ws(), helppage, url_encode(tagname), html_esc(tagname))
+  elseif vim.tbl_contains({'codespan', 'keycode'}, node_name) then
+    if root:has_error() then
+      return text
+    end
+    return ('%s<code>%s</code>'):format(ws(), trimmed)
+  elseif node_name == 'argument' then
+    return ('%s<code>{%s}</code>'):format(ws(), text)
+  elseif node_name == 'codeblock' then
+    if is_blank(text) then
+      return ''
+    end
+    return ('<pre>%s</pre>'):format(trim(trim_indent(text), 2))
+  elseif node_name == 'tag' then  -- anchor
+    if root:has_error() then
+      return text
+    end
+    local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent)
+    local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag'
+    local tagname = node_text(root:child(1), false)
+    if vim.tbl_count(stats.first_tags) < 2 then
+      -- Force the first 2 tags in the doc to be anchored at the main heading.
+      table.insert(stats.first_tags, tagname)
+      return ''
+    end
+    local el = in_heading and 'span' or 'code'
+    local s = ('%s<a name="%s"></a><%s class="%s">%s</%s>'):format(ws(), url_encode(tagname), el, cssclass, trimmed, el)
+    if in_heading and prev ~= 'tag' then
+      -- Start the <span> container for tags in a heading.
+      -- This makes "justify-content:space-between" right-align the tags.
+      --    <h2>foo bar<span>tag1 tag2</span></h2>
+      return string.format('<span class="help-heading-tags">%s', s)
+    elseif in_heading and next_ == nil then
+      -- End the <span> container for tags in a heading.
+      return string.format('%s</span>', s)
+    end
+    return s
+  elseif node_name == 'ERROR' then
+    if ignore_parse_error(trimmed) then
+      return text
+    end
+
+    -- Store the raw text to give context to the bug report.
+    local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]'
+    table.insert(stats.parse_errors, sample_text)
+    return ('<a class="parse-error" target="_blank" title="Report bug... (parse error)" href="%s">%s</a>'):format(
+      get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed)
+  else  -- Unknown token.
+    local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]'
+    return ('<a class="unknown-token" target="_blank" title="Report bug... (unhandled token "%s")" href="%s">%s</a>'):format(
+      node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name)
+  end
+end
+
+local function get_helpfiles(include)
+  local dir = './build/runtime/doc'
+  local rv = {}
+  for f, type in vim.fs.dir(dir) do
+    if (vim.endswith(f, '.txt')
+        and type == 'file'
+        and (not include or vim.tbl_contains(include, f))) then
+      local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p')
+      table.insert(rv, fullpath)
+    end
+  end
+  return rv
+end
+
+-- Populates the helptags map.
+local function get_helptags(help_dir)
+  local m = {}
+  -- Load a random help file to convince taglist() to do its job.
+  vim.cmd(string.format('split %s/api.txt', help_dir))
+  vim.cmd('lcd %:p:h')
+  for _, item in ipairs(vim.fn.taglist('.*')) do
+    if vim.endswith(item.filename, '.txt') then
+      m[item.name] = item.filename
+    end
+  end
+  vim.cmd('q!')
+  return m
+end
+
+-- Use the help.so parser defined in the build, not whatever happens to be installed on the system.
+local function ensure_runtimepath()
+  if not vim.o.runtimepath:find('build/lib/nvim/') then
+    vim.cmd[[set runtimepath^=./build/lib/nvim/]]
+  end
+end
+
+-- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents.
+--
+-- @returns lang_tree, bufnr
+local function parse_buf(fname)
+  local buf
+  if type(fname) == 'string' then
+    vim.cmd('split '..vim.fn.fnameescape(fname))  -- Filename.
+    buf = vim.api.nvim_get_current_buf()
+  else
+    buf = fname
+    vim.cmd('sbuffer '..tostring(fname))          -- Buffer number.
+  end
+  -- vim.treesitter.require_language('help', './build/lib/nvim/parser/help.so')
+  local lang_tree = vim.treesitter.get_parser(buf, 'help')
+  return lang_tree, buf
+end
+
+-- Validates one :help file `fname`:
+--  - checks that |tag| links point to valid helptags.
+--  - recursively counts parse errors ("ERROR" nodes)
+--
+-- @returns { invalid_links: number, parse_errors: number }
+local function validate_one(fname)
+  local stats = {
+    parse_errors = {},
+  }
+  local lang_tree, buf = parse_buf(fname)
+  for _, tree in ipairs(lang_tree:trees()) do
+    visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname, }, stats)
+  end
+  lang_tree:destroy()
+  vim.cmd.close()
+  return stats
+end
+
+-- Generates HTML from one :help file `fname` and writes the result to `to_fname`.
+--
+-- @param fname Source :help file
+-- @param to_fname Destination .html file
+-- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace)
+--
+-- @returns html, stats
+local function gen_one(fname, to_fname, old, commit)
+  local stats = {
+    noise_lines = {},
+    parse_errors = {},
+    first_tags = {},  -- Track the first few tags in doc.
+  }
+  local lang_tree, buf = parse_buf(fname)
+  local headings = {}  -- Headings (for ToC). 2-dimensional: h1 contains h2/h3.
+  local title = to_titlecase(basename_noext(fname))
+
+  local html = ([[
+  <!DOCTYPE html>
+  <html>
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="Neovim user documentation">
+    <link href="/css/normalize.min.css" rel="stylesheet">
+    <link href="/css/bootstrap.css" rel="stylesheet">
+    <link href="/css/main.css" rel="stylesheet">
+    <link href="help.css" rel="stylesheet">
+    <title>%s - Neovim docs</title>
+  </head>
+  <body>
+  ]]):format(title)
+
+  local logo_svg = [[
+    <svg xmlns="http://www.w3.org/2000/svg" role="img" width="173" height="50" viewBox="0 0 742 214" aria-label="Neovim">
+      <title>Neovim</title>
+      <defs>
+        <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="a">
+          <stop stop-color="#16B0ED" stop-opacity=".8" offset="0%" />
+          <stop stop-color="#0F59B2" stop-opacity=".837" offset="100%" />
+        </linearGradient>
+        <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="b">
+          <stop stop-color="#7DB643" offset="0%" />
+          <stop stop-color="#367533" offset="100%" />
+        </linearGradient>
+        <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="c">
+          <stop stop-color="#88C649" stop-opacity=".8" offset="0%" />
+          <stop stop-color="#439240" stop-opacity=".84" offset="100%" />
+        </linearGradient>
+      </defs>
+      <g fill="none" fill-rule="evenodd">
+        <path
+          d="M.027 45.459L45.224-.173v212.171L.027 166.894V45.459z"
+          fill="url(#a)"
+          transform="translate(1 1)"
+        />
+        <path
+          d="M129.337 45.89L175.152-.149l-.928 212.146-45.197-45.104.31-121.005z"
+          fill="url(#b)"
+          transform="matrix(-1 0 0 1 305 1)"
+        />
+        <path
+          d="M45.194-.137L162.7 179.173l-32.882 32.881L12.25 33.141 45.194-.137z"
+          fill="url(#c)"
+          transform="translate(1 1)"
+        />
+        <path
+          d="M46.234 84.032l-.063 7.063-36.28-53.563 3.36-3.422 32.983 49.922z"
+          fill-opacity=".13"
+          fill="#000"
+        />
+        <g fill="#444">
+          <path
+            d="M227 154V64.44h4.655c1.55 0 2.445.75 2.685 2.25l.806 13.502c4.058-5.16 8.786-9.316 14.188-12.466 5.4-3.15 11.413-4.726 18.037-4.726 4.893 0 9.205.781 12.935 2.34 3.729 1.561 6.817 3.811 9.264 6.751 2.448 2.942 4.297 6.48 5.55 10.621 1.253 4.14 1.88 8.821 1.88 14.042V154h-8.504V96.754c0-8.402-1.91-14.987-5.729-19.757-3.82-4.771-9.667-7.156-17.544-7.156-5.851 0-11.28 1.516-16.292 4.545-5.013 3.032-9.489 7.187-13.427 12.467V154H227zM350.624 63c5.066 0 9.755.868 14.069 2.605 4.312 1.738 8.052 4.268 11.219 7.592s5.638 7.412 7.419 12.264C385.11 90.313 386 95.883 386 102.17c0 1.318-.195 2.216-.588 2.696-.393.48-1.01.719-1.851.719h-64.966v1.70c0 6.708.784 12.609 2.353 17.7 1.567 5.09 3.8 9.357 6.695 12.802 2.895 3.445 6.393 6.034 10.495 7.771 4.1 1.738 8.686 2.606 13.752 2.606 4.524 0 8.446-.494 11.762-1.483 3.317-.988 6.108-2.097 8.37-3.324 2.261-1.227 4.056-2.336 5.383-3.324 1.326-.988 2.292-1.482 2.895-1.482.784 0 1.388.3 1.81.898l2.352 2.875c-1.448 1.797-3.362 3.475-5.745 5.031-2.383 1.558-5.038 2.891-7.962 3.998-2.926 1.109-6.062 1.991-9.41 2.65a52.21 52.21 0 01-10.088.989c-6.152 0-11.762-1.064-16.828-3.19-5.067-2.125-9.415-5.225-13.043-9.298-3.63-4.074-6.435-9.06-8.415-14.96C310.99 121.655 310 114.9 310 107.294c0-6.408.92-12.323 2.76-17.744 1.84-5.421 4.493-10.093 7.961-14.016 3.467-3.922 7.72-6.991 12.758-9.209C338.513 64.11 344.229 63 350.624 63zm.573 6c-4.696 0-8.904.702-12.623 2.105-3.721 1.404-6.936 3.421-9.65 6.053-2.713 2.631-4.908 5.79-6.586 9.474S319.55 94.439 319 99h60c0-4.679-.672-8.874-2.013-12.588-1.343-3.712-3.232-6.856-5.67-9.43-2.44-2.571-5.367-4.545-8.782-5.92-3.413-1.374-7.192-2.062-11.338-2.062zM435.546 63c6.526 0 12.368 1.093 17.524 3.28 5.154 2.186 9.5 5.286 13.04 9.298 3.538 4.013 6.238 8.85 8.099 14.51 1.861 5.66 2.791 11.994 2.791 19.002 0 7.008-.932 13.327-2.791 18.957-1.861 5.631-4.561 10.452-8.099 14.465-3.54 4.012-7.886 7.097-13.04 9.254-5.156 2.156-10.998 3.234-17.524 3.234-6.529 0-12.369-1.078-17.525-3.234-5.155-2.157-9.517-5.242-13.085-9.254-3.57-4.013-6.285-8.836-8.145-14.465-1.861-5.63-2.791-11.95-2.791-18.957 0-7.008.93-13.342 2.791-19.002 1.861-5.66 4.576-10.496 8.145-14.51 3.568-4.012 7.93-7.112 13.085-9.299C423.177 64.094 429.017 63 435.546 63zm-.501 86c5.341 0 10.006-.918 13.997-2.757 3.99-1.838 7.32-4.474 9.992-7.909 2.67-3.435 4.664-7.576 5.986-12.428 1.317-4.85 1.98-10.288 1.98-16.316 0-5.965-.66-11.389-1.98-16.27-1.322-4.88-3.316-9.053-5.986-12.519-2.67-3.463-6-6.13-9.992-7.999-3.991-1.867-8.657-2.802-13.997-2.802s-10.008.935-13.997 2.802c-3.991 1.87-7.322 4.536-9.992 8-2.671 3.465-4.68 7.637-6.03 12.518-1.35 4.881-2.026 10.305-2.026 16.27 0 6.026.675 11.465 2.025 16.316 1.35 4.852 3.36 8.993 6.031 12.428 2.67 3.435 6 6.07 9.992 7.91 3.99 1.838 8.656 2.756 13.997 2.756z"
+            fill="currentColor"
+          />
+          <path
+            d="M530.57 152h-20.05L474 60h18.35c1.61 0 2.967.39 4.072 1.166 1.103.778 1.865 1.763 2.283 2.959l17.722 49.138a92.762 92.762 0 012.551 8.429c.686 2.751 1.298 5.5 1.835 8.25.537-2.75 1.148-5.499 1.835-8.25a77.713 77.713 0 012.64-8.429l18.171-49.138c.417-1.196 1.164-2.181 2.238-2.96 1.074-.776 2.356-1.165 3.849-1.165H567l-36.43 92zM572 61h23v92h-23zM610 153V60.443h13.624c2.887 0 4.78 1.354 5.682 4.06l1.443 6.856a52.7 52.7 0 015.097-4.962 32.732 32.732 0 015.683-3.879 30.731 30.731 0 016.496-2.57c2.314-.632 4.855-.948 7.624-.948 5.832 0 10.63 1.579 14.39 4.736 3.758 3.157 6.57 7.352 8.434 12.585 1.444-3.068 3.248-5.698 5.413-7.894 2.165-2.194 4.541-3.984 7.127-5.367a32.848 32.848 0 018.254-3.068 39.597 39.597 0 018.796-.992c5.111 0 9.653.783 13.622 2.345 3.97 1.565 7.307 3.849 10.014 6.857 2.706 3.007 4.766 6.675 6.18 11.005C739.29 83.537 740 88.5 740 94.092V153h-22.284V94.092c0-5.894-1.294-10.329-3.878-13.306-2.587-2.977-6.376-4.465-11.368-4.465-2.286 0-4.404.391-6.358 1.172a15.189 15.189 0 00-5.144 3.383c-1.473 1.474-2.631 3.324-3.474 5.548-.842 2.225-1.263 4.781-1.263 7.668V153h-22.37V94.092c0-6.194-1.249-10.704-3.744-13.532-2.497-2.825-6.18-4.24-11.051-4.24-3.19 0-6.18.798-8.976 2.391-2.799 1.593-5.399 3.775-7.804 6.54V153H610zM572 30h23v19h-23z"
+            fill="currentColor"
+            fill-opacity=".8"
+          />
+        </g>
+      </g>
+    </svg>
+  ]]
+
+  local main = ''
+  for _, tree in ipairs(lang_tree:trees()) do
+    main = main .. (visit_node(tree:root(), 0, tree, headings,
+      { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1, },
+      stats))
+  end
+
+  main = ([[
+  <header class="container">
+    <nav class="navbar navbar-expand-lg">
+      <div>
+        <a href="/" class="navbar-brand" aria-label="logo">
+          <!--TODO: use <img src="….svg"> here instead. Need one that has green lettering instead of gray. -->
+          %s
+          <!--<img src="https://neovim.io/logos/neovim-logo.svg" width="173" height="50" alt="Neovim" />-->
+        </a>
+      </div>
+    </nav>
+  </header>
+
+  <div class="container golden-grid help-body">
+  <div class="col-wide">
+  <a name="%s"></a><a name="%s"></a><h1>%s</h1>
+  <p>
+    <i>
+    Nvim <code>:help</code> pages, <a href="https://github.com/neovim/neovim/blob/master/scripts/gen_help_html.lua">generated</a>
+    from <a href="https://github.com/neovim/neovim/blob/master/runtime/doc/%s">source</a>
+    using the <a href="https://github.com/neovim/tree-sitter-vimdoc">tree-sitter-vimdoc</a> parser.
+    </i>
+  </p>
+  <hr/>
+  %s
+  </div>
+  ]]):format(logo_svg, stats.first_tags[1] or '', stats.first_tags[2] or '', title, vim.fs.basename(fname), main)
+
+  local toc = [[
+    <div class="col-narrow toc">
+      <div><a href="index.html">Main</a></div>
+      <div><a href="vimindex.html">Commands index</a></div>
+      <div><a href="quickref.html">Quick reference</a></div>
+      <hr/>
+  ]]
+
+  local n = 0  -- Count of all headings + subheadings.
+  for _, h1 in ipairs(headings) do n = n + 1 + #h1.subheadings end
+  for _, h1 in ipairs(headings) do
+    toc = toc .. ('<div class="help-toc-h1"><a href="#%s">%s</a>\n'):format(h1.tag, h1.name)
+    if n < 30 or #headings < 10 then  -- Show subheadings only if there aren't too many.
+      for _, h2 in ipairs(h1.subheadings) do
+        toc = toc .. ('<div class="help-toc-h2"><a href="#%s">%s</a></div>\n'):format(h2.tag, h2.name)
+      end
+    end
+    toc = toc .. '</div>'
+  end
+  toc = toc .. '</div>\n'
+
+  local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil)
+  local bug_link = string.format('(<a href="%s" target="_blank">report docs bug...</a>)', bug_url)
+
+  local footer = ([[
+  <footer>
+    <div class="container flex">
+      <div class="generator-stats">
+        Generated at %s from <code><a href="https://github.com/neovim/neovim/commit/%s">%s</a></code>
+      </div>
+      <div class="generator-stats">
+      parse_errors: %d %s | <span title="%s">noise_lines: %d</span>
+      </div>
+    <div>
+  </footer>
+  ]]):format(
+    os.date('%Y-%m-%d %H:%M'), commit, commit:sub(1, 7), #stats.parse_errors, bug_link,
+    html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines)
+
+  html = ('%s%s%s</div>\n%s</body>\n</html>\n'):format(
+    html, main, toc, footer)
+  vim.cmd('q!')
+  lang_tree:destroy()
+  return html, stats
+end
+
+local function gen_css(fname)
+  local css = [[
+    :root {
+      --code-color: #008B8B;
+      --tag-color: gray;
+    }
+    @media (min-width: 40em) {
+      .toc {
+        position: fixed;
+        left: 67%;
+      }
+      .golden-grid {
+          display: grid;
+          grid-template-columns: 65% auto;
+          grid-gap: 1em;
+      }
+    }
+    @media (max-width: 40em) {
+      .golden-grid {
+        /* Disable grid for narrow viewport (mobile phone). */
+        display: block;
+      }
+    }
+    @media (prefers-color-scheme: dark) {
+      :root {
+        --code-color: cyan;
+      }
+    }
+    .toc {
+      /* max-width: 12rem; */
+      height: 85%;  /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */
+      overflow: auto;  /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */
+    }
+    .toc > div {
+      text-overflow: ellipsis;
+      overflow: hidden;
+      white-space: nowrap;
+    }
+    html {
+      scroll-behavior: auto;
+    }
+    body {
+      font-size: 18px;
+      line-height: 1.5;
+    }
+    h1, h2, h3, h4, h5 {
+      font-family: sans-serif;
+      border-bottom: 1px solid #41464bd6; /*rgba(0, 0, 0, .9);*/
+    }
+    h3, h4, h5 {
+      border-bottom-style: dashed;
+    }
+    .help-column_heading {
+      color: var(--code-color);
+    }
+    .help-body {
+      padding-bottom: 2em;
+    }
+    .help-line {
+      /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */
+    }
+    .help-li {
+      white-space: normal;
+      display: list-item;
+      margin-left: 1.5rem; /* padding-left: 1rem; */
+    }
+    .help-para {
+      padding-top: 10px;
+      padding-bottom: 10px;
+    }
+    .old-help-para {
+      padding-top: 10px;
+      padding-bottom: 10px;
+      /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */
+      tab-size: 8;
+      white-space: pre;
+      font-size: 16px;
+      font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace;
+    }
+    a.help-tag, a.help-tag:focus, a.help-tag:hover {
+      color: inherit;
+      text-decoration: none;
+    }
+    .help-tag {
+      color: var(--tag-color);
+    }
+    /* Tag pseudo-header common in :help docs. */
+    .help-tag-right {
+      color: var(--tag-color);
+    }
+    h1 .help-tag, h2 .help-tag, h3 .help-tag {
+      font-size: smaller;
+    }
+    .help-heading {
+      overflow: hidden;
+      white-space: nowrap;
+      display: flex;
+      justify-content: space-between;
+    }
+    /* The (right-aligned) "tags" part of a section heading. */
+    .help-heading-tags {
+      margin-right: 10px;
+    }
+    .help-toc-h1 {
+    }
+    .help-toc-h2 {
+      margin-left: 1em;
+    }
+    .parse-error {
+      background-color: red;
+    }
+    .unknown-token {
+      color: black;
+      background-color: yellow;
+    }
+    code {
+      color: var(--code-color);
+      font-size: 16px;
+    }
+    pre {
+      /* Tabs are used in codeblocks only for indentation, not alignment, so we can aggressively shrink them. */
+      tab-size: 2;
+      white-space: pre;
+      line-height: 1.3;  /* Important for ascii art. */
+      overflow: visible;
+      /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */
+      font-size: 16px;
+      margin-top: 10px;
+    }
+    pre:hover,
+    .help-heading:hover {
+      overflow: visible;
+    }
+    .generator-stats {
+      color: gray;
+      font-size: smaller;
+    }
+  ]]
+  tofile(fname, css)
+end
+
+function M._test()
+  tagmap = get_helptags('./build/runtime/doc')
+  helpfiles = get_helpfiles()
+
+  local function ok(cond, expected, actual)
+    assert((not expected and not actual) or (expected and actual), 'if "expected" is given, "actual" is also required')
+    if expected then
+      return assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual)))
+    else
+      return assert(cond)
+    end
+  end
+  local function eq(expected, actual)
+    return ok(expected == actual, expected, actual)
+  end
+
+  ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap))
+  ok(vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), tagmap['vim.diagnostic.set()'], 'diagnostic.txt')
+  ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt')
+  ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]]))
+  ok(is_noise([[          NVIM  REFERENCE  MANUAL     by  Thiago  de  Arruda      ]]))
+  ok(not is_noise([[vim:tw=78]]))
+
+  eq(0, get_indent('a'))
+  eq(1, get_indent(' a'))
+  eq(2, get_indent('  a\n  b\n  c\n'))
+  eq(5, get_indent('     a\n      \n        b\n      c\n      d\n      e\n'))
+  eq('a\n        \n   b\n c\n d\n e\n', trim_indent('     a\n             \n        b\n      c\n      d\n      e\n'))
+
+  local fixed_url, removed_chars = fix_url('https://example.com).')
+  eq('https://example.com', fixed_url)
+  eq(').', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com.)')
+  eq('https://example.com.', fixed_url)
+  eq(')', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com.')
+  eq('https://example.com', fixed_url)
+  eq('.', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com)')
+  eq('https://example.com', fixed_url)
+  eq(')', removed_chars)
+  fixed_url, removed_chars = fix_url('https://example.com')
+  eq('https://example.com', fixed_url)
+  eq('', removed_chars)
+
+  print('all tests passed')
+end
+
+--- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`.
+---
+--- Example:
+---
+---   gen('./build/runtime/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil)
+---
+--- @param help_dir string Source directory containing the :help files. Must run `make helptags` first.
+--- @param to_dir string Target directory where the .html files will be written.
+--- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'}
+---
+--- @returns info dict
+function M.gen(help_dir, to_dir, include, commit)
+  vim.validate{
+    help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'},
+    to_dir={to_dir, 's'},
+    include={include, 't', true},
+    commit={commit, 's', true},
+  }
+
+  local err_count = 0
+  ensure_runtimepath()
+  tagmap = get_helptags(help_dir)
+  helpfiles = get_helpfiles(include)
+
+  print(('output dir: %s'):format(to_dir))
+  vim.fn.mkdir(to_dir, 'p')
+  gen_css(('%s/help.css'):format(to_dir))
+
+  for _, f in ipairs(helpfiles) do
+    local helpfile = vim.fs.basename(f)
+    local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile))
+    local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?')
+    tofile(to_fname, html)
+    print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname)))
+    err_count = err_count + #stats.parse_errors
+  end
+  print(('generated %d html pages'):format(#helpfiles))
+  print(('total errors: %d'):format(err_count))
+  print(('invalid tags:\n%s'):format(vim.inspect(invalid_links)))
+
+  return {
+    helpfiles = helpfiles,
+    err_count = err_count,
+    invalid_links = invalid_links,
+  }
+end
+
+-- Validates all :help files found in `help_dir`:
+--  - checks that |tag| links point to valid helptags.
+--  - recursively counts parse errors ("ERROR" nodes)
+--
+-- This is 10x faster than gen(), for use in CI.
+--
+-- @returns results dict
+function M.validate(help_dir, include)
+  vim.validate{
+    help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'},
+    include={include, 't', true},
+  }
+  local err_count = 0
+  ensure_runtimepath()
+  tagmap = get_helptags(help_dir)
+  helpfiles = get_helpfiles(include)
+
+  for _, f in ipairs(helpfiles) do
+    local helpfile = vim.fs.basename(f)
+    local rv = validate_one(f)
+    print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile))
+    err_count = err_count + #rv.parse_errors
+  end
+
+  return {
+    helpfiles = #helpfiles,
+    err_count = err_count,
+    invalid_links = invalid_links,
+    invalid_urls = invalid_urls,
+    invalid_spelling = invalid_spelling,
+  }
+end
+
+return M
diff --git a/scripts/gen_help_html.py b/scripts/gen_help_html.py
deleted file mode 100644
index 0b8e77ac22..0000000000
--- a/scripts/gen_help_html.py
+++ /dev/null
@@ -1,389 +0,0 @@
-# Converts Vim/Nvim documentation to HTML.
-#
-# USAGE:
-#   1. python3 scripts/gen_help_html.py runtime/doc/ ~/neovim.github.io/t/
-#   3. cd ~/neovim.github.io/ && jekyll serve --host 0.0.0.0
-#   2. Visit http://localhost:4000/t/help.txt.html
-#
-# Adapted from https://github.com/c4rlo/vimhelp/
-# License: MIT
-#
-# Copyright (c) 2016 Carlo Teubner
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-import os
-import re
-import urllib.parse
-import datetime
-import sys
-from itertools import chain
-
-HEAD = """\
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
-    "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<head>
-<meta http-equiv="Content-type" content="text/html; charset={encoding}"/>
-<style>
-.h {{
-  font-weight: bold;
-}}
-h1 {{
-  font-family: sans-serif;
-}}
-pre {{
-  font-family: sans-serif;
-}}
-</style>
-<title>Nvim: {filename}</title>
-"""
-
-HEAD_END = '</head>\n<body>\n'
-
-INTRO = """
-<h1>Nvim help files</h1>
-<p>
-<a href="https://neovim.io/">Nvim</a> help pages{vers-note}.
-Updated <a href="https://github.com/neovim/bot-ci" class="d">automatically</a>
-from the <a href="https://github.com/neovim/neovim" class="d">Nvim source</a>.
-</p>
-"""
-
-VERSION_NOTE = ", current as of Nvim {version}"
-
-SITENAVI_LINKS = """
-<a href="quickref.txt.html">Quick reference</a> &middot;
-<a href="usr_toc.txt.html">User manual</a> &middot;
-<a href="{helptxt}#reference_toc">Reference manual</a> &middot;
-"""
-
-SITENAVI_LINKS_PLAIN = SITENAVI_LINKS.format(helptxt='help.txt.html')
-SITENAVI_LINKS_WEB = SITENAVI_LINKS.format(helptxt='/')
-
-SITENAVI_PLAIN = '<p>' + SITENAVI_LINKS_PLAIN + '</p>'
-SITENAVI_WEB = '<p>' + SITENAVI_LINKS_WEB + '</p>'
-
-SITENAVI_SEARCH = '<table width="100%"><tbody><tr><td>' + SITENAVI_LINKS_WEB + \
-    '</td><td style="text-align: right; max-width: 25vw"><div class="gcse-searchbox">' \
-    '</div></td></tr></tbody></table><div class="gcse-searchresults"></div>'
-
-TEXTSTART = """
-<div id="d1">
-<pre id="sp">""" + (" " * 80) + """</pre>
-<div id="d2">
-<pre>
-"""
-
-FOOTER = '</pre>'
-
-FOOTER2 = """
-<p id="footer">Generated {generated_date} from <code>{commit}</code></p>
-</div>
-</div>
-</body>
-</html>
-""".format(
-    generated_date='{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()),
-    commit='?')
-
-RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)')
-
-PAT_WORDCHAR = '[!#-)+-{}~\xC0-\xFF]'
-
-PAT_HEADER = r'(^.*~$)'
-PAT_GRAPHIC = r'(^.* `$)'
-PAT_PIPEWORD = r'(?<!\\)\|([#-)!+-~]+)\|'
-PAT_STARWORD = r'\*([#-)!+-~]+)\*(?:(?=\s)|$)'
-PAT_COMMAND = r'`([^` ]+)`'
-PAT_OPTWORD = r"('(?:[a-z]{2,}|t_..)')"
-PAT_CTRL = r'(CTRL-(?:W_)?(?:\{char\}|<[A-Za-z]+?>|.)?)'
-PAT_SPECIAL = r'(<.+?>|\{.+?}|' \
-    r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' \
-    r'arg|arguments|ident|addr|group)]|' \
-    r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])'
-PAT_TITLE = r'(Vim version [0-9.a-z]+|VIM REFERENCE.*)'
-PAT_NOTE = r'((?<!' + PAT_WORDCHAR + r')(?:note|NOTE|Notes?):?' \
-    r'(?!' + PAT_WORDCHAR + r'))'
-PAT_URL = r'((?:https?|ftp)://[^\'"<> \t]+[a-zA-Z0-9/])'
-PAT_WORD = r'((?<!' + PAT_WORDCHAR + r')' + PAT_WORDCHAR + r'+' \
-    r'(?!' + PAT_WORDCHAR + r'))'
-
-RE_LINKWORD = re.compile(
-    PAT_OPTWORD + '|' +
-    PAT_CTRL + '|' +
-    PAT_SPECIAL)
-RE_TAGWORD = re.compile(
-    PAT_HEADER + '|' +
-    PAT_GRAPHIC + '|' +
-    PAT_PIPEWORD + '|' +
-    PAT_STARWORD + '|' +
-    PAT_COMMAND + '|' +
-    PAT_OPTWORD + '|' +
-    PAT_CTRL + '|' +
-    PAT_SPECIAL + '|' +
-    PAT_TITLE + '|' +
-    PAT_NOTE + '|' +
-    PAT_URL + '|' +
-    PAT_WORD)
-RE_NEWLINE = re.compile(r'[\r\n]')
-# H1 header "=====…"
-# H2 header "-----…"
-RE_HRULE = re.compile(r'[-=]{3,}.*[-=]{3,3}$')
-RE_EG_START = re.compile(r'(?:.* )?>$')
-RE_EG_END = re.compile(r'\S')
-RE_SECTION = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)')
-RE_STARTAG = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)')
-RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$')
-
-
-class Link(object):
-    __slots__ = 'link_plain_same',    'link_pipe_same', \
-                'link_plain_foreign', 'link_pipe_foreign', \
-                'filename'
-
-    def __init__(self, link_plain_same, link_plain_foreign,
-                 link_pipe_same,  link_pipe_foreign, filename):
-        self.link_plain_same = link_plain_same
-        self.link_plain_foreign = link_plain_foreign
-        self.link_pipe_same = link_pipe_same
-        self.link_pipe_foreign = link_pipe_foreign
-        self.filename = filename
-
-
-class VimH2H(object):
-    def __init__(self, tags, version=None, is_web_version=True):
-        self._urls = {}
-        self._version = version
-        self._is_web_version = is_web_version
-        for line in RE_NEWLINE.split(tags):
-            m = RE_TAGLINE.match(line)
-            if m:
-                tag, filename = m.group(1, 2)
-                self.do_add_tag(filename, tag)
-
-    def add_tags(self, filename, contents):
-        for match in RE_STARTAG.finditer(contents):
-            tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/')
-            self.do_add_tag(str(filename), tag)
-
-    def do_add_tag(self, filename, tag):
-        tag_quoted = urllib.parse.quote_plus(tag)
-
-        def mkpart1(doc):
-            return '<a href="' + doc + '#' + tag_quoted + '" class="'
-        part1_same = mkpart1('')
-        if self._is_web_version and filename == 'help.txt':
-            doc = '/'
-        else:
-            doc = filename + '.html'
-        part1_foreign = mkpart1(doc)
-        part2 = '">' + html_escape[tag] + '</a>'
-
-        def mklinks(cssclass):
-            return (part1_same + cssclass + part2,
-                    part1_foreign + cssclass + part2)
-        cssclass_plain = 'd'
-        m = RE_LINKWORD.match(tag)
-        if m:
-            opt, ctrl, special = m.groups()
-            if opt is not None:
-                cssclass_plain = 'o'
-            elif ctrl is not None:
-                cssclass_plain = 'k'
-            elif special is not None:
-                cssclass_plain = 's'
-        links_plain = mklinks(cssclass_plain)
-        links_pipe = mklinks('l')
-        self._urls[tag] = Link(
-            links_plain[0], links_plain[1],
-            links_pipe[0],  links_pipe[1],
-            filename)
-
-    def maplink(self, tag, curr_filename, css_class=None):
-        links = self._urls.get(tag)
-        if links is not None:
-            if links.filename == curr_filename:
-                if css_class == 'l':
-                    return links.link_pipe_same
-                else:
-                    return links.link_plain_same
-            else:
-                if css_class == 'l':
-                    return links.link_pipe_foreign
-                else:
-                    return links.link_plain_foreign
-        elif css_class is not None:
-            return '<span class="' + css_class + '">' + html_escape[tag] + \
-                '</span>'
-        else:
-            return html_escape[tag]
-
-    def to_html(self, filename, contents, encoding):
-        out = []
-
-        inexample = 0
-        filename = str(filename)
-        is_help_txt = (filename == 'help.txt')
-        last = ''
-        for line in RE_NEWLINE.split(contents):
-            line = line.rstrip('\r\n')
-            line_tabs = line
-            line = line.expandtabs()
-            if last == 'h1':
-                out.extend(('</pre>'))  # XXX
-                out.extend(('<h1>', line.rstrip(), '</h1>\n'))
-                out.extend(('<pre>'))
-                last = ''
-                continue
-            if RE_HRULE.match(line):
-                # out.extend(('<span class="h">', line, '</span>\n'))
-                last = 'h1'
-                continue
-            if inexample == 2:
-                if RE_EG_END.match(line):
-                    inexample = 0
-                    if line[0] == '<':
-                        line = line[1:]
-                else:
-                    out.extend(('<span class="e">', html_escape[line],
-                                '</span>\n'))
-                    continue
-            if RE_EG_START.match(line_tabs):
-                inexample = 1
-                line = line[0:-1]
-            if RE_SECTION.match(line_tabs):
-                m = RE_SECTION.match(line)
-                out.extend((r'<span class="c">', m.group(0), r'</span>'))
-                line = line[m.end():]
-            lastpos = 0
-            for match in RE_TAGWORD.finditer(line):
-                pos = match.start()
-                if pos > lastpos:
-                    out.append(html_escape[line[lastpos:pos]])
-                lastpos = match.end()
-                header, graphic, pipeword, starword, command, opt, ctrl, \
-                    special, title, note, url, word = match.groups()
-                if pipeword is not None:
-                    out.append(self.maplink(pipeword, filename, 'l'))
-                elif starword is not None:
-                    out.extend(('<a name="', urllib.parse.quote_plus(starword),
-                                '" class="t">', html_escape[starword], '</a>'))
-                elif command is not None:
-                    out.extend(('<span class="e">', html_escape[command],
-                                '</span>'))
-                elif opt is not None:
-                    out.append(self.maplink(opt, filename, 'o'))
-                elif ctrl is not None:
-                    out.append(self.maplink(ctrl, filename, 'k'))
-                elif special is not None:
-                    out.append(self.maplink(special, filename, 's'))
-                elif title is not None:
-                    out.extend(('<span class="i">', html_escape[title],
-                                '</span>'))
-                elif note is not None:
-                    out.extend(('<span class="n">', html_escape[note],
-                                '</span>'))
-                elif header is not None:
-                    out.extend(('<span class="h">', html_escape[header[:-1]],
-                                '</span>'))
-                elif graphic is not None:
-                    out.append(html_escape[graphic[:-2]])
-                elif url is not None:
-                    out.extend(('<a class="u" href="', url, '">' +
-                                html_escape[url], '</a>'))
-                elif word is not None:
-                    out.append(self.maplink(word, filename))
-            if lastpos < len(line):
-                out.append(html_escape[line[lastpos:]])
-            out.append('\n')
-            if inexample == 1:
-                inexample = 2
-
-        header = []
-        header.append(HEAD.format(encoding=encoding, filename=filename))
-        header.append(HEAD_END)
-        if self._is_web_version and is_help_txt:
-            vers_note = VERSION_NOTE.replace('{version}', self._version) \
-                if self._version else ''
-            header.append(INTRO.replace('{vers-note}', vers_note))
-        if self._is_web_version:
-            header.append(SITENAVI_SEARCH)
-            sitenavi_footer = SITENAVI_WEB
-        else:
-            header.append(SITENAVI_PLAIN)
-            sitenavi_footer = SITENAVI_PLAIN
-        header.append(TEXTSTART)
-        return ''.join(chain(header, out, (FOOTER, sitenavi_footer, FOOTER2)))
-
-
-class HtmlEscCache(dict):
-    def __missing__(self, key):
-        r = key.replace('&', '&amp;') \
-               .replace('<', '&lt;') \
-               .replace('>', '&gt;')
-        self[key] = r
-        return r
-
-
-html_escape = HtmlEscCache()
-
-
-def slurp(filename):
-    try:
-        with open(filename, encoding='UTF-8') as f:
-            return f.read(), 'UTF-8'
-    except UnicodeError:
-        # 'ISO-8859-1' ?
-        with open(filename, encoding='latin-1') as f:
-            return f.read(), 'latin-1'
-
-
-def usage():
-    return "usage: " + sys.argv[0] + " IN_DIR OUT_DIR [BASENAMES...]"
-
-
-def main():
-    if len(sys.argv) < 3:
-        sys.exit(usage())
-
-    in_dir = sys.argv[1]
-    out_dir = sys.argv[2]
-    basenames = sys.argv[3:]
-
-    print("Processing tags...")
-    h2h = VimH2H(slurp(os.path.join(in_dir, 'tags'))[0], is_web_version=False)
-
-    if len(basenames) == 0:
-        basenames = os.listdir(in_dir)
-
-    for basename in basenames:
-        if os.path.splitext(basename)[1] != '.txt' and basename != 'tags':
-            print("Ignoring " + basename)
-            continue
-        print("Processing " + basename + "...")
-        path = os.path.join(in_dir, basename)
-        text, encoding = slurp(path)
-        outpath = os.path.join(out_dir, basename + '.html')
-        of = open(outpath, 'w')
-        of.write(h2h.to_html(basename, text, encoding))
-        of.close()
-
-
-main()
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index bca3dd816f..3ee9d8b5dd 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -2,9 +2,7 @@
 """Generates Nvim :help docs from C/Lua docstrings, using Doxygen.
 
 Also generates *.mpack files. To inspect the *.mpack structure:
-
-    :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack')))
-
+    :new | put=v:lua.vim.inspect(v:lua.vim.mpack.unpack(readfile('runtime/doc/api.mpack','B')))
 
 Flow:
     main
@@ -260,7 +258,7 @@ CONFIG = {
         'helptag_fmt': lambda name: (
             '*lua-treesitter-core*'
             if name.lower() == 'treesitter'
-            else f'*treesitter-{name.lower()}*'),
+            else f'*lua-treesitter-{name.lower()}*'),
         'fn_helptag_fmt': lambda fstem, name: (
             f'*{name}()*'
             if name != 'new'
@@ -287,7 +285,7 @@ annotation_map = {
     'FUNC_API_FAST': '|api-fast|',
     'FUNC_API_CHECK_TEXTLOCK': 'not allowed when |textlock| is active',
     'FUNC_API_REMOTE_ONLY': '|RPC| only',
-    'FUNC_API_LUA_ONLY': '|vim.api| only',
+    'FUNC_API_LUA_ONLY': 'Lua |vim.api| only',
 }
 
 
@@ -295,14 +293,16 @@ annotation_map = {
 # or if `cond()` is callable and returns True.
 def debug_this(o, cond=True):
     name = ''
+    if cond is False:
+        return
     if not isinstance(o, str):
         try:
             name = o.nodeName
             o = o.toprettyxml(indent='  ', newl='\n')
         except Exception:
             pass
-    if ((callable(cond) and cond())
-            or (not callable(cond) and cond)
+    if (cond is True
+            or (callable(cond) and cond())
             or (not callable(cond) and cond in o)):
         raise RuntimeError('xxx: {}\n{}'.format(name, o))
 
@@ -671,7 +671,7 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='',
         max_name_len = max_name(m.keys()) + 4
         out = ''
         for name, desc in m.items():
-            name = '    {}'.format('{{{}}}'.format(name).ljust(max_name_len))
+            name = '  • {}'.format('{{{}}}'.format(name).ljust(max_name_len))
             out += '{}{}\n'.format(name, desc)
         return out.rstrip()
 
@@ -801,7 +801,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
 
         prefix = '%s(' % name
         suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params
-                                   if a[0] not in ('void', 'Error'))
+                                   if a[0] not in ('void', 'Error', 'Arena'))
 
         if not fmt_vimhelp:
             c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
@@ -887,7 +887,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
 def fmt_doxygen_xml_as_vimhelp(filename, target):
     """Entrypoint for generating Vim :help from from Doxygen XML.
 
-    Returns 3 items:
+    Returns 2 items:
       1. Vim help text for functions found in `filename`.
       2. Vim help text for deprecated functions.
     """
@@ -1094,7 +1094,11 @@ def main(config, args):
                         fn_map_full.update(fn_map)
 
         if len(sections) == 0:
-            fail(f'no sections for target: {target}')
+            if target == 'lua':
+                fail(f'no sections for target: {target} (this usually means'
+                     + ' "luajit" was not found by scripts/lua2dox_filter)')
+            else:
+                fail(f'no sections for target: {target}')
         if len(sections) > len(CONFIG[target]['section_order']):
             raise RuntimeError(
                 'found new modules "{}"; update the "section_order" map'.format(
diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua
index 6a206066b8..86afe97a7e 100644
--- a/scripts/lua2dox.lua
+++ b/scripts/lua2dox.lua
@@ -50,108 +50,46 @@ However I have put in a hack that will insert the "missing" close paren.
 The effect is that you will get the function documented, but not with the parameter list you might expect.
 ]]
 
-local function class(BaseClass, ClassInitialiser)
-  local newClass = {}    -- a new class newClass
-  if not ClassInitialiser and type(BaseClass) == 'function' then
-    ClassInitialiser = BaseClass
-    BaseClass = nil
-  elseif type(BaseClass) == 'table' then
-    -- our new class is a shallow copy of the base class!
-    for i,v in pairs(BaseClass) do
-      newClass[i] = v
-    end
-    newClass._base = BaseClass
-  end
+local function class()
+  local newClass = {} -- a new class newClass
   -- the class will be the metatable for all its newInstanceects,
   -- and they will look up their methods in it.
   newClass.__index = newClass
 
   -- expose a constructor which can be called by <classname>(<args>)
-  local classMetatable = {}
-  classMetatable.__call = function(class_tbl, ...)
-    local newInstance = {}
-    setmetatable(newInstance,newClass)
-    --if init then
-    --  init(newInstance,...)
-    if class_tbl.init then
-      class_tbl.init(newInstance,...)
-    else
-      -- make sure that any stuff from the base class is initialized!
-      if BaseClass and BaseClass.init then
-        BaseClass.init(newInstance, ...)
-      end
-    end
-    return newInstance
-  end
-  newClass.init = ClassInitialiser
-  newClass.is_a = function(this, klass)
-    local thisMetatable = getmetatable(this)
-    while thisMetatable do
-      if thisMetatable == klass then
-        return true
+  setmetatable(newClass, {
+    __call = function(class_tbl, ...)
+      local newInstance = {}
+      setmetatable(newInstance, newClass)
+      --if init then
+      --  init(newInstance,...)
+      if class_tbl.init then
+        class_tbl.init(newInstance, ...)
       end
-      thisMetatable = thisMetatable._base
+      return newInstance
     end
-    return false
-  end
-  setmetatable(newClass, classMetatable)
+  })
   return newClass
 end
 
---! \class TCore_Clock
---! \brief a clock
-local TCore_Clock = class()
-
---! \brief get the current time
-function TCore_Clock.GetTimeNow()
-  local gettimeofday = os.gettimeofday  -- luacheck: ignore 143 Accessing an undefined field of a global variable.
-  if gettimeofday then
-    return gettimeofday()
-  else
-    return os.time()
-  end
-end
-
---! \brief constructor
-function TCore_Clock.init(this,T0)
-  if T0 then
-    this.t0 = T0
-  else
-    this.t0 = TCore_Clock.GetTimeNow()
-  end
-end
-
---! \brief get time string
-function TCore_Clock.getTimeStamp(this,T0)
-  local t0
-  if T0 then
-    t0 = T0
-  else
-    t0 = this.t0
-  end
-  return os.date('%c %Z',t0)
-end
-
-
 --! \brief write to stdout
 local function TCore_IO_write(Str)
-  if (Str) then
+  if Str then
     io.write(Str)
   end
 end
 
 --! \brief write to stdout
 local function TCore_IO_writeln(Str)
-  if (Str) then
+  if Str then
     io.write(Str)
   end
-  io.write("\n")
+  io.write('\n')
 end
 
-
 --! \brief trims a string
 local function string_trim(Str)
-  return Str:match("^%s*(.-)%s*$")
+  return Str:match('^%s*(.-)%s*$')
 end
 
 --! \brief split a string
@@ -159,26 +97,26 @@ end
 --! \param Str
 --! \param Pattern
 --! \returns table of string fragments
+---@return string[]
 local function string_split(Str, Pattern)
   local splitStr = {}
-  local fpat = "(.-)" .. Pattern
+  local fpat = '(.-)' .. Pattern
   local last_end = 1
-  local str, e, cap = string.find(Str,fpat, 1)
+  local str, e, cap = string.find(Str, fpat, 1)
   while str do
-    if str ~= 1 or cap ~= "" then
-      table.insert(splitStr,cap)
+    if str ~= 1 or cap ~= '' then
+      table.insert(splitStr, cap)
     end
-    last_end = e+1
-    str, e, cap = string.find(Str,fpat, last_end)
+    last_end = e + 1
+    str, e, cap = string.find(Str, fpat, last_end)
   end
   if last_end <= #Str then
-    cap = string.sub(Str,last_end)
+    cap = string.sub(Str, last_end)
     table.insert(splitStr, cap)
   end
   return splitStr
 end
 
-
 --! \class TCore_Commandline
 --! \brief reads/parses commandline
 local TCore_Commandline = class()
@@ -191,7 +129,7 @@ function TCore_Commandline.init(this)
 end
 
 --! \brief get value
-function TCore_Commandline.getRaw(this,Key,Default)
+function TCore_Commandline.getRaw(this, Key, Default)
   local val = this.argv[Key]
   if not val then
     val = Default
@@ -208,14 +146,13 @@ local TStream_Read = class()
 --! \brief get contents of file
 --!
 --! \param Filename name of file to read (or nil == stdin)
-function    TStream_Read.getContents(this,Filename)
+function TStream_Read.getContents(this, Filename)
   assert(Filename)
   -- get lines from file
   -- syphon lines to our table
-  --TCore_Debug_show_var('Filename',Filename)
-  local filecontents={}
+  local filecontents = {}
   for line in io.lines(Filename) do
-    table.insert(filecontents,line)
+    table.insert(filecontents, line)
   end
 
   if filecontents then
@@ -240,7 +177,7 @@ function TStream_Read.getLine(this)
     this.currentLine = nil
   else
     -- get line
-    if this.currentLineNo<=this.contentsLen then
+    if this.currentLineNo <= this.contentsLen then
       line = this.filecontents[this.currentLineNo]
       this.currentLineNo = this.currentLineNo + 1
     else
@@ -251,13 +188,13 @@ function TStream_Read.getLine(this)
 end
 
 --! \brief save line fragment
-function TStream_Read.ungetLine(this,LineFrag)
+function TStream_Read.ungetLine(this, LineFrag)
   this.currentLine = LineFrag
 end
 
 --! \brief is it eof?
 function TStream_Read.eof(this)
-  if this.currentLine or this.currentLineNo<=this.contentsLen then
+  if this.currentLine or this.currentLineNo <= this.contentsLen then
     return false
   end
   return true
@@ -272,32 +209,32 @@ function TStream_Write.init(this)
 end
 
 --! \brief write immediately
-function TStream_Write.write(_,Str)
+function TStream_Write.write(_, Str)
   TCore_IO_write(Str)
 end
 
 --! \brief write immediately
-function TStream_Write.writeln(_,Str)
+function TStream_Write.writeln(_, Str)
   TCore_IO_writeln(Str)
 end
 
 --! \brief write immediately
-function TStream_Write.writelnComment(_,Str)
+function TStream_Write.writelnComment(_, Str)
   TCore_IO_write('// ZZ: ')
   TCore_IO_writeln(Str)
 end
 
 --! \brief write to tail
-function TStream_Write.writelnTail(this,Line)
+function TStream_Write.writelnTail(this, Line)
   if not Line then
     Line = ''
   end
-  table.insert(this.tailLine,Line)
+  table.insert(this.tailLine, Line)
 end
 
 --! \brief output tail lines
 function TStream_Write.write_tailLines(this)
-  for _,line in ipairs(this.tailLine) do
+  for _, line in ipairs(this.tailLine) do
     TCore_IO_writeln(line)
   end
   TCore_IO_write('// Lua2DoX new eof')
@@ -307,9 +244,9 @@ end
 local TLua2DoX_filter = class()
 
 --! \brief allow us to do errormessages
-function TLua2DoX_filter.warning(this,Line,LineNo,Legend)
+function TLua2DoX_filter.warning(this, Line, LineNo, Legend)
   this.outStream:writelnTail(
-  '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"'
+    '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"'
   )
 end
 
@@ -318,47 +255,47 @@ end
 --! If the string has a comment on the end, this trims it off.
 --!
 local function TString_removeCommentFromLine(Line)
-  local pos_comment = string.find(Line,'%-%-')
+  local pos_comment = string.find(Line, '%-%-')
   local tailComment
   if pos_comment then
-    Line = string.sub(Line,1,pos_comment-1)
-    tailComment = string.sub(Line,pos_comment)
+    Line = string.sub(Line, 1, pos_comment - 1)
+    tailComment = string.sub(Line, pos_comment)
   end
-  return Line,tailComment
+  return Line, tailComment
 end
 
 --! \brief get directive from magic
 local function getMagicDirective(Line)
-  local macro,tail
+  local macro, tail
   local macroStr = '[\\@]'
-  local pos_macro = string.find(Line,macroStr)
+  local pos_macro = string.find(Line, macroStr)
   if pos_macro then
     --! ....\\ macro...stuff
     --! ....\@ macro...stuff
-    local line = string.sub(Line,pos_macro+1)
-    local space = string.find(line,'%s+')
+    local line = string.sub(Line, pos_macro + 1)
+    local space = string.find(line, '%s+')
     if space then
-      macro = string.sub(line,1,space-1)
-      tail  = string_trim(string.sub(line,space+1))
+      macro = string.sub(line, 1, space - 1)
+      tail = string_trim(string.sub(line, space + 1))
     else
       macro = line
-      tail  = ''
+      tail = ''
     end
   end
-  return macro,tail
+  return macro, tail
 end
 
 --! \brief check comment for fn
-local function checkComment4fn(Fn_magic,MagicLines)
+local function checkComment4fn(Fn_magic, MagicLines)
   local fn_magic = Fn_magic
   --    TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"')
 
-  local magicLines = string_split(MagicLines,'\n')
+  local magicLines = string_split(MagicLines, '\n')
 
-  local macro,tail
+  local macro, tail
 
   for _, line in ipairs(magicLines) do
-    macro,tail = getMagicDirective(line)
+    macro, tail = getMagicDirective(line)
     if macro == 'fn' then
       fn_magic = tail
       --    TCore_IO_writeln('// found fn "' .. fn_magic .. '"')
@@ -369,13 +306,16 @@ local function checkComment4fn(Fn_magic,MagicLines)
 
   return fn_magic
 end
+
+local types = { 'number', 'string', 'table', 'list', 'boolean', 'function' }
+
 --! \brief run the filter
-function TLua2DoX_filter.readfile(this,AppStamp,Filename)
+function TLua2DoX_filter.readfile(this, AppStamp, Filename)
   local inStream = TStream_Read()
   local outStream = TStream_Write()
   this.outStream = outStream -- save to this obj
 
-  if (inStream:getContents(Filename)) then
+  if inStream:getContents(Filename) then
     -- output the file
     local line
     local fn_magic -- function name/def from  magic comment
@@ -385,13 +325,14 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename)
     outStream:writelnTail('// #######################')
     outStream:writelnTail()
 
-    local state = ''  -- luacheck: ignore 231 variable is set but never accessed.
+    local state = '' -- luacheck: ignore 231 variable is set but never accessed.
     local offset = 0
+    local generic = {}
+    local l = 0
     while not (inStream:eof()) do
       line = string_trim(inStream:getLine())
-      --            TCore_Debug_show_var('inStream',inStream)
-      --            TCore_Debug_show_var('line',line )
-      if string.sub(line,1,2) == '--' then -- it's a comment
+      l = l + 1
+      if string.sub(line, 1, 2) == '--' then -- it's a comment
         -- Allow people to write style similar to EmmyLua (since they are basically the same)
         -- instead of silently skipping things that start with ---
         if string.sub(line, 3, 3) == '@' then -- it's a magic comment
@@ -405,55 +346,87 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename)
           local magic = string.sub(line, 4 + offset)
 
           local magic_split = string_split(magic, ' ')
-
-          local type_index = 2
           if magic_split[1] == 'param' then
-            type_index = type_index + 1
+            for _, type in ipairs(types) do
+              magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2')
+              magic =
+                magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2')
+            end
+            magic_split = string_split(magic, ' ')
+          elseif magic_split[1] == 'return' then
+            for _, type in ipairs(types) do
+              magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1')
+              magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1')
+            end
+            magic_split = string_split(magic, ' ')
           end
 
-          if magic_split[type_index] == 'number' or
-              magic_split[type_index] == 'number|nil' or
-              magic_split[type_index] == 'string' or
-              magic_split[type_index] == 'string|nil' or
-              magic_split[type_index] == 'table' or
-              magic_split[type_index] == 'table|nil' or
-              magic_split[type_index] == 'boolean' or
-              magic_split[type_index] == 'boolean|nil' or
-              magic_split[type_index] == 'function' or
-              magic_split[type_index] == 'function|nil'
-              then
-            magic_split[type_index] = '(' .. magic_split[type_index] .. ')'
+          if magic_split[1] == 'generic' then
+            local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)')
+            if generic_type == '' then
+              generic_type = 'any'
+            end
+            generic[generic_name] = generic_type
+          else
+            local type_index = 2
+            if magic_split[1] == 'param' then
+              type_index = type_index + 1
+            end
+
+            if magic_split[type_index] then
+              -- fix optional parameters
+              if magic_split[type_index] and magic_split[2]:find('%?$') then
+                if not magic_split[type_index]:find('nil') then
+                  magic_split[type_index] = magic_split[type_index] .. '|nil'
+                end
+                magic_split[2] = magic_split[2]:sub(1, -2)
+              end
+              -- replace generic types
+              if magic_split[type_index] then
+                for k, v in pairs(generic) do
+                  magic_split[type_index] = magic_split[type_index]:gsub(k, v)
+                end
+              end
+              -- surround some types by ()
+              for _, type in ipairs(types) do
+                magic_split[type_index] =
+                  magic_split[type_index]:gsub('^(' .. type .. '|nil):?$', '(%1)')
+                magic_split[type_index] =
+                  magic_split[type_index]:gsub('^(' .. type .. '):?$', '(%1)')
+              end
+            end
+
+            magic = table.concat(magic_split, ' ')
+
+            outStream:writeln('/// @' .. magic)
+            fn_magic = checkComment4fn(fn_magic, magic)
           end
-          magic = table.concat(magic_split, ' ')
-
-          outStream:writeln('/// @' .. magic)
-          fn_magic = checkComment4fn(fn_magic,magic)
-        elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment
-          local comment = string.sub(line,4)
-          outStream:writeln('/// '.. comment)
-        elseif string.sub(line,3,4)=='[[' then -- it's a long comment
-          line = string.sub(line,5) -- nibble head
+        elseif string.sub(line, 3, 3) == '-' then -- it's a nonmagic doc comment
+          local comment = string.sub(line, 4)
+          outStream:writeln('/// ' .. comment)
+        elseif string.sub(line, 3, 4) == '[[' then -- it's a long comment
+          line = string.sub(line, 5) -- nibble head
           local comment = ''
-          local closeSquare,hitend,thisComment
-          while (not hitend) and (not inStream:eof()) do
-            closeSquare = string.find(line,']]')
+          local closeSquare, hitend, thisComment
+          while not hitend and (not inStream:eof()) do
+            closeSquare = string.find(line, ']]')
             if not closeSquare then -- need to look on another line
               thisComment = line .. '\n'
               line = inStream:getLine()
             else
-              thisComment = string.sub(line,1,closeSquare-1)
+              thisComment = string.sub(line, 1, closeSquare - 1)
               hitend = true
 
               -- unget the tail of the line
               -- in most cases it's empty. This may make us less efficient but
               -- easier to program
-              inStream:ungetLine(string_trim(string.sub(line,closeSquare+2)))
+              inStream:ungetLine(string_trim(string.sub(line, closeSquare + 2)))
             end
             comment = comment .. thisComment
           end
-          if string.sub(comment,1,1)=='@' then -- it's a long magic comment
+          if string.sub(comment, 1, 1) == '@' then -- it's a long magic comment
             outStream:write('/*' .. comment .. '*/  ')
-            fn_magic = checkComment4fn(fn_magic,comment)
+            fn_magic = checkComment4fn(fn_magic, comment)
           else -- discard
             outStream:write('/* zz:' .. comment .. '*/  ')
             fn_magic = nil
@@ -467,64 +440,62 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename)
           fn_magic = nil
         end
       elseif string.find(line, '^function') or string.find(line, '^local%s+function') then
-        state = 'in_function'  -- it's a function
-        local pos_fn = string.find(line,'function')
+        generic = {}
+        state = 'in_function' -- it's a function
+        local pos_fn = string.find(line, 'function')
         -- function
         -- ....v...
         if pos_fn then
           -- we've got a function
-          local fn_type
-          if string.find(line,'^local%s+') then
-            fn_type = ''--'static ' -- static functions seem to be excluded
-          else
-            fn_type = ''
-          end
-          local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8)))
+          local fn = TString_removeCommentFromLine(string_trim(string.sub(line, pos_fn + 8)))
           if fn_magic then
             fn = fn_magic
           end
 
-          if string.sub(fn,1,1)=='(' then
+          if string.sub(fn, 1, 1) == '(' then
             -- it's an anonymous function
             outStream:writelnComment(line)
           else
             -- fn has a name, so is interesting
 
             -- want to fix for iffy declarations
-            local open_paren = string.find(fn,'[%({]')
+            local open_paren = string.find(fn, '[%({]')
             if open_paren then
               -- we might have a missing close paren
-              if not string.find(fn,'%)') then
+              if not string.find(fn, '%)') then
                 fn = fn .. ' ___MissingCloseParenHere___)'
               end
             end
 
             -- Big hax
-            if string.find(fn, ":") then
+            if string.find(fn, ':') then
               -- TODO: We need to add a first parameter of "SELF" here
               -- local colon_place = string.find(fn, ":")
               -- local name = string.sub(fn, 1, colon_place)
-              fn = fn:gsub(":", ".", 1)
-              outStream:writeln("/// @param self")
+              fn = fn:gsub(':', '.', 1)
+              outStream:writeln('/// @param self')
 
-              local paren_start = string.find(fn, "(", 1, true)
-              local paren_finish = string.find(fn, ")", 1, true)
+              local paren_start = string.find(fn, '(', 1, true)
+              local paren_finish = string.find(fn, ')', 1, true)
 
               -- Nothing in between the parens
               local comma
               if paren_finish == paren_start + 1 then
-                comma = ""
+                comma = ''
               else
-                comma = ", "
+                comma = ', '
               end
-              fn = string.sub(fn, 1, paren_start) .. "self" .. comma .. string.sub(fn, paren_start + 1)
+              fn = string.sub(fn, 1, paren_start)
+                .. 'self'
+                .. comma
+                .. string.sub(fn, paren_start + 1)
             end
 
             -- add vanilla function
-            outStream:writeln(fn_type .. 'function ' .. fn .. '{}')
+            outStream:writeln('function ' .. fn .. '{}')
           end
         else
-          this:warning(inStream:getLineNo(),'something weird here')
+          this:warning(inStream:getLineNo(), 'something weird here')
         end
         fn_magic = nil -- mustn't indavertently use it again
 
@@ -535,8 +506,8 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename)
 
       --   fn_magic = nil -- mustn't indavertently use it again
       else
-        state = ''  -- unknown
-        if #line>0 then  -- we don't know what this line means, so just comment it out
+        state = '' -- unknown
+        if #line > 0 then -- we don't know what this line means, so just comment it out
           outStream:writeln('// zz: ' .. line)
         else
           outStream:writeln() -- keep this line blank
@@ -556,16 +527,14 @@ local TApp = class()
 
 --! \brief constructor
 function TApp.init(this)
-  local t0 = TCore_Clock()
-  this.timestamp = t0:getTimeStamp()
+  this.timestamp = os.date('%c %Z', os.time())
   this.name = 'Lua2DoX'
   this.version = '0.2 20130128'
   this.copyright = 'Copyright (c) Simon Dales 2012-13'
 end
 
 function TApp.getRunStamp(this)
-  return this.name .. ' (' .. this.version .. ') '
-  .. this.timestamp
+  return this.name .. ' (' .. this.version .. ') ' .. this.timestamp
 end
 
 function TApp.getVersion(this)
@@ -602,8 +571,7 @@ else
   local filename = argv1
 
   local filter = TLua2DoX_filter()
-  filter:readfile(appStamp,filename)
+  filter:readfile(appStamp, filename)
 end
 
-
 --eof
diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter
index 22484a807f..e3fa95d0cf 100755
--- a/scripts/lua2dox_filter
+++ b/scripts/lua2dox_filter
@@ -22,67 +22,69 @@
 LANG=""
 
 ##! \brief test executable to see if it exists
-test_executable(){
-	P_EXE="$1"
-	#########
-	WHICH=`which ${P_EXE}`
-	if test -z "${WHICH}"
-	then
-		echo "not found \"${P_EXE}\""
-	else
-		EXE="${P_EXE}"
-	fi
-	}
+test_executable() {
+  P_EXE="$1"
+  #########
+  WHICH=$(which "$P_EXE")
+  if test -z "${WHICH}"; then
+    echo "not found \"${P_EXE}\""
+  else
+    EXE="${P_EXE}"
+  fi
+}
 
 ##! \brief sets the lua interpreter
-set_lua(){
-	if test -z "${EXE}"; then
-		test_executable 'luajit'
-	fi
+set_lua() {
+  if test -z "${EXE}"; then
+    test_executable '.deps/usr/bin/luajit'
+  fi
+
+  if test -z "${EXE}"; then
+    test_executable 'luajit'
+  fi
 
-	if test -z "${EXE}"; then
-		test_executable 'lua'
-	fi
+  if test -z "${EXE}"; then
+    test_executable 'lua'
+  fi
 }
 
 ##! \brief makes canonical name of file
-##! 
+##!
 ##! Note that "readlink -f" doesn't work in MacOSX
-##! 
-do_readlink(){
-	pushd . > /dev/null
-	TARGET_FILE=$1
-	
-	cd `dirname $TARGET_FILE`
-	TARGET_FILE=`basename $TARGET_FILE`
-	
-	# Iterate down a (possible) chain of symlinks
-	while [ -L "$TARGET_FILE" ]
-	do
-		TARGET_FILE=`readlink $TARGET_FILE`
-		cd `dirname $TARGET_FILE`
-		TARGET_FILE=`basename $TARGET_FILE`
-	done
-	
-	PHYS_DIR=`pwd -P`
-	RESULT=$PHYS_DIR
-	popd > /dev/null
-	}
+##!
+do_readlink() {
+  pushd . > /dev/null
+  TARGET_FILE=$1
+
+  cd "$(dirname $TARGET_FILE)"
+  TARGET_FILE=$(basename "$TARGET_FILE")
+
+  # Iterate down a (possible) chain of symlinks
+  while [ -L "$TARGET_FILE" ]; do
+    TARGET_FILE=$(readlink "$TARGET_FILE")
+    cd $(dirname "$TARGET_FILE")
+    TARGET_FILE=$(basename "$TARGET_FILE")
+  done
+
+  PHYS_DIR=$(pwd -P)
+  RESULT=$PHYS_DIR
+  popd > /dev/null
+}
 
 ##main
 set_lua
-if test -z "${EXE}"
-then
-	echo "no lua interpreter available"
+if test -z "${EXE}"; then
+  echo "no lua interpreter found"
+  exit 1
 else
-	BASENAME=`basename "$0"`
-	do_readlink "$0"
-	DIRNAME="${RESULT}"
-		
-	LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}"
-	#echo "lua[${LUASCRIPT}]"
+  BASENAME=$(basename "$0")
+  do_readlink "$0"
+  DIRNAME="${RESULT}"
 
-	${EXE} ${LUASCRIPT} $@
+  LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}"
+  #echo "lua[${LUASCRIPT}]"
+
+  ${EXE} ${LUASCRIPT} $@
 fi
-# 
+
 ##eof