path: root/scripts/gen_help_html.lua

-- Converts Vim :help files to HTML.  Validates |tag| links and document syntax (parser errors).
--
-- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags).
--
-- USAGE (GENERATE HTML):
--   1. Run `make helptags` first; this script depends on vim.fn.taglist().
--   2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')"
--      - Read the docstring at gen().
--   3. cd target/dir/ && jekyll serve --host 0.0.0.0
--   4. Visit http://localhost:4000/…/help.txt.html
--
-- USAGE (VALIDATE):
--   1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate()"
--      - validate() is 10x faster than gen(), so it is used in CI.
--
-- SELF-TEST MODE:
--   1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()"
--
-- NOTES:
--   * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too
--     slow (~1 min) to run in per-commit CI.
--   * visit_node() is the core function used by gen() to traverse the document tree and produce HTML.
--   * visit_validate() is the core function used by validate().
--   * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout.

local tagmap = nil
local helpfiles = nil
local invalid_links = {}
local invalid_urls = {}
local invalid_spelling = {}
local spell_dict = {
  Neovim = 'Nvim',
  NeoVim = 'Nvim',
  neovim = 'Nvim',
  lua = 'Lua',
  VimL = 'Vimscript',
  vimL = 'Vimscript',
  viml = 'Vimscript',
}
local language = nil

local M = {}

-- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs).
-- All other files are "legacy" files which require fixed-width layout.
local new_layout = {
  ['api.txt'] = true,
  ['lsp.txt'] = true,
  ['channel.txt'] = true,
  ['deprecated.txt'] = true,
  ['develop.txt'] = true,
  ['lua.txt'] = true,
  ['luaref.txt'] = true,
  ['news.txt'] = true,
  ['nvim.txt'] = true,
  ['pi_health.txt'] = true,
  ['provider.txt'] = true,
  ['ui.txt'] = true,
}

-- TODO: These known invalid |links| require an update to the relevant docs.
local exclude_invalid = {
  ["'string'"] = "eval.txt",
  Query = 'treesitter.txt',
  matchit = 'vim_diff.txt',
  ["set!"] = "treesitter.txt",
}

-- False-positive "invalid URLs".
local exclude_invalid_urls = {
  ["http://"] = "usr_23.txt",
  ["http://."] = "usr_23.txt",
  ["http://aspell.net/man-html/Affix-Compression.html"] = "spell.txt",
  ["http://aspell.net/man-html/Phonetic-Code.html"] = "spell.txt",
  ["http://canna.sourceforge.jp/"] = "mbyte.txt",
  ["http://gnuada.sourceforge.net"] = "ft_ada.txt",
  ["http://lua-users.org/wiki/StringLibraryTutorial"] = "lua.txt",
  ["http://michael.toren.net/code/"] = "pi_tar.txt",
  ["http://papp.plan9.de"] = "syntax.txt",
  ["http://wiki.services.openoffice.org/wiki/Dictionaries"] = "spell.txt",
  ["http://www.adapower.com"] = "ft_ada.txt",
  ["http://www.jclark.com/"] = "quickfix.txt",
}

-- Deprecated, brain-damaged files that I don't care about.
local ignore_errors = {
  ['pi_netrw.txt'] = true,
}

local function tofile(fname, text)
  local f = io.open(fname, 'w')
  if not f then
    error(('failed to write: %s'):format(f))
  else
    f:write(text)
    f:close()
  end
end

local function html_esc(s)
  return s:gsub(
    '&', '&'):gsub(
    '<', '&lt;'):gsub(
    '>', '&gt;')
end

local function url_encode(s)
  -- Credit: tpope / vim-unimpaired
  -- NOTE: these chars intentionally *not* escaped: ' ( )
  return vim.fn.substitute(vim.fn.iconv(s, 'latin1', 'utf-8'),
    [=[[^A-Za-z0-9()'_.~-]]=],
    [=[\="%".printf("%02X",char2nr(submatch(0)))]=],
    'g')
end

local function expandtabs(s)
  return s:gsub('\t', (' '):rep(8))
end

local function to_titlecase(s)
  local text = ''
  for w in vim.gsplit(s, '[ \t]+') do
    text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2))
  end
  return text
end

local function to_heading_tag(text)
  -- Prepend "_" to avoid conflicts with actual :help tags.
  return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown'
end

local function basename_noext(f)
 return vim.fs.basename(f:gsub('%.txt', ''))
end

local function is_blank(s)
  return not not s:find([[^[\t ]*$]])
end

local function trim(s, dir)
  return vim.fn.trim(s, '\r\t\n ', dir or 0)
end

--- Removes common punctuation from URLs.
---
--- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc
---
--- @returns (fixed_url, removed_chars) where `removed_chars` is in the order found in the input.
local function fix_url(url)
  local removed_chars = ''
  local fixed_url = url
  -- Remove up to one of each char from end of the URL, in this order.
  for _, c in ipairs({ '.', ')', }) do
    if fixed_url:sub(-1) == c then
      removed_chars = c .. removed_chars
      fixed_url = fixed_url:sub(1, -2)
    end
  end
  return fixed_url, removed_chars
end

--- Checks if a given line is a "noise" line that doesn't look good in HTML form.
local function is_noise(line, noise_lines)
  if (
    -- First line is always noise.
    (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0)
    or line:find('Type .*gO.* to see the table of contents')
    -- Title line of traditional :help pages.
    -- Example: "NVIM REFERENCE MANUAL    by ..."
    or line: