diff options
Diffstat (limited to 'scripts/gen_help_html.lua')
| -rw-r--r-- | scripts/gen_help_html.lua | 1491 |
1 files changed, 0 insertions, 1491 deletions
diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua deleted file mode 100644 index 53a65fd65f..0000000000 --- a/scripts/gen_help_html.lua +++ /dev/null @@ -1,1491 +0,0 @@ ---- Converts Nvim :help files to HTML. Validates |tag| links and document syntax (parser errors). --- --- USAGE (For CI/local testing purposes): Simply `make lintdoc` or `scripts/lintdoc.lua`, which --- basically does the following: --- 1. :helptags ALL --- 2. nvim -V1 -es +"lua require('scripts.gen_help_html').run_validate()" +q --- 3. nvim -V1 -es +"lua require('scripts.gen_help_html').test_gen()" +q --- --- USAGE (GENERATE HTML): --- 1. `:helptags ALL` first; this script depends on vim.fn.taglist(). --- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./runtime/doc', 'target/dir/')" +q --- - Read the docstring at gen(). --- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 --- 4. Visit http://localhost:4000/…/help.txt.html --- --- USAGE (VALIDATE): --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate('./runtime/doc')" +q --- - validate() is 10x faster than gen(), so it is used in CI. --- --- SELF-TEST MODE: --- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +q --- --- NOTES: --- * This script is used by the automation repo: https://github.com/neovim/doc --- * :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). --- * gen() and validate() are the primary (programmatic) entrypoints. validate() only exists --- because gen() is too slow (~1 min) to run in per-commit CI. --- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. --- * visit_validate() is the core function used by validate(). --- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. - -local tagmap = nil ---@type table<string, string> -local helpfiles = nil ---@type string[] -local invalid_links = {} ---@type table<string, any> -local invalid_urls = {} ---@type table<string, any> -local invalid_spelling = {} ---@type table<string, table<string, string>> -local spell_dict = { - Neovim = 'Nvim', - NeoVim = 'Nvim', - neovim = 'Nvim', - lua = 'Lua', - VimL = 'Vimscript', - vimL = 'Vimscript', - viml = 'Vimscript', - ['tree-sitter'] = 'treesitter', - ['Tree-sitter'] = 'Treesitter', -} ---- specify the list of keywords to ignore (i.e. allow), or true to disable spell check completely. ---- @type table<string, true|string[]> -local spell_ignore_files = { - ['credits.txt'] = { 'Neovim' }, - ['news.txt'] = { 'tree-sitter' }, -- in news, may refer to the upstream "tree-sitter" library - ['news-0.10.txt'] = { 'tree-sitter' }, -} -local language = nil - -local M = {} - --- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). --- All other files are "legacy" files which require fixed-width layout. -local new_layout = { - ['api.txt'] = true, - ['lsp.txt'] = true, - ['channel.txt'] = true, - ['deprecated.txt'] = true, - ['develop.txt'] = true, - ['dev_style.txt'] = true, - ['dev_theme.txt'] = true, - ['dev_tools.txt'] = true, - ['dev_vimpatch.txt'] = true, - ['editorconfig.txt'] = true, - ['faq.txt'] = true, - ['gui.txt'] = true, - ['intro.txt'] = true, - ['lua.txt'] = true, - ['luaref.txt'] = true, - ['news.txt'] = true, - ['news-0.9.txt'] = true, - ['news-0.10.txt'] = true, - ['nvim.txt'] = true, - ['provider.txt'] = true, - ['tui.txt'] = true, - ['ui.txt'] = true, - ['vim_diff.txt'] = true, -} - --- Map of new:old pages, to redirect renamed pages. -local redirects = { - ['credits'] = 'backers', - ['tui'] = 'term', - ['terminal'] = 'nvim_terminal_emulator', -} - --- TODO: These known invalid |links| require an update to the relevant docs. -local exclude_invalid = { - ["'string'"] = 'eval.txt', - Query = 'treesitter.txt', - matchit = 'vim_diff.txt', - ['set!'] = 'treesitter.txt', -} - --- False-positive "invalid URLs". -local exclude_invalid_urls = { - ['http://'] = 'usr_23.txt', - ['http://.'] = 'usr_23.txt', - ['http://aspell.net/man-html/Affix-Compression.html'] = 'spell.txt', - ['http://aspell.net/man-html/Phonetic-Code.html'] = 'spell.txt', - ['http://canna.sourceforge.jp/'] = 'mbyte.txt', - ['http://gnuada.sourceforge.net'] = 'ft_ada.txt', - ['http://lua-users.org/wiki/StringLibraryTutorial'] = 'lua.txt', - ['http://michael.toren.net/code/'] = 'pi_tar.txt', - ['http://papp.plan9.de'] = 'syntax.txt', - ['http://wiki.services.openoffice.org/wiki/Dictionaries'] = 'spell.txt', - ['http://www.adapower.com'] = 'ft_ada.txt', - ['http://www.jclark.com/'] = 'quickfix.txt', - ['http://oldblog.antirez.com/post/redis-and-scripting.html'] = 'faq.txt', -} - --- Deprecated, brain-damaged files that I don't care about. -local ignore_errors = { - ['pi_netrw.txt'] = true, - ['credits.txt'] = true, -} - -local function tofile(fname, text) - local f = io.open(fname, 'w') - if not f then - error(('failed to write: %s'):format(f)) - else - f:write(text) - f:close() - end -end - ----@type fun(s: string): string -local function html_esc(s) - return (s:gsub('&', '&'):gsub('<', '<'):gsub('>', '>')) -end - -local function url_encode(s) - -- Credit: tpope / vim-unimpaired - -- NOTE: these chars intentionally *not* escaped: ' ( ) - return vim.fn.substitute( - vim.fn.iconv(s, 'latin1', 'utf-8'), - [=[[^A-Za-z0-9()'_.~-]]=], - [=[\="%".printf("%02X",char2nr(submatch(0)))]=], - 'g' - ) -end - -local function expandtabs(s) - return s:gsub('\t', (' '):rep(8)) --[[ @as string ]] -end - -local function to_titlecase(s) - local text = '' - for w in vim.gsplit(s, '[ \t]+') do - text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2)) - end - return text -end - -local function to_heading_tag(text) - -- Prepend "_" to avoid conflicts with actual :help tags. - return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown' -end - -local function basename_noext(f) - return vim.fs.basename(f:gsub('%.txt', '')) -end - -local function is_blank(s) - return not not s:find([[^[\t ]*$]]) -end - ----@type fun(s: string, dir?:0|1|2): string -local function trim(s, dir) - return vim.fn.trim(s, '\r\t\n ', dir or 0) -end - ---- Removes common punctuation from URLs. ---- ---- TODO: fix this in the parser instead... https://github.com/neovim/tree-sitter-vimdoc ---- ---- @param url string ---- @return string, string (fixed_url, removed_chars) where `removed_chars` is in the order found in the input. -local function fix_url(url) - local removed_chars = '' - local fixed_url = url - -- Remove up to one of each char from end of the URL, in this order. - for _, c in ipairs({ '.', ')' }) do - if fixed_url:sub(-1) == c then - removed_chars = c .. removed_chars - fixed_url = fixed_url:sub(1, -2) - end - end - return fixed_url, removed_chars -end - ---- Checks if a given line is a "noise" line that doesn't look good in HTML form. -local function is_noise(line, noise_lines) - if - -- First line is always noise. - (noise_lines ~= nil and vim.tbl_count(noise_lines) == 0) - or line:find('Type .*gO.* to see the table of contents') - -- Title line of traditional :help pages. - -- Example: "NVIM REFERENCE MANUAL by ..." - or line:find([[^%s*N?VIM[ \t]*REFERENCE[ \t]*MANUAL]]) - -- First line of traditional :help pages. - -- Example: "*api.txt* Nvim" - or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') - -- modeline - -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" - or line:find('^%s*vim?%:.*ft=help') - or line:find('^%s*vim?%:.*filetype=help') - or line:find('[*>]local%-additions[*<]') - then - -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) - table.insert(noise_lines or {}, line) - return true - end - return false -end - ---- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. ---- @return string -local function get_bug_url_vimdoc(fname, to_fname, sample_text) - local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ( - 'https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' - .. vim.fs.basename(fname) - .. '+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' - .. this_url - .. '%0D%0DContext%3A%0D%0D%60%60%60%0D' - .. url_encode(sample_text) - .. '%0D%60%60%60' - ) - return bug_url -end - ---- Creates a github issue URL at neovim/neovim with prefilled content. ---- @return string -local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) - local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ( - 'https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' - .. vim.fs.basename(fname) - .. '+&body=%60gen_help_html.lua%60+problem+at%3A+' - .. this_url - .. '%0D' - .. (token_name and '+unhandled+token%3A+%60' .. token_name .. '%60' or '') - .. '%0DContext%3A%0D%0D%60%60%60%0D' - .. url_encode(sample_text) - .. '%0D%60%60%60' - ) - return bug_url -end - ---- Gets a "foo.html" name from a "foo.txt" helpfile name. -local function get_helppage(f) - if not f then - return nil - end - -- Special case: help.txt is the "main landing page" of :help files, not index.txt. - if f == 'index.txt' then - return 'vimindex.html' - elseif f == 'help.txt' then - return 'index.html' - end - - return (f:gsub('%.txt$', '')) .. '.html' -end - ---- Counts leading spaces (tab=8) to decide the indent size of multiline text. ---- ---- Blank lines (empty or whitespace-only) are ignored. -local function get_indent(s) - local min_indent = nil - for line in vim.gsplit(s, '\n') do - if line and not is_blank(line) then - local ws = expandtabs(line:match('^%s+') or '') - min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent - end - end - return min_indent or 0 -end - ---- Removes the common indent level, after expanding tabs to 8 spaces. -local function trim_indent(s) - local indent_size = get_indent(s) - local trimmed = '' - for line in vim.gsplit(s, '\n') do - line = expandtabs(line) - trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1)) - end - return trimmed:sub(1, -2) -end - ---- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. ----@param node TSNode ----@param bufnr integer ----@param offset integer -local function getbuflinestr(node, bufnr, offset) - local line1, _, line2, _ = node:range() - line1 = line1 - offset - line2 = line2 + offset - local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1) - return table.concat(lines, '\n') -end - ---- Gets the whitespace just before `node` from the raw buffer text. ---- Needed for preformatted `old` lines. ----@param node TSNode ----@param bufnr integer ----@return string -local function getws(node, bufnr) - local line1, c1, line2, _ = node:range() - ---@type string - local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] - local text_before = raw:sub(1, c1) - local leading_ws = text_before:match('%s+$') or '' - return leading_ws -end - -local function get_tagname(node, bufnr) - local text = vim.treesitter.get_node_text(node, bufnr) - local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') - and ("'%s'"):format(text) - or text - local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" - local helppage = get_helppage(helpfile) -- "api.html" - return helppage, tag -end - ---- Returns true if the given invalid tagname is a false positive. -local function ignore_invalid(s) - return not not ( - exclude_invalid[s] - -- Strings like |~/====| appear in various places and the parser thinks they are links, but they - -- are just table borders. - or s:find('===') - or s:find('%-%-%-') - ) -end - -local function ignore_parse_error(fname, s) - if ignore_errors[vim.fs.basename(fname)] then - return true - end - -- Ignore parse errors for unclosed tag. - -- This is common in vimdocs and is treated as plaintext by :help. - return s:find("^[`'|*]") -end - ----@param node TSNode -local function has_ancestor(node, ancestor_name) - local p = node ---@type TSNode? - while p do - p = p:parent() - if not p or p:type() == 'help_file' then - break - elseif p:type() == ancestor_name then - return true - end - end - return false -end - ---- Gets the first matching child node matching `name`. ----@param node TSNode -local function first(node, name) - for c, _ in node:iter_children() do - if c:named() and c:type() == name then - return c - end - end - return nil -end - -local function validate_link(node, bufnr, fname) - local helppage, tagname = get_tagname(node:child(1), bufnr) - local ignored = false - if not tagmap[tagname] then - ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname) - if not ignored then - invalid_links[tagname] = vim.fs.basename(fname) - end - end - return helppage, tagname, ignored -end - ---- TODO: port the logic from scripts/check_urls.vim -local function validate_url(text, fname) - local ignored = false - if ignore_errors[vim.fs.basename(fname)] then - ignored = true - elseif text:find('http%:') and not exclude_invalid_urls[text] then - invalid_urls[text] = vim.fs.basename(fname) - end - return ignored -end - ---- Traverses the tree at `root` and checks that |tag| links point to valid helptags. ----@param root TSNode ----@param level integer ----@param lang_tree TSTree ----@param opt table ----@param stats table -local function visit_validate(root, level, lang_tree, opt, stats) - level = level or 0 - local node_name = (root.named and root:named()) and root:type() or nil - -- Parent kind (string). - local parent = root:parent() and root:parent():type() or nil - local toplevel = level < 1 - local function node_text(node) - return vim.treesitter.get_node_text(node or root, opt.buf) - end - local text = trim(node_text()) - - if root:child_count() > 0 then - for node, _ in root:iter_children() do - if node:named() then - visit_validate(node, level + 1, lang_tree, opt, stats) - end - end - end - - if node_name == 'ERROR' then - if ignore_parse_error(opt.fname, text) then - return - end - -- Store the raw text to give context to the error report. - local sample_text = not toplevel and getbuflinestr(root, opt.buf, 0) or '[top level!]' - -- Flatten the sample text to a single, truncated line. - sample_text = vim.trim(sample_text):gsub('[\t\n]', ' '):sub(1, 80) - table.insert(stats.parse_errors, sample_text) - elseif - (node_name == 'word' or node_name == 'uppercase_name') - and (not vim.tbl_contains({ 'codespan', 'taglink', 'tag' }, parent)) - then - local text_nopunct = vim.fn.trim(text, '.,', 0) -- Ignore some punctuation. - local fname_basename = assert(vim.fs.basename(opt.fname)) - if spell_dict[text_nopunct] then - local should_ignore = ( - spell_ignore_files[fname_basename] == true - or vim.tbl_contains( - (spell_ignore_files[fname_basename] or {}) --[[ @as string[] ]], - text_nopunct - ) - ) - if not should_ignore then - invalid_spelling[text_nopunct] = invalid_spelling[text_nopunct] or {} - invalid_spelling[text_nopunct][fname_basename] = node_text(root:parent()) - end - end - elseif node_name == 'url' then - local fixed_url, _ = fix_url(trim(text)) - validate_url(fixed_url, opt.fname) - elseif node_name == 'taglink' or node_name == 'optionlink' then - local _, _, _ = validate_link(root, opt.buf, opt.fname) - end -end - --- Fix tab alignment issues caused by concealed characters like |, `, * in tags --- and code blocks. ----@param text string ----@param next_node_text string -local function fix_tab_after_conceal(text, next_node_text) - -- Vim tabs take into account the two concealed characters even though they - -- are invisible, so we need to add back in the two spaces if this is - -- followed by a tab to make the tab alignment to match Vim's behavior. - if string.sub(next_node_text, 1, 1) == '\t' then - text = text .. ' ' - end - return text -end - ----@class (exact) nvim.gen_help_html.heading ----@field name string ----@field subheadings nvim.gen_help_html.heading[] ----@field tag string - --- Generates HTML from node `root` recursively. ----@param root TSNode ----@param level integer ----@param lang_tree TSTree ----@param headings nvim.gen_help_html.heading[] ----@param opt table ----@param stats table -local function visit_node(root, level, lang_tree, headings, opt, stats) - level = level or 0 - - local node_name = (root.named and root:named()) and root:type() or nil - -- Previous sibling kind (string). - local prev = root:prev_sibling() - and (root:prev_sibling().named and root:prev_sibling():named()) - and root:prev_sibling():type() - or nil - -- Next sibling kind (string). - local next_ = root:next_sibling() - and (root:next_sibling().named and root:next_sibling():named()) - and root:next_sibling():type() - or nil - -- Parent kind (string). - local parent = root:parent() and root:parent():type() or nil - -- Gets leading whitespace of `node`. - local function ws(node) - node = node or root - local ws_ = getws(node, opt.buf) - -- XXX: first node of a (line) includes whitespace, even after - -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ? - if ws_ == '' then - ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or '' - end - return ws_ - end - local function node_text(node, ws_) - node = node or root - ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or '' - return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) - end - - local text = '' - local trimmed ---@type string - if root:named_child_count() == 0 or node_name == 'ERROR' then - text = node_text() - trimmed = html_esc(trim(text)) - text = html_esc(text) - else - -- Process children and join them with whitespace. - for node, _ in root:iter_children() do - if node:named() then - local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) - text = string.format('%s%s', text, r) - end - end - trimmed = trim(text) - end - - if node_name == 'help_file' then -- root node - return text - elseif node_name == 'url' then - local fixed_url, removed_chars = fix_url(trimmed) - return ('%s<a href="%s">%s</a>%s'):format(ws(), fixed_url, fixed_url, removed_chars) - elseif node_name == 'word' or node_name == 'uppercase_name' then - return text - elseif node_name == 'note' then - return ('<b>%s</b>'):format(text) - elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then - if is_noise(text, stats.noise_lines) then - return '' -- Discard common "noise" lines. - end - -- Remove tags from ToC text. - local heading_node = first(root, 'heading') - local hname = trim(node_text(heading_node):gsub('%*.*%*', '')) - if not heading_node or hname == '' then - return '' -- Spurious "===" or "---" in the help doc. - end - - -- Generate an anchor id from the heading text. - local tagname = to_heading_tag(hname) - if node_name == 'h1' or #headings == 0 then - ---@type nvim.gen_help_html.heading - local heading = { name = hname, subheadings = {}, tag = tagname } - headings[#headings + 1] = heading - else - table.insert( - headings[#headings].subheadings, - { name = hname, subheadings = {}, tag = tagname } - ) - end - local el = node_name == 'h1' and 'h2' or 'h3' - return ('<%s id="%s" class="help-heading">%s</%s>\n'):format(el, tagname, trimmed, el) - elseif node_name == 'heading' then - return trimmed - elseif node_name == 'column_heading' or node_name == 'column_name' then - if root:has_error() then - return text - end - return ('<div class="help-column_heading">%s</div>'):format(text) - elseif node_name == 'block' then - if is_blank(text) then - return '' - end - if opt.old then - -- XXX: Treat "old" docs as preformatted: they use indentation for layout. - -- Trim trailing newlines to avoid too much whitespace between divs. - return ('<div class="old-help-para">%s</div>\n'):format(trim(text, 2)) - end - return string.format('<div class="help-para">\n%s\n</div>\n', text) - elseif node_name == 'line' then - if - (parent ~= 'codeblock' or parent ~= 'code') - and (is_blank(text) or is_noise(text, stats.noise_lines)) - then - return '' -- Discard common "noise" lines. - end - -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. - local div = opt.old - and root:child(0) - and vim.list_contains({ 'column_heading', 'h1', 'h2', 'h3' }, root:child(0):type()) - return string.format('%s%s', div and trim(text) or text, div and '' or '\n') - elseif node_name == 'line_li' then - local sib = root:prev_sibling() - local prev_li = sib and sib:type() == 'line_li' - - if not prev_li then - opt.indent = 1 - else - -- The previous listitem _sibling_ is _logically_ the _parent_ if it is indented less. - local parent_indent = get_indent(node_text(sib)) - local this_indent = get_indent(node_text()) - if this_indent > parent_indent then - opt.indent = opt.indent + 1 - elseif this_indent < parent_indent then - opt.indent = math.max(1, opt.indent - 1) - end - end - local margin = opt.indent == 1 and '' or ('margin-left: %drem;'):format((1.5 * opt.indent)) - - return string.format('<div class="help-li" style="%s">%s</div>', margin, text) - elseif node_name == 'taglink' or node_name == 'optionlink' then - local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname) - if ignored then - return text - end - local s = ('%s<a href="%s#%s">%s</a>'):format( - ws(), - helppage, - url_encode(tagname), - html_esc(tagname) - ) - if opt.old and node_name == 'taglink' then - s = fix_tab_after_conceal(s, node_text(root:next_sibling())) - end - return s - elseif vim.list_contains({ 'codespan', 'keycode' }, node_name) then - if root:has_error() then - return text - end - local s = ('%s<code>%s</code>'):format(ws(), trimmed) - if opt.old and node_name == 'codespan' then - s = fix_tab_after_conceal(s, node_text(root:next_sibling())) - end - return s - elseif node_name == 'argument' then - return ('%s<code>{%s}</code>'):format(ws(), text) - elseif node_name == 'codeblock' then - return text - elseif node_name == 'language' then - language = node_text(root) - return '' - elseif node_name == 'code' then -- Highlighted codeblock (child). - if is_blank(text) then - return '' - end - local code ---@type string - if language then - code = ('<pre><code class="language-%s">%s</code></pre>'):format( - language, - trim(trim_indent(text), 2) - ) - language = nil - else - code = ('<pre>%s</pre>'):format(trim(trim_indent(text), 2)) - end - return code - elseif node_name == 'tag' then -- anchor, h4 pseudo-heading - if root:has_error() then - return text - end - local in_heading = vim.list_contains({ 'h1', 'h2', 'h3' }, parent) - local h4 = not in_heading and not next_ and get_indent(node_text()) > 8 -- h4 pseudo-heading - local cssclass = h4 and 'help-tag-right' or 'help-tag' - local tagname = node_text(root:child(1), false) - if vim.tbl_count(stats.first_tags) < 2 then - -- Force the first 2 tags in the doc to be anchored at the main heading. - table.insert(stats.first_tags, tagname) - return '' - end - local el = 'span' - local encoded_tagname = url_encode(tagname) - local s = ('%s<%s id="%s" class="%s"><a href="#%s">%s</a></%s>'):format( - ws(), - el, - encoded_tagname, - cssclass, - encoded_tagname, - trimmed, - el - ) - if opt.old then - s = fix_tab_after_conceal(s, node_text(root:next_sibling())) - end - - if in_heading and prev ~= 'tag' then - -- Start the <span> container for tags in a heading. - -- This makes "justify-content:space-between" right-align the tags. - -- <h2>foo bar<span>tag1 tag2</span></h2> - return string.format('<span class="help-heading-tags">%s', s) - elseif in_heading and next_ == nil then - -- End the <span> container for tags in a heading. - return string.format('%s</span>', s) - end - return s .. (h4 and '<br>' or '') -- HACK: <br> avoids h4 pseudo-heading mushing with text. - elseif node_name == 'delimiter' or node_name == 'modeline' then - return '' - elseif node_name == 'ERROR' then - if ignore_parse_error(opt.fname, trimmed) then - return text - end - - -- Store the raw text to give context to the bug report. - local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' - table.insert(stats.parse_errors, sample_text) - return ('<a class="parse-error" target="_blank" title="Report bug... (parse error)" href="%s">%s</a>'):format( - get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), - trimmed - ) - else -- Unknown token. - local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' - return ('<a class="unknown-token" target="_blank" title="Report bug... (unhandled token "%s")" href="%s">%s</a>'):format( - node_name, - get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), - trimmed - ), - ('unknown-token:"%s"'):format(node_name) - end -end - ---- @param dir string e.g. '$VIMRUNTIME/doc' ---- @param include string[]|nil ---- @return string[] -local function get_helpfiles(dir, include) - local rv = {} - for f, type in vim.fs.dir(dir) do - if - vim.endswith(f, '.txt') - and type == 'file' - and (not include or vim.list_contains(include, f)) - then - local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') - table.insert(rv, fullpath) - end - end - return rv -end - ---- Populates the helptags map. -local function get_helptags(help_dir) - local m = {} - -- Load a random help file to convince taglist() to do its job. - vim.cmd(string.format('split %s/api.txt', help_dir)) - vim.cmd('lcd %:p:h') - for _, item in ipairs(vim.fn.taglist('.*')) do - if vim.endswith(item.filename, '.txt') then - m[item.name] = item.filename - end - end - vim.cmd('q!') - return m -end - ---- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system. -local function ensure_runtimepath() - if not vim.o.runtimepath:find('build/lib/nvim/') then - vim.cmd [[set runtimepath^=./build/lib/nvim/]] - end -end - ---- Opens `fname` (or `text`, if given) in a buffer and gets a treesitter parser for the buffer contents. ---- ---- @param fname string :help file to parse ---- @param text string? :help file contents ---- @param parser_path string? path to non-default vimdoc.so ---- @return vim.treesitter.LanguageTree, integer (lang_tree, bufnr) -local function parse_buf(fname, text, parser_path) - local buf ---@type integer - if text then - vim.cmd('split new') -- Text contents. - vim.api.nvim_put(vim.split(text, '\n'), '', false, false) - vim.cmd('setfiletype help') - -- vim.treesitter.language.add('vimdoc') - buf = vim.api.nvim_get_current_buf() - elseif type(fname) == 'string' then - vim.cmd('split ' .. vim.fn.fnameescape(fname)) -- Filename. - buf = vim.api.nvim_get_current_buf() - else - -- Left for debugging - ---@diagnostic disable-next-line: no-unknown - buf = fname - vim.cmd('sbuffer ' .. tostring(fname)) -- Buffer number. - end - if parser_path then - vim.treesitter.language.add('vimdoc', { path = parser_path }) - end - local lang_tree = assert(vim.treesitter.get_parser(buf, nil, { error = false })) - return lang_tree, buf -end - ---- Validates one :help file `fname`: ---- - checks that |tag| links point to valid helptags. ---- - recursively counts parse errors ("ERROR" nodes) ---- ---- @param fname string help file to validate ---- @param parser_path string? path to non-default vimdoc.so ---- @return { invalid_links: number, parse_errors: string[] } -local function validate_one(fname, parser_path) - local stats = { - parse_errors = {}, - } - local lang_tree, buf = parse_buf(fname, nil, parser_path) - for _, tree in ipairs(lang_tree:trees()) do - visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname }, stats) - end - lang_tree:destroy() - vim.cmd.close() - return stats -end - ---- Generates HTML from one :help file `fname` and writes the result to `to_fname`. ---- ---- @param fname string Source :help file. ---- @param text string|nil Source :help file contents, or nil to read `fname`. ---- @param to_fname string Destination .html file ---- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) ---- @param parser_path string? path to non-default vimdoc.so ---- ---- @return string html ---- @return table stats -local function gen_one(fname, text, to_fname, old, commit, parser_path) - local stats = { - noise_lines = {}, - parse_errors = {}, - first_tags = {}, -- Track the first few tags in doc. - } - local lang_tree, buf = parse_buf(fname, text, parser_path) - ---@type nvim.gen_help_html.heading[] - local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. - local title = to_titlecase(basename_noext(fname)) - - local html = ([[ - <!DOCTYPE html> - <html> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <meta name="description" content="Neovim user documentation"> - - <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ --> - <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@docsearch/css@3" /> - <link rel="preconnect" href="https://X185E15FPG-dsn.algolia.net" crossorigin /> - - <link href="/css/bootstrap.min.css" rel="stylesheet"> - <link href="/css/main.css" rel="stylesheet"> - <link href="help.css" rel="stylesheet"> - <link href="/highlight/styles/neovim.min.css" rel="stylesheet"> - - <script src="/highlight/highlight.min.js"></script> - <script>hljs.highlightAll();</script> - <title>%s - Neovim docs</title> - </head> - <body> - ]]):format(title) - - local logo_svg = [[ - <svg xmlns="http://www.w3.org/2000/svg" role="img" width="173" height="50" viewBox="0 0 742 214" aria-label="Neovim"> - <title>Neovim</title> - <defs> - <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="a"> - <stop stop-color="#16B0ED" stop-opacity=".8" offset="0%" /> - <stop stop-color="#0F59B2" stop-opacity=".837" offset="100%" /> - </linearGradient> - <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="b"> - <stop stop-color="#7DB643" offset="0%" /> - <stop stop-color="#367533" offset="100%" /> - </linearGradient> - <linearGradient x1="50%" y1="0%" x2="50%" y2="100%" id="c"> - <stop stop-color="#88C649" stop-opacity=".8" offset="0%" /> - <stop stop-color="#439240" stop-opacity=".84" offset="100%" /> - </linearGradient> - </defs> - <g fill="none" fill-rule="evenodd"> - <path - d="M.027 45.459L45.224-.173v212.171L.027 166.894V45.459z" - fill="url(#a)" - transform="translate(1 1)" - /> - <path - d="M129.337 45.89L175.152-.149l-.928 212.146-45.197-45.104.31-121.005z" - fill="url(#b)" - transform="matrix(-1 0 0 1 305 1)" - /> - <path - d="M45.194-.137L162.7 179.173l-32.882 32.881L12.25 33.141 45.194-.137z" - fill="url(#c)" - transform="translate(1 1)" - /> - <path - d="M46.234 84.032l-.063 7.063-36.28-53.563 3.36-3.422 32.983 49.922z" - fill-opacity=".13" - fill="#000" - /> - <g fill="#444"> - <path - d="M227 154V64.44h4.655c1.55 0 2.445.75 2.685 2.25l.806 13.502c4.058-5.16 8.786-9.316 14.188-12.466 5.4-3.15 11.413-4.726 18.037-4.726 4.893 0 9.205.781 12.935 2.34 3.729 1.561 6.817 3.811 9.264 6.751 2.448 2.942 4.297 6.48 5.55 10.621 1.253 4.14 1.88 8.821 1.88 14.042V154h-8.504V96.754c0-8.402-1.91-14.987-5.729-19.757-3.82-4.771-9.667-7.156-17.544-7.156-5.851 0-11.28 1.516-16.292 4.545-5.013 3.032-9.489 7.187-13.427 12.467V154H227zM350.624 63c5.066 0 9.755.868 14.069 2.605 4.312 1.738 8.052 4.268 11.219 7.592s5.638 7.412 7.419 12.264C385.11 90.313 386 95.883 386 102.17c0 1.318-.195 2.216-.588 2.696-.393.48-1.01.719-1.851.719h-64.966v1.70c0 6.708.784 12.609 2.353 17.7 1.567 5.09 3.8 9.357 6.695 12.802 2.895 3.445 6.393 6.034 10.495 7.771 4.1 1.738 8.686 2.606 13.752 2.606 4.524 0 8.446-.494 11.762-1.483 3.317-.988 6.108-2.097 8.37-3.324 2.261-1.227 4.056-2.336 5.383-3.324 1.326-.988 2.292-1.482 2.895-1.482.784 0 1.388.3 1.81.898l2.352 2.875c-1.448 1.797-3.362 3.475-5.745 5.031-2.383 1.558-5.038 2.891-7.962 3.998-2.926 1.109-6.062 1.991-9.41 2.65a52.21 52.21 0 01-10.088.989c-6.152 0-11.762-1.064-16.828-3.19-5.067-2.125-9.415-5.225-13.043-9.298-3.63-4.074-6.435-9.06-8.415-14.96C310.99 121.655 310 114.9 310 107.294c0-6.408.92-12.323 2.76-17.744 1.84-5.421 4.493-10.093 7.961-14.016 3.467-3.922 7.72-6.991 12.758-9.209C338.513 64.11 344.229 63 350.624 63zm.573 6c-4.696 0-8.904.702-12.623 2.105-3.721 1.404-6.936 3.421-9.65 6.053-2.713 2.631-4.908 5.79-6.586 9.474S319.55 94.439 319 99h60c0-4.679-.672-8.874-2.013-12.588-1.343-3.712-3.232-6.856-5.67-9.43-2.44-2.571-5.367-4.545-8.782-5.92-3.413-1.374-7.192-2.062-11.338-2.062zM435.546 63c6.526 0 12.368 1.093 17.524 3.28 5.154 2.186 9.5 5.286 13.04 9.298 3.538 4.013 6.238 8.85 8.099 14.51 1.861 5.66 2.791 11.994 2.791 19.002 0 7.008-.932 13.327-2.791 18.957-1.861 5.631-4.561 10.452-8.099 14.465-3.54 4.012-7.886 7.097-13.04 9.254-5.156 2.156-10.998 3.234-17.524 3.234-6.529 0-12.369-1.078-17.525-3.234-5.155-2.157-9.517-5.242-13.085-9.254-3.57-4.013-6.285-8.836-8.145-14.465-1.861-5.63-2.791-11.95-2.791-18.957 0-7.008.93-13.342 2.791-19.002 1.861-5.66 4.576-10.496 8.145-14.51 3.568-4.012 7.93-7.112 13.085-9.299C423.177 64.094 429.017 63 435.546 63zm-.501 86c5.341 0 10.006-.918 13.997-2.757 3.99-1.838 7.32-4.474 9.992-7.909 2.67-3.435 4.664-7.576 5.986-12.428 1.317-4.85 1.98-10.288 1.98-16.316 0-5.965-.66-11.389-1.98-16.27-1.322-4.88-3.316-9.053-5.986-12.519-2.67-3.463-6-6.13-9.992-7.999-3.991-1.867-8.657-2.802-13.997-2.802s-10.008.935-13.997 2.802c-3.991 1.87-7.322 4.536-9.992 8-2.671 3.465-4.68 7.637-6.03 12.518-1.35 4.881-2.026 10.305-2.026 16.27 0 6.026.675 11.465 2.025 16.316 1.35 4.852 3.36 8.993 6.031 12.428 2.67 3.435 6 6.07 9.992 7.91 3.99 1.838 8.656 2.756 13.997 2.756z" - fill="currentColor" - /> - <path - d="M530.57 152h-20.05L474 60h18.35c1.61 0 2.967.39 4.072 1.166 1.103.778 1.865 1.763 2.283 2.959l17.722 49.138a92.762 92.762 0 012.551 8.429c.686 2.751 1.298 5.5 1.835 8.25.537-2.75 1.148-5.499 1.835-8.25a77.713 77.713 0 012.64-8.429l18.171-49.138c.417-1.196 1.164-2.181 2.238-2.96 1.074-.776 2.356-1.165 3.849-1.165H567l-36.43 92zM572 61h23v92h-23zM610 153V60.443h13.624c2.887 0 4.78 1.354 5.682 4.06l1.443 6.856a52.7 52.7 0 015.097-4.962 32.732 32.732 0 015.683-3.879 30.731 30.731 0 016.496-2.57c2.314-.632 4.855-.948 7.624-.948 5.832 0 10.63 1.579 14.39 4.736 3.758 3.157 6.57 7.352 8.434 12.585 1.444-3.068 3.248-5.698 5.413-7.894 2.165-2.194 4.541-3.984 7.127-5.367a32.848 32.848 0 018.254-3.068 39.597 39.597 0 018.796-.992c5.111 0 9.653.783 13.622 2.345 3.97 1.565 7.307 3.849 10.014 6.857 2.706 3.007 4.766 6.675 6.18 11.005C739.29 83.537 740 88.5 740 94.092V153h-22.284V94.092c0-5.894-1.294-10.329-3.878-13.306-2.587-2.977-6.376-4.465-11.368-4.465-2.286 0-4.404.391-6.358 1.172a15.189 15.189 0 00-5.144 3.383c-1.473 1.474-2.631 3.324-3.474 5.548-.842 2.225-1.263 4.781-1.263 7.668V153h-22.37V94.092c0-6.194-1.249-10.704-3.744-13.532-2.497-2.825-6.18-4.24-11.051-4.24-3.19 0-6.18.798-8.976 2.391-2.799 1.593-5.399 3.775-7.804 6.54V153H610zM572 30h23v19h-23z" - fill="currentColor" - fill-opacity=".8" - /> - </g> - </g> - </svg> - ]] - - local main = '' - for _, tree in ipairs(lang_tree:trees()) do - main = main - .. ( - visit_node( - tree:root(), - 0, - tree, - headings, - { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1 }, - stats - ) - ) - end - - main = ([[ - <header class="container"> - <nav class="navbar navbar-expand-lg"> - <div class="container-fluid"> - <a href="/" class="navbar-brand" aria-label="logo"> - <!--TODO: use <img src="….svg"> here instead. Need one that has green lettering instead of gray. --> - %s - <!--<img src="https://neovim.io/logos/neovim-logo.svg" width="173" height="50" alt="Neovim" />--> - </a> - <div id="docsearch"></div> <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ --> - </div> - </nav> - </header> - - <div class="container golden-grid help-body"> - <div class="col-wide"> - <a name="%s" href="#%s"><h1 id="%s">%s</h1></a> - <p> - <i> - Nvim <code>:help</code> pages, <a href="https://github.com/neovim/neovim/blob/master/scripts/gen_help_html.lua">generated</a> - from <a href="https://github.com/neovim/neovim/blob/master/runtime/doc/%s">source</a> - using the <a href="https://github.com/neovim/tree-sitter-vimdoc">tree-sitter-vimdoc</a> parser. - </i> - </p> - <hr/> - %s - </div> - ]]):format( - logo_svg, - stats.first_tags[1] or '', - stats.first_tags[2] or '', - stats.first_tags[2] or '', - title, - vim.fs.basename(fname), - main - ) - - ---@type string - local toc = [[ - <div class="col-narrow toc"> - <div><a href="index.html">Main</a></div> - <div><a href="vimindex.html">Commands index</a></div> - <div><a href="quickref.html">Quick reference</a></div> - <hr/> - ]] - - local n = 0 -- Count of all headings + subheadings. - for _, h1 in ipairs(headings) do - n = n + 1 + #h1.subheadings - end - for _, h1 in ipairs(headings) do - ---@type string - toc = toc .. ('<div class="help-toc-h1"><a href="#%s">%s</a>\n'):format(h1.tag, h1.name) - if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. - for _, h2 in ipairs(h1.subheadings) do - toc = toc - .. ('<div class="help-toc-h2"><a href="#%s">%s</a></div>\n'):format(h2.tag, h2.name) - end - end - toc = toc .. '</div>' - end - toc = toc .. '</div>\n' - - local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil) - local bug_link = string.format('(<a href="%s" target="_blank">report docs bug...</a>)', bug_url) - - local footer = ([[ - <footer> - <div class="container flex"> - <div class="generator-stats"> - Generated at %s from <code><a href="https://github.com/neovim/neovim/commit/%s">%s</a></code> - </div> - <div class="generator-stats"> - parse_errors: %d %s | <span title="%s">noise_lines: %d</span> - </div> - <div> - - <!-- algolia docsearch https://docsearch.algolia.com/docs/docsearch-v3/ --> - <script src="https://cdn.jsdelivr.net/npm/@docsearch/js@3"></script> - <script type="module"> - docsearch({ - container: '#docsearch', - appId: 'X185E15FPG', - apiKey: 'b5e6b2f9c636b2b471303205e59832ed', - indexName: 'nvim', - }); - </script> - - </footer> - ]]):format( - os.date('%Y-%m-%d %H:%M'), - commit, - commit:sub(1, 7), - #stats.parse_errors, - bug_link, - html_esc(table.concat(stats.noise_lines, '\n')), - #stats.noise_lines - ) - - html = ('%s%s%s</div>\n%s</body>\n</html>\n'):format(html, main, toc, footer) - vim.cmd('q!') - lang_tree:destroy() - return html, stats -end - -local function gen_css(fname) - local css = [[ - :root { - --code-color: #004b4b; - --tag-color: #095943; - } - @media (prefers-color-scheme: dark) { - :root { - --code-color: #00c243; - --tag-color: #00b7b7; - } - } - @media (min-width: 40em) { - .toc { - position: fixed; - left: 67%; - } - .golden-grid { - display: grid; - grid-template-columns: 65% auto; - grid-gap: 1em; - } - } - @media (max-width: 40em) { - .golden-grid { - /* Disable grid for narrow viewport (mobile phone). */ - display: block; - } - } - .toc { - /* max-width: 12rem; */ - height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ - overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ - } - .toc > div { - text-overflow: ellipsis; - overflow: hidden; - white-space: nowrap; - } - html { - scroll-behavior: auto; - } - body { - font-size: 18px; - line-height: 1.5; - } - h1, h2, h3, h4, h5 { - font-family: sans-serif; - border-bottom: 1px solid var(--tag-color); /*rgba(0, 0, 0, .9);*/ - } - h3, h4, h5 { - border-bottom-style: dashed; - } - .help-column_heading { - color: var(--code-color); - } - .help-body { - padding-bottom: 2em; - } - .help-line { - /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ - } - .help-li { - white-space: normal; - display: list-item; - margin-left: 1.5rem; /* padding-left: 1rem; */ - } - .help-para { - padding-top: 10px; - padding-bottom: 10px; - } - - .old-help-para { - padding-top: 10px; - padding-bottom: 10px; - /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */ - tab-size: 8; - white-space: pre-wrap; - font-size: 16px; - font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; - word-wrap: break-word; - } - .old-help-para pre, .old-help-para pre:hover { - /* Text following <pre> is already visually separated by the linebreak. */ - margin-bottom: 0; - /* Long lines that exceed the textwidth should not be wrapped (no "pre-wrap"). - Since text may overflow horizontally, we make the contents to be scrollable - (only if necessary) to prevent overlapping with the navigation bar at the right. */ - white-space: pre; - overflow-x: auto; - } - - /* TODO: should this rule be deleted? help tags are rendered as <code> or <span>, not <a> */ - a.help-tag, a.help-tag:focus, a.help-tag:hover { - color: inherit; - text-decoration: none; - } - .help-tag { - color: var(--tag-color); - } - /* Tag pseudo-header common in :help docs. */ - .help-tag-right { - color: var(--tag-color); - margin-left: auto; - margin-right: 0; - float: right; - display: block; - } - .help-tag a, - .help-tag-right a { - color: inherit; - } - .help-tag a:not(:hover), - .help-tag-right a:not(:hover) { - text-decoration: none; - } - h1 .help-tag, h2 .help-tag, h3 .help-tag { - font-size: smaller; - } - .help-heading { - white-space: normal; - display: flex; - flex-flow: row wrap; - justify-content: space-between; - gap: 0 15px; - } - /* The (right-aligned) "tags" part of a section heading. */ - .help-heading-tags { - margin-right: 10px; - } - .help-toc-h1 { - } - .help-toc-h2 { - margin-left: 1em; - } - .parse-error { - background-color: red; - } - .unknown-token { - color: black; - background-color: yellow; - } - code { - color: var(--code-color); - font-size: 16px; - } - pre { - /* Tabs are used in codeblocks only for indentation, not alignment, so we can aggressively shrink them. */ - tab-size: 2; - white-space: pre-wrap; - line-height: 1.3; /* Important for ascii art. */ - overflow: visible; - /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ - font-size: 16px; - margin-top: 10px; - } - pre:last-child { - margin-bottom: 0; - } - pre:hover { - overflow: visible; - } - .generator-stats { - color: gray; - font-size: smaller; - } - ]] - tofile(fname, css) -end - --- Testing - -local function ok(cond, expected, actual, message) - assert( - (not expected and not actual) or (expected and actual), - 'if "expected" is given, "actual" is also required' - ) - if expected then - assert( - cond, - ('%sexpected %s, got: %s'):format( - message and (message .. '\n') or '', - vim.inspect(expected), - vim.inspect(actual) - ) - ) - return cond - else - return assert(cond) - end -end -local function eq(expected, actual, message) - return ok(vim.deep_equal(expected, actual), expected, actual, message) -end - -function M._test() - tagmap = get_helptags('$VIMRUNTIME/doc') - helpfiles = get_helpfiles(vim.fs.normalize('$VIMRUNTIME/doc')) - - ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) - ok( - vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), - tagmap['vim.diagnostic.set()'], - 'diagnostic.txt' - ) - ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt') - ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]])) - ok(is_noise([[ NVIM REFERENCE MANUAL by Thiago de Arruda ]])) - ok(not is_noise([[vim:tw=78]])) - - eq(0, get_indent('a')) - eq(1, get_indent(' a')) - eq(2, get_indent(' a\n b\n c\n')) - eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) - eq( - 'a\n \n b\n c\n d\n e\n', - trim_indent(' a\n \n b\n c\n d\n e\n') - ) - - local fixed_url, removed_chars = fix_url('https://example.com).') - eq('https://example.com', fixed_url) - eq(').', removed_chars) - fixed_url, removed_chars = fix_url('https://example.com.)') - eq('https://example.com.', fixed_url) - eq(')', removed_chars) - fixed_url, removed_chars = fix_url('https://example.com.') - eq('https://example.com', fixed_url) - eq('.', removed_chars) - fixed_url, removed_chars = fix_url('https://example.com)') - eq('https://example.com', fixed_url) - eq(')', removed_chars) - fixed_url, removed_chars = fix_url('https://example.com') - eq('https://example.com', fixed_url) - eq('', removed_chars) - - print('all tests passed.\n') -end - ---- @class nvim.gen_help_html.gen_result ---- @field helpfiles string[] list of generated HTML files, from the source docs {include} ---- @field err_count integer number of parse errors in :help docs ---- @field invalid_links table<string, any> - ---- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`. ---- ---- Example: ---- ---- gen('$VIMRUNTIME/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) ---- ---- @param help_dir string Source directory containing the :help files. Must run `make helptags` first. ---- @param to_dir string Target directory where the .html files will be written. ---- @param include string[]|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} ---- ---- @return nvim.gen_help_html.gen_result result -function M.gen(help_dir, to_dir, include, commit, parser_path) - vim.validate('help_dir', help_dir, function(d) - return vim.fn.isdirectory(vim.fs.normalize(d)) == 1 - end, 'valid directory') - vim.validate('to_dir', to_dir, 'string') - vim.validate('include', include, 'table', true) - vim.validate('commit', commit, 'string', true) - vim.validate('parser_path', parser_path, function(f) - return vim.fn.filereadable(vim.fs.normalize(f)) == 1 - end, true, 'valid vimdoc.{so,dll} filepath') - - local err_count = 0 - local redirects_count = 0 - ensure_runtimepath() - tagmap = get_helptags(vim.fs.normalize(help_dir)) - helpfiles = get_helpfiles(help_dir, include) - to_dir = vim.fs.normalize(to_dir) - parser_path = parser_path and vim.fs.normalize(parser_path) or nil - - print(('output dir: %s\n\n'):format(to_dir)) - vim.fn.mkdir(to_dir, 'p') - gen_css(('%s/help.css'):format(to_dir)) - - for _, f in ipairs(helpfiles) do - -- "foo.txt" - local helpfile = vim.fs.basename(f) - -- "to/dir/foo.html" - local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) - local html, stats = - gen_one(f, nil, to_fname, not new_layout[helpfile], commit or '?', parser_path) - tofile(to_fname, html) - print( - ('generated (%-2s errors): %-15s => %s'):format( - #stats.parse_errors, - helpfile, - vim.fs.basename(to_fname) - ) - ) - - -- Generate redirect pages for renamed help files. - local helpfile_tag = (helpfile:gsub('%.txt$', '')) - local redirect_from = redirects[helpfile_tag] - if redirect_from then - local redirect_text = ([[ -*%s* Nvim - -This document moved to: |%s| - -============================================================================== -This document moved to: |%s| - -This document moved to: |%s| - -============================================================================== - vim:tw=78:ts=8:ft=help:norl: - ]]):format( - redirect_from, - helpfile_tag, - helpfile_tag, - helpfile_tag, - helpfile_tag, - helpfile_tag - ) - local redirect_to = ('%s/%s'):format(to_dir, get_helppage(redirect_from)) - local redirect_html, _ = - gen_one(redirect_from, redirect_text, redirect_to, false, commit or '?', parser_path) - assert(redirect_html:find(helpfile_tag)) - tofile(redirect_to, redirect_html) - - print( - ('generated (redirect) : %-15s => %s'):format( - redirect_from .. '.txt', - vim.fs.basename(to_fname) - ) - ) - redirects_count = redirects_count + 1 - end - - err_count = err_count + #stats.parse_errors - end - - print(('\ngenerated %d html pages'):format(#helpfiles + redirects_count)) - print(('total errors: %d'):format(err_count)) - print(('invalid tags: %s'):format(vim.inspect(invalid_links))) - assert(#(include or {}) > 0 or redirects_count == vim.tbl_count(redirects)) -- sanity check - print(('redirects: %d'):format(redirects_count)) - print('\n') - - --- @type nvim.gen_help_html.gen_result - return { - helpfiles = helpfiles, - err_count = err_count, - invalid_links = invalid_links, - } -end - ---- @class nvim.gen_help_html.validate_result ---- @field helpfiles integer number of generated helpfiles ---- @field err_count integer number of parse errors ---- @field parse_errors table<string, string[]> ---- @field invalid_links table<string, any> invalid tags in :help docs ---- @field invalid_urls table<string, any> invalid URLs in :help docs ---- @field invalid_spelling table<string, table<string, string>> invalid spelling in :help docs - ---- Validates all :help files found in `help_dir`: ---- - checks that |tag| links point to valid helptags. ---- - recursively counts parse errors ("ERROR" nodes) ---- ---- This is 10x faster than gen(), for use in CI. ---- ---- @return nvim.gen_help_html.validate_result result -function M.validate(help_dir, include, parser_path) - vim.validate('help_dir', help_dir, function(d) - return vim.fn.isdirectory(vim.fs.normalize(d)) == 1 - end, 'valid directory') - vim.validate('include', include, 'table', true) - vim.validate('parser_path', parser_path, function(f) - return vim.fn.filereadable(vim.fs.normalize(f)) == 1 - end, true, 'valid vimdoc.{so,dll} filepath') - local err_count = 0 ---@type integer - local files_to_errors = {} ---@type table<string, string[]> - ensure_runtimepath() - tagmap = get_helptags(vim.fs.normalize(help_dir)) - helpfiles = get_helpfiles(help_dir, include) - parser_path = parser_path and vim.fs.normalize(parser_path) or nil - - for _, f in ipairs(helpfiles) do - local helpfile = vim.fs.basename(f) - local rv = validate_one(f, parser_path) - print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) - if #rv.parse_errors > 0 then - files_to_errors[helpfile] = rv.parse_errors - vim.print(('%s'):format(vim.iter(rv.parse_errors):fold('', function(s, v) - return s .. '\n ' .. v - end))) - end - err_count = err_count + #rv.parse_errors - end - - ---@type nvim.gen_help_html.validate_result - return { - helpfiles = #helpfiles, - err_count = err_count, - parse_errors = files_to_errors, - invalid_links = invalid_links, - invalid_urls = invalid_urls, - invalid_spelling = invalid_spelling, - } -end - ---- Validates vimdoc files on $VIMRUNTIME. and print human-readable error messages if fails. ---- ---- If this fails, try these steps (in order): ---- 1. Fix/cleanup the :help docs. ---- 2. Fix the parser: https://github.com/neovim/tree-sitter-vimdoc ---- 3. File a parser bug, and adjust the tolerance of this test in the meantime. ---- ---- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc' -function M.run_validate(help_dir) - help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc') - print('doc path = ' .. vim.uv.fs_realpath(help_dir)) - - local rv = M.validate(help_dir) - - -- Check that we actually found helpfiles. - ok(rv.helpfiles > 100, '>100 :help files', rv.helpfiles) - - eq({}, rv.parse_errors, 'no parse errors') - eq(0, rv.err_count, 'no parse errors') - eq({}, rv.invalid_links, 'invalid tags in :help docs') - eq({}, rv.invalid_urls, 'invalid URLs in :help docs') - eq( - {}, - rv.invalid_spelling, - 'invalid spelling in :help docs (see spell_dict in scripts/gen_help_html.lua)' - ) -end - ---- Test-generates HTML from docs. ---- ---- 1. Test that gen_help_html.lua actually works. ---- 2. Test that parse errors did not increase wildly. Because we explicitly test only a few ---- :help files, we can be precise about the tolerances here. ---- @param help_dir? string e.g. '$VIMRUNTIME/doc' or './runtime/doc' -function M.test_gen(help_dir) - local tmpdir = vim.fs.dirname(vim.fn.tempname()) - help_dir = vim.fs.normalize(help_dir or '$VIMRUNTIME/doc') - print('doc path = ' .. vim.uv.fs_realpath(help_dir)) - - -- Because gen() is slow (~30s), this test is limited to a few files. - local input = { 'help.txt', 'index.txt', 'nvim.txt' } - local rv = M.gen(help_dir, tmpdir, input) - eq(#input, #rv.helpfiles) - eq(0, rv.err_count, 'parse errors in :help docs') - eq({}, rv.invalid_links, 'invalid tags in :help docs') -end - -return M |