From 09b64d75bd92a95d89c4f39f9df7918760abe98d Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 27 Mar 2022 19:47:34 -0700 Subject: feat(docs): gen_help_html.lua MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: The :help docs HTML generated is driven by an old awk script `runtime/doc/makehtml.awk` that is hard to maintain (ad hoc parser and no one has touched it in decades) and has bugs like: - https://github.com/neovim/neovim.github.io/issues/96 - https://github.com/neovim/neovim.github.io/issues/97 Solution: Use Lua + treesitter (https://github.com/vigoux/tree-sitter-vimdoc) to generate :help docs HTML. Also validates tag links. fix https://github.com/neovim/neovim.github.io/issues/96 fix https://github.com/neovim/neovim.github.io/issues/97 TODO: - delete doc_html build task - delete runtime/doc/Makefile - delete makehtml.awk - delete maketags.awk OUTPUT: $ nvim -V1 -es --clean +"lua require('scripts.gen_help_html')" output dir: /…/neovim.github.io/_site/doc/ generated (207 errors): api.txt => api.html generated (122 errors): arabic.txt => arabic.html generated (285 errors): autocmd.txt => autocmd.html generated (641 errors): builtin.txt => builtin.html generated (623 errors): change.txt => change.html generated (65 errors): channel.txt => channel.html generated (353 errors): cmdline.txt => cmdline.html generated (3 errors): debug.txt => debug.html generated (28 errors): deprecated.txt => deprecated.html generated (193 errors): dev_style.txt => dev_style.html generated (460 errors): develop.txt => develop.html generated (19 errors): diagnostic.txt => diagnostic.html generated (57 errors): diff.txt => diff.html generated (818 errors): digraph.txt => digraph.html generated (330 errors): editing.txt => editing.html generated (368 errors): eval.txt => eval.html generated (184 errors): fold.txt => fold.html generated (61 errors): ft_ada.txt => ft_ada.html generated (0 errors): ft_ps1.txt => ft_ps1.html generated (20 errors): ft_raku.txt => ft_raku.html generated (5 errors): ft_rust.txt => ft_rust.html generated (41 errors): ft_sql.txt => ft_sql.html generated (110 errors): gui.txt => gui.html generated (79 errors): hebrew.txt => hebrew.html generated (17 errors): help.txt => index.html generated (104 errors): helphelp.txt => helphelp.html generated (0 errors): if_cscop.txt => if_cscop.html generated (23 errors): if_perl.txt => if_perl.html generated (16 errors): if_pyth.txt => if_pyth.html generated (9 errors): if_ruby.txt => if_ruby.html generated (216 errors): indent.txt => indent.html generated (634 errors): index.txt => vimindex.html generated (320 errors): insert.txt => insert.html generated (265 errors): intro.txt => intro.html generated (9 errors): job_control.txt => job_control.html generated (0 errors): lsp-extension.txt => lsp-extension.html generated (214 errors): lsp.txt => lsp.html generated (311 errors): lua.txt => lua.html generated (592 errors): luaref.txt => luaref.html generated (798 errors): luvref.txt => luvref.html generated (663 errors): map.txt => map.html generated (228 errors): mbyte.txt => mbyte.html generated (228 errors): message.txt => message.html generated (0 errors): mlang.txt => mlang.html generated (761 errors): motion.txt => motion.html generated (4 errors): nvim.txt => nvim.html generated (226 errors): nvim_terminal_emulator.txt => nvim_terminal_emulator.html generated (988 errors): options.txt => options.html generated (567 errors): pattern.txt => pattern.html generated (15 errors): pi_gzip.txt => pi_gzip.html generated (10 errors): pi_health.txt => pi_health.html generated (27 errors): pi_msgpack.txt => pi_msgpack.html generated (2177 errors): pi_netrw.txt => pi_netrw.html generated (41 errors): pi_paren.txt => pi_paren.html generated (9 errors): pi_spec.txt => pi_spec.html generated (218 errors): pi_tar.txt => pi_tar.html generated (0 errors): pi_tutor.txt => pi_tutor.html generated (235 errors): pi_zip.txt => pi_zip.html generated (265 errors): print.txt => print.html generated (31 errors): provider.txt => provider.html generated (335 errors): quickfix.txt => quickfix.html generated (572 errors): quickref.txt => quickref.html generated (109 errors): recover.txt => recover.html generated (14 errors): remote.txt => remote.html generated (14 errors): remote_plugin.txt => remote_plugin.html generated (351 errors): repeat.txt => repeat.html generated (23 errors): rileft.txt => rileft.html generated (12 errors): russian.txt => russian.html generated (6 errors): scroll.txt => scroll.html generated (106 errors): sign.txt => sign.html generated (347 errors): spell.txt => spell.html generated (784 errors): starting.txt => starting.html generated (1499 errors): syntax.txt => syntax.html generated (23 errors): tabpage.txt => tabpage.html generated (257 errors): tagsrch.txt => tagsrch.html generated (31 errors): term.txt => term.html generated (0 errors): testing.txt => testing.html generated (96 errors): tips.txt => tips.html generated (57 errors): treesitter.txt => treesitter.html generated (71 errors): uganda.txt => uganda.html generated (74 errors): ui.txt => ui.html generated (87 errors): undo.txt => undo.html generated (17 errors): userfunc.txt => userfunc.html generated (1 errors): usr_01.txt => usr_01.html generated (89 errors): usr_02.txt => usr_02.html generated (293 errors): usr_03.txt => usr_03.html generated (46 errors): usr_04.txt => usr_04.html generated (96 errors): usr_05.txt => usr_05.html generated (54 errors): usr_06.txt => usr_06.html generated (20 errors): usr_07.txt => usr_07.html generated (241 errors): usr_08.txt => usr_08.html generated (130 errors): usr_09.txt => usr_09.html generated (50 errors): usr_10.txt => usr_10.html generated (33 errors): usr_11.txt => usr_11.html generated (32 errors): usr_12.txt => usr_12.html generated (22 errors): usr_20.txt => usr_20.html generated (75 errors): usr_21.txt => usr_21.html generated (8 errors): usr_22.txt => usr_22.html generated (3 errors): usr_23.txt => usr_23.html generated (163 errors): usr_25.txt => usr_25.html generated (13 errors): usr_26.txt => usr_26.html generated (84 errors): usr_27.txt => usr_27.html generated (173 errors): usr_28.txt => usr_28.html generated (285 errors): usr_29.txt => usr_29.html generated (280 errors): usr_30.txt => usr_30.html generated (11 errors): usr_31.txt => usr_31.html generated (13 errors): usr_32.txt => usr_32.html generated (156 errors): usr_40.txt => usr_40.html generated (134 errors): usr_41.txt => usr_41.html generated (35 errors): usr_42.txt => usr_42.html generated (19 errors): usr_43.txt => usr_43.html generated (60 errors): usr_44.txt => usr_44.html generated (13 errors): usr_45.txt => usr_45.html generated (1 errors): usr_toc.txt => usr_toc.html generated (69 errors): various.txt => various.html generated (68 errors): vi_diff.txt => vi_diff.html generated (437 errors): vim_diff.txt => vim_diff.html generated (296 errors): visual.txt => visual.html generated (181 errors): windows.txt => windows.html generated 119 html pages total errors: 23862 invalid tags: 537 --- scripts/gen_help_html.lua | 830 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 830 insertions(+) create mode 100644 scripts/gen_help_html.lua (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua new file mode 100644 index 0000000000..d79ece53f3 --- /dev/null +++ b/scripts/gen_help_html.lua @@ -0,0 +1,830 @@ +-- Converts Vim :help files to HTML. Validates |tag| links and document syntax (parser errors). +-- +-- USAGE (GENERATE HTML): +-- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" +-- - Read the docstring at gen(). +-- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 +-- 4. Visit http://localhost:4000/…/help.txt.html +-- +-- USAGE (VALIDATE): +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate()" +-- - validate() is 10x faster than gen(), so it is used in CI. +-- +-- SELF-TEST MODE: +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +-- +-- NOTES: +-- * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too +-- slow (~1 min) to run in per-commit CI. +-- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. +-- * visit_validate() is the core function used by validate(). +-- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. +-- +-- parser bugs: +-- * Should NOT be code_block: +-- tab:xy The 'x' is always used, then 'y' as many times as will +-- fit. Thus "tab:>-" displays: +-- > +-- >- +-- >-- +-- etc. +-- +-- tab:xyz The 'z' is always used, then 'x' is prepended, and +-- then 'y' is used as many times as will fit. Thus +-- "tab:<->" displays: +-- > +-- <> +-- <-> +-- <--> +-- etc. +-- * Should NOT be a "headline". Perhaps a "table" (or just "line"). +-- expr5 and expr6 *expr5* *expr6* +-- --------------- +-- expr6 + expr6 Number addition, |List| or |Blob| concatenation *expr-+* +-- expr6 - expr6 Number subtraction *expr--* +-- expr6 . expr6 String concatenation *expr-.* +-- expr6 .. expr6 String concatenation *expr-..* + +local tagmap = nil +local helpfiles = nil +local invalid_tags = {} + +local commit = '?' +local api = vim.api +local M = {} + +-- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). +-- All other files are "legacy" files which require fixed-width layout. +local new_layout = { + ['api.txt'] = true, + ['channel.txt'] = true, + ['develop.txt'] = true, + ['nvim.txt'] = true, + ['pi_health.txt'] = true, + ['provider.txt'] = true, + ['ui.txt'] = true, +} + +-- TODO: treesitter gets stuck on these files... +local exclude = { + ['filetype.txt'] = true, + ['usr_24.txt'] = true, +} + +local function tofile(fname, text) + local f = io.open(fname, 'w') + if not f then + error(('failed to write: %s'):format(f)) + else + f:write(text) + f:close() + end +end + +local function html_esc(s) + if s:find('', '>') +end + +local function url_encode(s) + -- Credit: tpope / vim-unimpaired + -- NOTE: these chars intentionally *not* escaped: ' ( ) + return vim.fn.substitute(vim.fn.iconv(s, 'latin1', 'utf-8'), + [=[[^A-Za-z0-9()'_.~-]]=], + [=[\="%".printf("%02X",char2nr(submatch(0)))]=], + 'g') +end + +-- Removes the ">" and "<" chars that delineate a codeblock in Vim :help files. +local function trim_gt_lt(s) + return s:gsub('^%s*>%s*\n', ''):gsub('\n<', '') +end + +local function expandtabs(s) + return s:gsub('\t', (' '):rep(8)) +end + +local function to_titlecase(s) + local text = '' + for w in vim.gsplit(s, '[ \t]+') do + text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2)) + end + return text +end + +local function to_heading_tag(text) + -- Prepend "_" to avoid conflicts with actual :help tags. + return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown' +end + +local function basename_noext(f) + return vim.fs.basename(f:gsub('%.txt', '')) +end + +local function is_blank(s) + return not not s:find('^%s*$') +end + +local function trim(s) + return vim.trim(s) +end + +local function trim_bullet(s) + return s:gsub('^%s*[-*•]%s', '') +end + +local function startswith_bullet(s) + return s:find('^%s*[-*•]%s') +end + +-- Checks if a given line is a "noise" line that doesn't look good in HTML form. +local function is_noise(line) + return ( + line:find('Type .*gO.* to see the table of contents') + -- Title line of traditional :help pages. + -- Example: "NVIM REFERENCE MANUAL by ..." + or line:find('^%s*N?VIM REFERENCE MANUAL') + -- First line of traditional :help pages. + -- Example: "*api.txt* Nvim" + or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') + -- modeline + -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" + or line:find('^%s*vi[m]%:.*ft=help') + or line:find('^%s*vi[m]%:.*filetype=help') + ) +end + +-- Creates a github issue URL at vigoux/tree-sitter-vimdoc with prefilled content. +local function get_bug_url_vimdoc(fname, to_fname, sample_text) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ('https://github.com/vigoux/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + ..vim.fs.basename(fname) + ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' + ..this_url + ..'%0D%0DContext%3A%0D%0D%60%60%60%0D' + ..url_encode(sample_text) + ..'%0D%60%60%60') + return bug_url +end + +-- Creates a github issue URL at neovim/neovim with prefilled content. +local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ('https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' + ..vim.fs.basename(fname) + ..'+&body=%60gen_help_html.lua%60+problem+at%3A+' + ..this_url + ..'%0D' + ..(token_name and '+unhandled+token%3A+%60'..token_name..'%60' or '') + ..'%0DContext%3A%0D%0D%60%60%60%0D' + ..url_encode(sample_text) + ..'%0D%60%60%60') + return bug_url +end + +-- Gets a "foo.html" name from a "foo.txt" helpfile name. +local function get_helppage(f) + if not f then + return nil + end + -- Special case: help.txt is the "main landing page" of :help files, not index.txt. + if f == 'index.txt' then + return 'vimindex.html' + elseif f == 'help.txt' then + return 'index.html' + end + + return f:gsub('%.txt$', '.html') +end + +-- Counts leading spaces (tab=8) to decide the indent size of multiline text. +-- +-- Blank lines (empty or whitespace-only) are ignored. +local function get_indent(s) + local min_indent = nil + for line in vim.gsplit(s, '\n') do + if line and not is_blank(line) then + local ws = expandtabs(line:match('^%s+') or '') + min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent + end + end + return min_indent or 0 +end + +-- Removes the common indent level, after expanding tabs to 8 spaces. +local function trim_indent(s) + local indent_size = get_indent(s) + local trimmed = '' + for line in vim.gsplit(s, '\n') do + line = expandtabs(line) + trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1)) + end + return trimmed:sub(1, -2) +end + +-- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. +local function getbuflinestr(node, bufnr, offset) + local line1, _, line2, _ = node:range() + line1 = line1 - offset + line2 = line2 + offset + local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1) + return table.concat(lines, '\n') +end + +-- Gets the whitespace just before `node` from the raw buffer text. +-- Needed for preformatted `old` lines. +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + +local function get_tagname(node, bufnr, link) + local node_name = (node.named and node:named()) and node:type() or nil + local node_text = vim.treesitter.get_node_text(node, bufnr) + local tag = ((node_name == 'option' and node_text) + or (link and node_text:gsub('^|', ''):gsub('|$', '') or node_text:gsub('^%*', ''):gsub('%*$', ''))) + local helpfile = tag and vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" + return helppage, tag +end + +-- Traverses the tree at `root` and checks that |tag| links point to valid helptags. +local function visit_validate(root, level, lang_tree, opt, stats) + level = level or 0 + local node_name = (root.named and root:named()) and root:type() or nil + local toplevel = level < 1 + + if root:child_count() > 0 then + for node, _ in root:iter_children() do + if node:named() then + visit_validate(node, level + 1, lang_tree, opt, stats) + end + end + end + + if node_name == 'ERROR' then + -- Store the raw text to give context to the bug report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + elseif node_name == 'hotlink' or node_name == 'option' then + local _, tagname = get_tagname(root, opt.buf, true) + if not root:has_error() and not tagmap[tagname] then + invalid_tags[tagname] = vim.fs.basename(opt.fname) + end + end +end + +-- Generates HTML from node `root` recursively. +local function visit_node(root, level, lang_tree, headings, opt, stats) + level = level or 0 + + local node_name = (root.named and root:named()) and root:type() or nil + -- Previous sibling kind (string). + local prev = root:prev_sibling() and (root:prev_sibling().named and root:prev_sibling():named()) and root:prev_sibling():type() or nil + -- Next sibling kind (string). + local next_ = root:next_sibling() and (root:next_sibling().named and root:next_sibling():named()) and root:next_sibling():type() or nil + -- Parent kind (string). + local parent = root:parent() and root:parent():type() or nil + local text = '' + local toplevel = level < 1 + local function node_text() + return vim.treesitter.get_node_text(root, opt.buf) + end + + if root:child_count() == 0 then + text = node_text() + else + -- Process children and join them with whitespace. + for node, _ in root:iter_children() do + if node:named() then + local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) + local ws = r == '' and '' or ((opt.old and (node:type() == 'word' or not node:named())) and getws(node, opt.buf) or ' ') + text = string.format('%s%s%s', text, ws, r) + end + end + end + local trimmed = trim(text) + + if node_name == 'help_file' then -- root node + return text + elseif node_name == 'word' or node_name == 'uppercase_name' then + if parent == 'headline' then + -- Start a new heading item, or update the current one. + local n = (prev == nil or #headings == 0) and #headings + 1 or #headings + headings[n] = string.format('%s%s', headings[n] and headings[n]..' ' or '', text) + end + + return html_esc(text) + elseif node_name == 'headline' then + return ('

%s

\n'):format(to_heading_tag(headings[#headings]), text) + elseif node_name == 'column_heading' or node_name == 'column_name' then + return ('

%s

\n'):format(trimmed) + elseif node_name == 'line' then + -- TODO: remove these "sibling inspection" hacks once the parser provides structured info + -- about paragraphs and listitems: https://github.com/vigoux/tree-sitter-vimdoc/issues/12 + local next_text = root:next_sibling() and vim.treesitter.get_node_text(root:next_sibling(), opt.buf) or '' + local li = startswith_bullet(text) -- Listitem? + local next_li = startswith_bullet(next_text) -- Next is listitem? + -- Close the paragraph/listitem if the next sibling is not a line. + local close = (next_ ~= 'line' or next_li or is_blank(next_text)) and '\n' or '' + + -- HACK: discard common "noise" lines. + if is_noise(text) then + table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + return (opt.old or prev ~= 'line') and '' or close + end + + if opt.old then + -- XXX: Treat old docs as preformatted. Until those docs are "fixed" or we get better info + -- from tree-sitter-vimdoc, this avoids broken layout for legacy docs. + return ('
%s
\n'):format(text) + end + + if li then + return string.format('
%s%s', trim_bullet(expandtabs(text)), close) + end + if prev ~= 'line' then -- Start a new paragraph. + return string.format('
%s%s', expandtabs(text), close) + end + + -- Continue in the current paragraph/listitem. + return string.format('%s%s', expandtabs(text), close) + elseif node_name == 'hotlink' or node_name == 'option' then + local helppage, tagname = get_tagname(root, opt.buf, true) + if not root:has_error() and not tagmap[tagname] then + invalid_tags[tagname] = vim.fs.basename(opt.fname) + end + return ('%s'):format(helppage, url_encode(tagname), html_esc(tagname)) + elseif node_name == 'backtick' then + return ('%s'):format(html_esc(text)) + elseif node_name == 'argument' then + return ('{%s}'):format(html_esc(trimmed)) + elseif node_name == 'code_block' then + return ('
\n%s
\n'):format(html_esc(trim_indent(trim_gt_lt(text)))) + elseif node_name == 'tag' then -- anchor + local _, tagname = get_tagname(root, opt.buf, false) + local s = ('%s'):format(url_encode(tagname), trimmed) + if parent == 'headline' and prev ~= 'tag' then + -- Start the container for tags in a heading. + -- This makes "justify-content:space-between" right-align the tags. + --

foo bartag1 tag2

+ return string.format('%s', s) + elseif parent == 'headline' and next_ == nil then + -- End the container for tags in a heading. + return string.format('%s', s) + end + return s + elseif node_name == 'ERROR' then + -- Store the raw text to give context to the bug report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + if prev == 'ERROR' then + -- Avoid trashing the text with cascading errors. + return trimmed, ('parse-error:"%s"'):format(node_text()) + end + return ('%s'):format( + get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) + else -- Unknown token. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + return ('%s'):format( + node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) + end +end + +local function get_helpfiles(include) + local dir = './build/runtime/doc' + local rv = {} + for f, type in vim.fs.dir(dir) do + if (vim.endswith(f, '.txt') + and type == 'file' + and (not include or vim.tbl_contains(include, f)) + and (not exclude[f])) then + local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') + table.insert(rv, fullpath) + end + end + return rv +end + +-- Populates the helptags map. +local function get_helptags(help_dir) + local m = {} + -- Load a random help file to convince taglist() to do its job. + vim.cmd(string.format('split %s/api.txt', help_dir)) + vim.cmd('lcd %:p:h') + for _, item in ipairs(vim.fn.taglist('.*')) do + if vim.endswith(item.filename, '.txt') then + m[item.name] = item.filename + end + end + vim.cmd('q!') + return m +end + +-- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents. +-- +-- @returns lang_tree, bufnr +local function parse_buf(fname) + local buf + if type(fname) == 'string' then + vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. + buf = api.nvim_get_current_buf() + else + buf = fname + vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. + end + -- vim.treesitter.require_language('help', './build/lib/nvim/parser/help.so') + local lang_tree = vim.treesitter.get_parser(buf, 'help') + return lang_tree, buf +end + +-- Validates one :help file `fname`: +-- - checks that |tag| links point to valid helptags. +-- - recursively counts parse errors ("ERROR" nodes) +-- +-- @returns { invalid_tags: number, parse_errors: number } +local function validate_one(fname) + local stats = { + invalid_tags = {}, + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname) + for _, tree in ipairs(lang_tree:trees()) do + visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname, }, stats) + end + lang_tree:destroy() + vim.cmd.close() + return { + invalid_tags = invalid_tags, + parse_errors = stats.parse_errors, + } +end + +-- Generates HTML from one :help file `fname` and writes the result to `to_fname`. +-- +-- @param fname Source :help file +-- @param to_fname Destination .html file +-- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) +-- +-- @returns html, stats +local function gen_one(fname, to_fname, old) + local stats = { + noise_lines = {}, + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname) + local headings = {} -- Headings (for ToC). + local title = to_titlecase(basename_noext(fname)) + + local html = ([[ + + + + + + + + + + + + %s - Neovim docs + + + ]]):format(title) + + local logo_svg = [[ + + Neovim + + + + + + + + + + + + + + + + + + + + + + + + + + ]] + + local main = ([[ +
+ +
+ +
+
+

%s

+

+ + Nvim help pages, updated automatically + from source. + Parsing by tree-sitter-vimdoc. + +

+ ]]):format(logo_svg, title, vim.fs.basename(fname)) + for _, tree in ipairs(lang_tree:trees()) do + main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + end + main = main .. '
\n' + + local toc = [[ +
+ + + +
+ ]] + for _, heading in ipairs(headings) do + toc = toc .. ('\n'):format(to_heading_tag(heading), heading) + end + toc = toc .. '
\n' + + local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil) + local bug_link = string.format('(report docs bug...)', bug_url) + + local footer = ([[ +
+
+
+ Generated on %s from {%s} +
+
+ parse_errors: %d %s | noise_lines: %d +
+
+
+ ]]):format( + os.date('%Y-%m-%d %H:%M:%S'), commit, #stats.parse_errors, bug_link, + html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines) + + html = ('%s%s%s
\n%s\n\n'):format( + html, main, toc, footer) + vim.cmd('q!') + lang_tree:destroy() + return html, stats +end + +local function gen_css(fname) + local css = [[ + @media (min-width: 40em) { + .toc { + position: fixed; + left: 67%; + } + } + .toc { + /* max-width: 12rem; */ + } + .toc > div { + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + } + html { + scroll-behavior: auto; + } + h1, h2, h3, h4 { + font-family: sans-serif; + } + .help-body { + padding-bottom: 2em; + } + .help-line { + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + } + .help-item { + display: list-item; + margin-left: 1.5rem; /* padding-left: 1rem; */ + } + .help-para { + padding-top: 10px; + padding-bottom: 10px; + } + .old-help-line { + /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */ + tab-size: 8; + white-space: pre; + font-size: .875em; + font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; + } + a.help-tag, a.help-tag:focus, a.help-tag:hover { + color: inherit; + text-decoration: none; + } + .help-tag { + color: gray; + } + h1 .help-tag, h2 .help-tag { + font-size: smaller; + } + .help-heading { + overflow: hidden; + white-space: nowrap; + display: flex; + justify-content: space-between; + } + /* The (right-aligned) "tags" part of a section heading. */ + .help-heading-tags { + margin-left: 10px; + } + .parse-error { + background-color: red; + } + .unknown-token { + color: black; + background-color: yellow; + } + pre { + /* Tabs are used in code_blocks only for indentation, not alignment, so we can aggressively shrink them. */ + tab-size: 2; + white-space: pre; + overflow: visible; + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + /* font-size: 14px; */ + /* border: 0px; */ + /* margin: 0px; */ + } + pre:hover, + .help-heading:hover { + overflow: visible; + } + .generator-stats { + color: gray; + font-size: smaller; + } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + ]] + tofile(fname, css) +end + +function M._test() + tagmap = get_helptags('./build/runtime/doc') + helpfiles = get_helpfiles() + + local function ok(cond, expected, actual) + assert((not expected and not actual) or (expected and actual), 'if "expected" is given, "actual" is also required') + if expected then + return assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual))) + else + return assert(cond) + end + end + local function eq(expected, actual) + return ok(expected == actual, expected, actual) + end + + eq(119, #helpfiles) + ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) + ok(vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), tagmap['vim.diagnostic.set()'], 'diagnostic.txt') + ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt') + ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]])) + ok(is_noise([[ VIM REFERENCE MANUAL by Abe Lincoln ]])) + ok(not is_noise([[vim:tw=78]])) + + eq(0, get_indent('a')) + eq(1, get_indent(' a')) + eq(2, get_indent(' a\n b\n c\n')) + eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) + eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + + print('all tests passed') +end + +--- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`. +--- +--- Example: +--- +--- gen('./build/runtime/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) +--- +--- @param help_dir string Source directory containing the :help files. Must run `make helptags` first. +--- @param to_dir string Target directory where the .html files will be written. +--- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} +--- +--- @returns info dict +function M.gen(help_dir, to_dir, include) + vim.validate{ + help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, + to_dir={to_dir, 's'}, + include={include, 't', true}, + } + + local err_count = 0 + tagmap = get_helptags(help_dir) + helpfiles = get_helpfiles(include) + + print(('output dir: %s'):format(to_dir)) + vim.fn.mkdir(to_dir, 'p') + gen_css(('%s/help.css'):format(to_dir)) + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) + local html, stats = gen_one(f, to_fname, not new_layout[helpfile]) + tofile(to_fname, html) + print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) + err_count = err_count + #stats.parse_errors + end + print(('generated %d html pages'):format(#helpfiles)) + print(('total errors: %d'):format(err_count)) + print(('invalid tags:\n%s'):format(vim.inspect(invalid_tags))) + + return { + helpfiles = helpfiles, + err_count = err_count, + invalid_tags = invalid_tags, + } +end + +-- Validates all :help files found in `help_dir`: +-- - checks that |tag| links point to valid helptags. +-- - recursively counts parse errors ("ERROR" nodes) +-- +-- This is 10x faster than gen(), for use in CI. +-- +-- @returns results dict +function M.validate(help_dir, include) + vim.validate{ + help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, + include={include, 't', true}, + } + local err_count = 0 + tagmap = get_helptags(help_dir) + helpfiles = get_helpfiles(include) + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local rv = validate_one(f) + print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) + err_count = err_count + #rv.parse_errors + end + + return { + helpfiles = helpfiles, + err_count = err_count, + invalid_tags = invalid_tags, + } +end + +return M -- cgit From 7b4c49888a97c21f346b8337330fbc8e196b9cf8 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Fri, 23 Sep 2022 14:26:59 +0200 Subject: feat(gen_help_html.lua): put commit-id in footer --- scripts/gen_help_html.lua | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index d79ece53f3..6e297814b8 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -16,7 +16,7 @@ -- -- NOTES: -- * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too --- slow (~1 min) to run in per-commit CI. +-- slow (~1 min) to run in per-commit CI. -- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. -- * visit_validate() is the core function used by validate(). -- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. @@ -50,7 +50,6 @@ local tagmap = nil local helpfiles = nil local invalid_tags = {} -local commit = '?' local api = vim.api local M = {} @@ -477,7 +476,7 @@ end -- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) -- -- @returns html, stats -local function gen_one(fname, to_fname, old) +local function gen_one(fname, to_fname, old, commit) local stats = { noise_lines = {}, parse_errors = {}, @@ -604,7 +603,7 @@ local function gen_one(fname, to_fname, old)
- Generated on %s from {%s} + Generated at %s from %s
parse_errors: %d %s | noise_lines: %d @@ -612,7 +611,7 @@ local function gen_one(fname, to_fname, old)
]]):format( - os.date('%Y-%m-%d %H:%M:%S'), commit, #stats.parse_errors, bug_link, + os.date('%Y-%m-%d %H:%M'), commit, commit:sub(1, 7), #stats.parse_errors, bug_link, html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines) html = ('%s%s%s
\n%s\n\n'):format( @@ -763,11 +762,12 @@ end --- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} --- --- @returns info dict -function M.gen(help_dir, to_dir, include) +function M.gen(help_dir, to_dir, include, commit) vim.validate{ help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, to_dir={to_dir, 's'}, include={include, 't', true}, + commit={commit, 's', true}, } local err_count = 0 @@ -781,7 +781,7 @@ function M.gen(help_dir, to_dir, include) for _, f in ipairs(helpfiles) do local helpfile = vim.fs.basename(f) local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) - local html, stats = gen_one(f, to_fname, not new_layout[helpfile]) + local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?') tofile(to_fname, html) print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) err_count = err_count + #stats.parse_errors -- cgit From 16336c486ecb5a60e85a870904316308c7d7fc3f Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 25 Sep 2022 02:20:47 +0200 Subject: feat(gen_help_html.lua): adapt to new parser - adapt to parser changes from https://github.com/vigoux/tree-sitter-vimdoc/pull/16 - numerous other generator improvements --- scripts/gen_help_html.lua | 467 +++++++++++++++++++++++++++++----------------- 1 file changed, 295 insertions(+), 172 deletions(-) (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 6e297814b8..fdbf5f605a 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -1,5 +1,7 @@ -- Converts Vim :help files to HTML. Validates |tag| links and document syntax (parser errors). -- +-- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). +-- -- USAGE (GENERATE HTML): -- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). -- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" @@ -20,37 +22,19 @@ -- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. -- * visit_validate() is the core function used by validate(). -- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. --- --- parser bugs: --- * Should NOT be code_block: --- tab:xy The 'x' is always used, then 'y' as many times as will --- fit. Thus "tab:>-" displays: --- > --- >- --- >-- --- etc. --- --- tab:xyz The 'z' is always used, then 'x' is prepended, and --- then 'y' is used as many times as will fit. Thus --- "tab:<->" displays: --- > --- <> --- <-> --- <--> --- etc. --- * Should NOT be a "headline". Perhaps a "table" (or just "line"). --- expr5 and expr6 *expr5* *expr6* --- --------------- --- expr6 + expr6 Number addition, |List| or |Blob| concatenation *expr-+* --- expr6 - expr6 Number subtraction *expr--* --- expr6 . expr6 String concatenation *expr-.* --- expr6 .. expr6 String concatenation *expr-..* local tagmap = nil local helpfiles = nil -local invalid_tags = {} +local invalid_links = {} +local invalid_urls = {} +local invalid_spelling = {} +local spell_dict = { + Neovim = 'Nvim', + NeoVim = 'Nvim', + neovim = 'Nvim', + lua = 'Lua', +} -local api = vim.api local M = {} -- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). @@ -59,16 +43,31 @@ local new_layout = { ['api.txt'] = true, ['channel.txt'] = true, ['develop.txt'] = true, + ['luaref.txt'] = true, ['nvim.txt'] = true, ['pi_health.txt'] = true, ['provider.txt'] = true, ['ui.txt'] = true, } --- TODO: treesitter gets stuck on these files... -local exclude = { - ['filetype.txt'] = true, - ['usr_24.txt'] = true, +-- TODO: These known invalid |links| require an update to the relevant docs. +local exclude_invalid = { + ["'previewpopup'"] = "quickref.txt", + ["'pvp'"] = "quickref.txt", + ["'string'"] = "eval.txt", + Query = "treesitter.txt", + ["eq?"] = "treesitter.txt", + ["lsp-request"] = "lsp.txt", + matchit = "vim_diff.txt", + ["matchit.txt"] = "help.txt", + ["set!"] = "treesitter.txt", + ["v:_null_blob"] = "builtin.txt", + ["v:_null_dict"] = "builtin.txt", + ["v:_null_list"] = "builtin.txt", + ["v:_null_string"] = "builtin.txt", + ["vim.lsp.buf_request()"] = "lsp.txt", + ["vim.lsp.util.get_progress_messages()"] = "lsp.txt", + ["vim.treesitter.start()"] = "treesitter.txt" } local function tofile(fname, text) @@ -82,10 +81,6 @@ local function tofile(fname, text) end local function html_esc(s) - if s:find(']local%-additions[*<]') + ) then + -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + table.insert(noise_lines or {}, line) + return true + end + return false end --- Checks if a given line is a "noise" line that doesn't look good in HTML form. -local function is_noise(line) - return ( - line:find('Type .*gO.* to see the table of contents') - -- Title line of traditional :help pages. - -- Example: "NVIM REFERENCE MANUAL by ..." - or line:find('^%s*N?VIM REFERENCE MANUAL') - -- First line of traditional :help pages. - -- Example: "*api.txt* Nvim" - or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') - -- modeline - -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" - or line:find('^%s*vi[m]%:.*ft=help') - or line:find('^%s*vi[m]%:.*filetype=help') - ) -end - --- Creates a github issue URL at vigoux/tree-sitter-vimdoc with prefilled content. +-- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. local function get_bug_url_vimdoc(fname, to_fname, sample_text) local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ('https://github.com/vigoux/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + local bug_url = ('https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' ..vim.fs.basename(fname) ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' ..this_url @@ -237,31 +236,57 @@ local function getbuflinestr(node, bufnr, offset) return table.concat(lines, '\n') end --- Gets the whitespace just before `node` from the raw buffer text. --- Needed for preformatted `old` lines. -local function getws(node, bufnr) - local line1, c1, line2, _ = node:range() - local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] - local text_before = raw:sub(1, c1) - local leading_ws = text_before:match('%s+$') or '' - return leading_ws -end - -local function get_tagname(node, bufnr, link) - local node_name = (node.named and node:named()) and node:type() or nil +local function get_tagname(node, bufnr) local node_text = vim.treesitter.get_node_text(node, bufnr) - local tag = ((node_name == 'option' and node_text) - or (link and node_text:gsub('^|', ''):gsub('|$', '') or node_text:gsub('^%*', ''):gsub('%*$', ''))) - local helpfile = tag and vim.fs.basename(tagmap[tag]) or nil -- "api.txt" - local helppage = get_helppage(helpfile) -- "api.html" + local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(node_text) or node_text + local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" return helppage, tag end +-- Returns true if the given invalid tagname is a false positive. +local function ignore_invalid(s) + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + return not not (s:find('===') or exclude_invalid[s]) +end + +local function ignore_parse_error(s) + -- Ignore parse errors for unclosed codespan/optionlink/tag. + -- This is common in vimdocs and is treated as plaintext by :help. + return s:find("^[`'|*]") +end + +local function has_ancestor(node, ancestor_name) + local p = node + while true do + p = p:parent() + if not p or p:type() == 'help_file' then + break + elseif p:type() == ancestor_name then + return true + end + end + return false +end + +local function validate_link(node, bufnr, fname) + local helppage, tagname = get_tagname(node:child(1), bufnr, true) + if not has_ancestor(node, 'column_heading') and not node:has_error() and not tagmap[tagname] and not ignore_invalid(tagname) then + invalid_links[tagname] = vim.fs.basename(fname) + end + return helppage, tagname +end + -- Traverses the tree at `root` and checks that |tag| links point to valid helptags. local function visit_validate(root, level, lang_tree, opt, stats) level = level or 0 local node_name = (root.named and root:named()) and root:type() or nil local toplevel = level < 1 + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) + end + local text = trim(node_text()) if root:child_count() > 0 then for node, _ in root:iter_children() do @@ -272,14 +297,26 @@ local function visit_validate(root, level, lang_tree, opt, stats) end if node_name == 'ERROR' then + if ignore_parse_error(text) then + return + end -- Store the raw text to give context to the bug report. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) - elseif node_name == 'hotlink' or node_name == 'option' then - local _, tagname = get_tagname(root, opt.buf, true) - if not root:has_error() and not tagmap[tagname] then - invalid_tags[tagname] = vim.fs.basename(opt.fname) + elseif node_name == 'word' or node_name == 'uppercase_name' then + if spell_dict[text] then + if not invalid_spelling[text] then + invalid_spelling[text] = { vim.fs.basename(opt.fname) } + else + table.insert(invalid_spelling[text], vim.fs.basename(opt.fname)) + end end + elseif node_name == 'url' then + if text:find('http%:') then + invalid_urls[text] = vim.fs.basename(opt.fname) + end + elseif node_name == 'taglink' or node_name == 'optionlink' then + local _, _ = validate_link(root, opt.buf, opt.fname) end end @@ -296,19 +333,18 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local parent = root:parent() and root:parent():type() or nil local text = '' local toplevel = level < 1 - local function node_text() - return vim.treesitter.get_node_text(root, opt.buf) + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) end - if root:child_count() == 0 then + if root:child_count() == 0 or node_name == 'ERROR' then text = node_text() else -- Process children and join them with whitespace. for node, _ in root:iter_children() do if node:named() then local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) - local ws = r == '' and '' or ((opt.old and (node:type() == 'word' or not node:named())) and getws(node, opt.buf) or ' ') - text = string.format('%s%s%s', text, ws, r) + text = string.format('%s%s', text, r) end end end @@ -316,82 +352,112 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if node_name == 'help_file' then -- root node return text + elseif node_name == 'url' then + return ('%s'):format(trimmed, trimmed) elseif node_name == 'word' or node_name == 'uppercase_name' then - if parent == 'headline' then - -- Start a new heading item, or update the current one. - local n = (prev == nil or #headings == 0) and #headings + 1 or #headings - headings[n] = string.format('%s%s', headings[n] and headings[n]..' ' or '', text) - end - return html_esc(text) - elseif node_name == 'headline' then - return ('

%s

\n'):format(to_heading_tag(headings[#headings]), text) + elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then + if is_noise(text, stats.noise_lines) then + return '' -- Discard common "noise" lines. + end + -- Remove "===" and tags from ToC text. + local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) + if node_name == 'h1' or #headings == 0 then + table.insert(headings, { name = hname, subheadings = {}, }) + else + table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, }) + end + local el = node_name == 'h1' and 'h2' or 'h3' + return ('<%s class="help-heading">%s\n'):format(to_heading_tag(hname), el, text, el) elseif node_name == 'column_heading' or node_name == 'column_name' then - return ('

%s

\n'):format(trimmed) - elseif node_name == 'line' then - -- TODO: remove these "sibling inspection" hacks once the parser provides structured info - -- about paragraphs and listitems: https://github.com/vigoux/tree-sitter-vimdoc/issues/12 - local next_text = root:next_sibling() and vim.treesitter.get_node_text(root:next_sibling(), opt.buf) or '' - local li = startswith_bullet(text) -- Listitem? - local next_li = startswith_bullet(next_text) -- Next is listitem? - -- Close the paragraph/listitem if the next sibling is not a line. - local close = (next_ ~= 'line' or next_li or is_blank(next_text)) and '
\n' or '' - - -- HACK: discard common "noise" lines. - if is_noise(text) then - table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) - return (opt.old or prev ~= 'line') and '' or close + if root:has_error() then + return text + end + return ('
%s
'):format(trimmed) + elseif node_name == 'block' then + if is_blank(text) then + return '' end - if opt.old then - -- XXX: Treat old docs as preformatted. Until those docs are "fixed" or we get better info - -- from tree-sitter-vimdoc, this avoids broken layout for legacy docs. - return ('
%s
\n'):format(text) + -- XXX: Treat old docs as preformatted; random indentation is used for layout there. + return ('
%s
\n'):format(text) end - - if li then - return string.format('
%s%s', trim_bullet(expandtabs(text)), close) + return string.format('
\n%s\n
\n', text) + elseif node_name == 'line' then + local sib = root:prev_sibling() + local sib_last = sib and sib:named_child(sib:named_child_count() - 1) + local in_li = false + + -- XXX: parser bug: (codeblock) without terminating "<" consumes first char of the next (line). Recover it here. + local recovered = (sib_last and sib_last:type() == 'codeblock') and node_text(root:prev_sibling()):sub(-1) or '' + recovered = recovered == '<' and '' or html_esc(recovered) + + -- XXX: see if we are currently "in" a listitem. + while sib ~= nil and not in_li do + in_li = (sib:type() == 'line_li') + sib = sib:prev_sibling() end - if prev ~= 'line' then -- Start a new paragraph. - return string.format('
%s%s', expandtabs(text), close) + + -- Close the current listitem. + local close = (in_li and next_ ~= 'line') and '
' or '' + + if is_blank(text) or is_noise(text, stats.noise_lines) then + return close -- Discard common "noise" lines. end - -- Continue in the current paragraph/listitem. - return string.format('%s%s', expandtabs(text), close) - elseif node_name == 'hotlink' or node_name == 'option' then - local helppage, tagname = get_tagname(root, opt.buf, true) - if not root:has_error() and not tagmap[tagname] then - invalid_tags[tagname] = vim.fs.basename(opt.fname) + local div = (root:child(0) and root:child(0):type() == 'column_heading') or close ~= '' + return string.format('%s%s%s%s', recovered, div and trim(text) or text, div and '' or '\n', close) + elseif node_name == 'line_li' then + -- Close the listitem immediately if the next sibling is not a line. + local close = (next_ ~= 'line') and '
' or '' + return string.format('
%s %s', trim_bullet(text), close) + elseif node_name == 'taglink' or node_name == 'optionlink' then + if root:has_error() then + return text + end + local helppage, tagname = validate_link(root, opt.buf, opt.fname) + return (' %s'):format(helppage, url_encode(tagname), html_esc(tagname)) + elseif node_name == 'codespan' then + if root:has_error() then + return text end - return ('%s'):format(helppage, url_encode(tagname), html_esc(tagname)) - elseif node_name == 'backtick' then - return ('%s'):format(html_esc(text)) + return (' %s'):format(text) elseif node_name == 'argument' then - return ('{%s}'):format(html_esc(trimmed)) - elseif node_name == 'code_block' then - return ('
\n%s
\n'):format(html_esc(trim_indent(trim_gt_lt(text)))) + return (' {%s}'):format(trimmed) + elseif node_name == 'codeblock' then + return ('
%s
'):format(html_esc(trim_indent(trim_gt_lt(text)))) elseif node_name == 'tag' then -- anchor - local _, tagname = get_tagname(root, opt.buf, false) - local s = ('%s'):format(url_encode(tagname), trimmed) - if parent == 'headline' and prev ~= 'tag' then + if root:has_error() then + return text + end + local in_heading = (parent == 'h1' or parent == 'h2') + local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' + local tagname = node_text(root:child(1)) + if vim.tbl_count(stats.first_tags) < 2 then + -- First 2 tags in the doc will be anchored at the main heading. + table.insert(stats.first_tags, tagname) + return '' + end + local s = (' %s'):format(url_encode(tagname), cssclass, trimmed) + if in_heading and prev ~= 'tag' then -- Start the container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. --

foo bartag1 tag2

return string.format('%s', s) - elseif parent == 'headline' and next_ == nil then + elseif in_heading and next_ == nil then -- End the container for tags in a heading. return string.format('%s', s) end return s elseif node_name == 'ERROR' then + if ignore_parse_error(trimmed) then + return text + end + -- Store the raw text to give context to the bug report. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) - if prev == 'ERROR' then - -- Avoid trashing the text with cascading errors. - return trimmed, ('parse-error:"%s"'):format(node_text()) - end - return ('%s'):format( + return ('%s'):format( get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) else -- Unknown token. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' @@ -406,8 +472,7 @@ local function get_helpfiles(include) for f, type in vim.fs.dir(dir) do if (vim.endswith(f, '.txt') and type == 'file' - and (not include or vim.tbl_contains(include, f)) - and (not exclude[f])) then + and (not include or vim.tbl_contains(include, f))) then local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') table.insert(rv, fullpath) end @@ -430,6 +495,13 @@ local function get_helptags(help_dir) return m end +-- Use the help.so parser defined in the build, not whatever happens to be installed on the system. +local function ensure_runtimepath() + if not vim.o.runtimepath:find('build/lib/nvim/') then + vim.cmd[[set runtimepath^=./build/lib/nvim/]] + end +end + -- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents. -- -- @returns lang_tree, bufnr @@ -437,7 +509,7 @@ local function parse_buf(fname) local buf if type(fname) == 'string' then vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. - buf = api.nvim_get_current_buf() + buf = vim.api.nvim_get_current_buf() else buf = fname vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. @@ -451,10 +523,9 @@ end -- - checks that |tag| links point to valid helptags. -- - recursively counts parse errors ("ERROR" nodes) -- --- @returns { invalid_tags: number, parse_errors: number } +-- @returns { invalid_links: number, parse_errors: number } local function validate_one(fname) local stats = { - invalid_tags = {}, parse_errors = {}, } local lang_tree, buf = parse_buf(fname) @@ -463,10 +534,7 @@ local function validate_one(fname) end lang_tree:destroy() vim.cmd.close() - return { - invalid_tags = invalid_tags, - parse_errors = stats.parse_errors, - } + return stats end -- Generates HTML from one :help file `fname` and writes the result to `to_fname`. @@ -480,9 +548,10 @@ local function gen_one(fname, to_fname, old, commit) local stats = { noise_lines = {}, parse_errors = {}, + first_tags = {}, -- Track the first few tags in doc. } local lang_tree, buf = parse_buf(fname) - local headings = {} -- Headings (for ToC). + local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. local title = to_titlecase(basename_noext(fname)) local html = ([[ @@ -555,7 +624,12 @@ local function gen_one(fname, to_fname, old, commit) ]] - local main = ([[ + local main = '' + for _, tree in ipairs(lang_tree:trees()) do + main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + end + + main = ([[
' or '' - return string.format('
%s %s', trim_bullet(text), close) + return string.format('
%s
', margin, text) elseif node_name == 'taglink' or node_name == 'optionlink' then if root:has_error() then return text @@ -429,7 +438,10 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) elseif node_name == 'codeblock' then - return ('
%s
'):format(html_esc(trim_indent(trim_gt_lt(text)))) + if is_blank(text) then + return '' + end + return ('
%s
'):format(html_esc(trim(trim_indent(text), 2))) elseif node_name == 'tag' then -- anchor if root:has_error() then return text @@ -630,7 +642,9 @@ local function gen_one(fname, to_fname, old, commit) local main = '' for _, tree in ipairs(lang_tree:trees()) do - main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + main = main .. (visit_node(tree:root(), 0, tree, headings, + { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1, }, + stats)) end main = ([[ @@ -718,6 +732,17 @@ local function gen_css(fname) position: fixed; left: 67%; } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + } + @media (max-width: 40em) { + .golden-grid { + /* Disable grid for narrow viewport (mobile phone). */ + display: block; + } } @media (prefers-color-scheme: dark) { :root { @@ -831,11 +856,6 @@ local function gen_css(fname) color: gray; font-size: smaller; } - .golden-grid { - display: grid; - grid-template-columns: 65% auto; - grid-gap: 1em; - } ]] tofile(fname, css) end @@ -869,6 +889,22 @@ function M._test() eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + local fixed_url, removed_chars = fix_url('https://example.com).') + eq('https://example.com', fixed_url) + eq(').', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.)') + eq('https://example.com.', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.') + eq('https://example.com', fixed_url) + eq('.', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com)') + eq('https://example.com', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com') + eq('https://example.com', fixed_url) + eq('', removed_chars) + print('all tests passed') end -- cgit From 6abb48105135ce3ae7eda22334f8104c5ddf20ce Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 4 Oct 2022 15:15:06 +0200 Subject: fix(docs): missing "(" in :help HTML Problem: Since https://github.com/neovim/tree-sitter-vimdoc/commit/eba7b5b646546d9fed9b40b2c72b9cc0048f1dfa any opening paren and its leading whitespace " (" are missing in the generated HTML. Example: Use ":qa!" (careful, all changes are lost!). ^^missing Position the cursor on a tag (e.g. bars) and hit CTRL-]. ^^missing Solution: The main recursive loop only processes named children, so check named_child_count() instead of child_count(). Then anonymous nodes won't get lost. --- scripts/gen_help_html.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 3141ff3cdf..ccfa2e567e 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -350,7 +350,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return node_text():match('^%s+') or '' end - if root:child_count() == 0 or node_name == 'ERROR' then + if root:named_child_count() == 0 or node_name == 'ERROR' then text = node_text() else -- Process children and join them with whitespace. -- cgit From f7b175e049db9262a45ee1c5eb41a38bd5b8ac38 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 6 Oct 2022 09:16:00 -0400 Subject: fix(docs-html): keycodes, taglinks, column_heading #20498 Problem: - Docs HTML: "foo ~" headings (column_heading) are not aligned with their table columns/contents because the leading whitespace is not emitted. - taglinks starting with hyphen like |-x| were not recognized. - keycodes like `` and `CTRL-x` were not recognized. - ToC is not scrollable. Solution: - Add ws() to the column_heading case. - Update help parser to latest version - supports `keycode` - fixes for taglink, argument - Update .toc CSS. https://github.com/neovim/neovim.github.io/issues/297 fix https://github.com/neovim/neovim.github.io/issues/297 --- scripts/gen_help_html.lua | 54 +++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 21 deletions(-) (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index ccfa2e567e..15269ce175 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -208,7 +208,7 @@ local function get_helppage(f) return 'index.html' end - return f:gsub('%.txt$', '.html') + return (f:gsub('%.txt$', '.html')) end -- Counts leading spaces (tab=8) to decide the indent size of multiline text. @@ -255,9 +255,13 @@ end -- Returns true if the given invalid tagname is a false positive. local function ignore_invalid(s) - -- Strings like |~/====| appear in various places and the parser thinks they are links, but they - -- are just table borders. - return not not (s:find('===') or exclude_invalid[s]) + return not not ( + exclude_invalid[s] + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + or s:find('===') + or s:find('---') + ) end local function ignore_parse_error(s) @@ -281,10 +285,14 @@ end local function validate_link(node, bufnr, fname) local helppage, tagname = get_tagname(node:child(1), bufnr) - if not has_ancestor(node, 'column_heading') and not node:has_error() and not tagmap[tagname] and not ignore_invalid(tagname) then - invalid_links[tagname] = vim.fs.basename(fname) + local ignored = false + if not tagmap[tagname] then + ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname) + if not ignored then + invalid_links[tagname] = vim.fs.basename(fname) + end end - return helppage, tagname + return helppage, tagname, ignored end -- Traverses the tree at `root` and checks that |tag| links point to valid helptags. @@ -325,7 +333,7 @@ local function visit_validate(root, level, lang_tree, opt, stats) invalid_urls[text] = vim.fs.basename(opt.fname) end elseif node_name == 'taglink' or node_name == 'optionlink' then - local _, _ = validate_link(root, opt.buf, opt.fname) + local _, _, _ = validate_link(root, opt.buf, opt.fname) end end @@ -341,7 +349,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) -- Parent kind (string). local parent = root:parent() and root:parent():type() or nil local text = '' - local toplevel = level < 1 + local trimmed local function node_text(node) return vim.treesitter.get_node_text(node or root, opt.buf) end @@ -352,6 +360,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:named_child_count() == 0 or node_name == 'ERROR' then text = node_text() + trimmed = html_esc(trim(text)) + text = html_esc(text) else -- Process children and join them with whitespace. for node, _ in root:iter_children() do @@ -360,8 +370,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) text = string.format('%s%s', text, r) end end + trimmed = trim(text) end - local trimmed = trim(text) if node_name == 'help_file' then -- root node return text @@ -369,7 +379,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local fixed_url, removed_chars = fix_url(trimmed) return ('%s%s%s'):format(ws(), fixed_url, fixed_url, removed_chars) elseif node_name == 'word' or node_name == 'uppercase_name' then - return html_esc(text) + return text elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then if is_noise(text, stats.noise_lines) then return '' -- Discard common "noise" lines. @@ -387,7 +397,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - return ('
%s
'):format(trimmed) + return ('
%s%s
'):format(ws(), trimmed) elseif node_name == 'block' then if is_blank(text) then return '' @@ -425,28 +435,28 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return string.format('
%s
', margin, text) elseif node_name == 'taglink' or node_name == 'optionlink' then - if root:has_error() then + local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname) + if ignored then return text end - local helppage, tagname = validate_link(root, opt.buf, opt.fname) return ('%s%s'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) - elseif node_name == 'codespan' then + elseif vim.tbl_contains({'codespan', 'keycode'}, node_name) then if root:has_error() then return text end - return ('%s%s'):format(ws(), text) + return ('%s%s'):format(ws(), trimmed) elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) elseif node_name == 'codeblock' then if is_blank(text) then return '' end - return ('
%s
'):format(html_esc(trim(trim_indent(text), 2))) + return ('
%s
'):format(trim(trim_indent(text), 2)) elseif node_name == 'tag' then -- anchor if root:has_error() then return text end - local in_heading = (parent == 'h1' or parent == 'h2') + local in_heading = vim.tbl_count({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' local tagname = node_text(root:child(1)) if vim.tbl_count(stats.first_tags) < 2 then @@ -471,12 +481,12 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end -- Store the raw text to give context to the bug report. - local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) return ('%s'):format( get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) else -- Unknown token. - local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' return ('%s'):format( node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) end @@ -751,6 +761,8 @@ local function gen_css(fname) } .toc { /* max-width: 12rem; */ + height: 95%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ } .toc > div { text-overflow: ellipsis; @@ -809,7 +821,7 @@ local function gen_css(fname) .help-tag-right { color: var(--tag-color); } - h1 .help-tag, h2 .help-tag { + h1 .help-tag, h2 .help-tag, h3 .help-tag { font-size: smaller; } .help-heading { -- cgit From 93117b358778487d85665035b6a95976eee70d9c Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Sat, 8 Oct 2022 17:49:09 +0200 Subject: docs(news): add news.txt and link from README (#20426) --- scripts/gen_help_html.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 15269ce175..540caa2ae3 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -44,6 +44,7 @@ local new_layout = { ['channel.txt'] = true, ['develop.txt'] = true, ['luaref.txt'] = true, + ['news.txt'] = true, ['nvim.txt'] = true, ['pi_health.txt'] = true, ['provider.txt'] = true, -- cgit From 09dffb9db7d16496e55e86f78ab60241533d86f6 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 9 Oct 2022 08:21:52 -0400 Subject: docs: various #12823 - increase python line-length limit from 88 => 100. - gen_help_html: fix bug in "tag" case (tbl_count => tbl_contains) ref #15632 fix #18215 fix #18479 fix #20527 fix #20532 Co-authored-by: Ben Weedon --- scripts/gen_help_html.lua | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 540caa2ae3..13601b91f5 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -33,6 +33,7 @@ local spell_dict = { NeoVim = 'Nvim', neovim = 'Nvim', lua = 'Lua', + VimL = 'Vimscript', } local M = {} @@ -42,7 +43,9 @@ local M = {} local new_layout = { ['api.txt'] = true, ['channel.txt'] = true, + ['deprecated.txt'] = true, ['develop.txt'] = true, + ['lua.txt'] = true, ['luaref.txt'] = true, ['news.txt'] = true, ['nvim.txt'] = true, @@ -158,8 +161,8 @@ local function is_noise(line, noise_lines) or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') -- modeline -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" - or line:find('^%s*vi[m]%:.*ft=help') - or line:find('^%s*vi[m]%:.*filetype=help') + or line:find('^%s*vim?%:.*ft=help') + or line:find('^%s*vim?%:.*filetype=help') or line:find('[*>]local%-additions[*<]') ) then -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) @@ -457,7 +460,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - local in_heading = vim.tbl_count({'h1', 'h2', 'h3'}, parent) + local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' local tagname = node_text(root:child(1)) if vim.tbl_count(stats.first_tags) < 2 then @@ -465,7 +468,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) table.insert(stats.first_tags, tagname) return '' end - local s = ('%s%s'):format(ws(), url_encode(tagname), cssclass, trimmed) + local el = in_heading and 'span' or 'code' + local s = ('%s<%s class="%s">%s'):format(ws(), url_encode(tagname), el, cssclass, trimmed, el) if in_heading and prev ~= 'tag' then -- Start the container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. @@ -762,7 +766,7 @@ local function gen_css(fname) } .toc { /* max-width: 12rem; */ - height: 95%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ } .toc > div { -- cgit From a7a83bc4c25d63f3ae0a7a56e5211df1444699c4 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 9 Oct 2022 18:19:43 +0200 Subject: fix(docs-html): update parser - Improve generated HTML by updating parser which includes fixes for single "'" and single "|": https://github.com/neovim/tree-sitter-vimdoc/pull/31 - Updated parser also fixes the conceal issue for "help" highlight queries https://github.com/neovim/tree-sitter-vimdoc/issues/23 by NOT including whitespace in nodes. - But this means we need to restore the getws() function which scrapes leading whitespace from the original input (buffer). --- scripts/gen_help_html.lua | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 13601b91f5..c647484905 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -249,6 +249,16 @@ local function getbuflinestr(node, bufnr, offset) return table.concat(lines, '\n') end +-- Gets the whitespace just before `node` from the raw buffer text. +-- Needed for preformatted `old` lines. +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + local function get_tagname(node, bufnr) local text = vim.treesitter.get_node_text(node, bufnr) local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(text) or text @@ -354,12 +364,21 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local parent = root:parent() and root:parent():type() or nil local text = '' local trimmed - local function node_text(node) - return vim.treesitter.get_node_text(node or root, opt.buf) + -- Gets leading whitespace of `node`. + local function ws(node) + node = node or root + local ws_ = getws(node, opt.buf) + -- XXX: first node of a (line) includes whitespace, even after + -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ? + if ws_ == '' then + ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or '' + end + return ws_ end - -- Gets leading whitespace of the current node. - local function ws() - return node_text():match('^%s+') or '' + local function node_text(node, ws_) + node = node or root + ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or '' + return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) end if root:named_child_count() == 0 or node_name == 'ERROR' then @@ -401,7 +420,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - return ('
%s%s
'):format(ws(), trimmed) + return ('
%s
'):format(text) elseif node_name == 'block' then if is_blank(text) then return '' @@ -462,9 +481,9 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' - local tagname = node_text(root:child(1)) + local tagname = node_text(root:child(1), false) if vim.tbl_count(stats.first_tags) < 2 then - -- First 2 tags in the doc will be anchored at the main heading. + -- Force the first 2 tags in the doc to be anchored at the main heading. table.insert(stats.first_tags, tagname) return '' end -- cgit From 6b01e9bf872cc91530b41fcd97c6bf984776ade6 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Mon, 10 Oct 2022 00:40:17 +0200 Subject: feat(docs-html): try to use tags for ToC headings Problem: The generated ToC (table of contents) uses anchors derived from the heading title, e.g. the "Global Plugins" heading yields: https://neovim.io/doc/user/usr_05.html#_global-plugins- so if the heading title changes, then the old URL (anchor) is broken. Solution: :help tags change less often than heading titles, so if a heading contains a *tag*, use that as its anchor name instead. Example: https://neovim.io/doc/user/usr_05.html#standard-plugin --- scripts/gen_help_html.lua | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'scripts/gen_help_html.lua') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index c647484905..06ea1831b0 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -297,6 +297,16 @@ local function has_ancestor(node, ancestor_name) return false end +-- Gets the first matching child node matching `name`. +local function first(node, name) + for c, _ in node:iter_children() do + if c:named() and c:type() == name then + return c + end + end + return nil +end + local function validate_link(node, bufnr, fname) local helppage, tagname = get_tagname(node:child(1), bufnr) local ignored = false @@ -409,13 +419,18 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end -- Remove "===" and tags from ToC text. local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) + -- Use the first *tag* node as the heading anchor, if any. + local tagnode = first(root, 'tag') + local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname) if node_name == 'h1' or #headings == 0 then - table.insert(headings, { name = hname, subheadings = {}, }) + table.insert(headings, { name = hname, subheadings = {}, tag = tagname }) else - table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, }) + table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, tag = tagname }) end local el = node_name == 'h1' and 'h2' or 'h3' - return ('<%s class="help-heading">%s\n'):format(to_heading_tag(hname), el, text, el) + -- If we are re-using the *tag*, this heading anchor is redundant. + local a = tagnode and '' or (''):format(tagname) + return ('%s<%s class="help-heading">%s\n'):format(a, el, text, el) elseif node_name == 'column_heading' or node_name == 'column_name' then if root:has_error() then return text @@ -720,10 +735,10 @@ local function gen_one(fname, to_fname, old, commit) local n = 0 -- Count of all headings + subheadings. for _, h1 in ipairs(headings) do n = n + 1 + #h1.subheadings end for _, h1 in ipairs(headings) do - toc = toc .. ('
%s\n'):format(to_heading_tag(h1.name), h1.name) + toc = toc .. ('
%s\n'):format(h1.tag, h1.name) if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. for _, h2 in ipairs(h1.subheadings) do - toc = toc .. ('
%s
\n'):format(to_heading_tag(h2.name), h2.name) + toc = toc .. ('
%s
\n'):format(h2.tag, h2.name) end end toc = toc .. '
' -- cgit