From 0c0a426e40692eabb3e1800cc93135eb70a28334 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 23 Aug 2022 14:18:05 +0100 Subject: doc(CONTRIBUTING): update #19891 - add section for managing includes via IWYU - recommend clangd over ctags - tidy up links - remove ./scripts/check-includes.py --- scripts/check-includes.py | 64 ----------------------------------------------- 1 file changed, 64 deletions(-) delete mode 100755 scripts/check-includes.py (limited to 'scripts') diff --git a/scripts/check-includes.py b/scripts/check-includes.py deleted file mode 100755 index ed1fe407c5..0000000000 --- a/scripts/check-includes.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python - -import sys -import re -import os - -from subprocess import Popen, PIPE -from argparse import ArgumentParser - - -GENERATED_INCLUDE_RE = re.compile( - r'^\s*#\s*include\s*"([/a-z_0-9.]+\.generated\.h)"(\s+//.*)?$') - - -def main(argv): - argparser = ArgumentParser() - argparser.add_argument('--generated-includes-dir', action='append', - help='Directory where generated includes are located.') - argparser.add_argument('--file', type=open, help='File to check.') - argparser.add_argument('iwyu_args', nargs='*', - help='IWYU arguments, must go after --.') - args = argparser.parse_args(argv) - - with args.file: - iwyu = Popen(['include-what-you-use', '-xc'] + args.iwyu_args + ['/dev/stdin'], - stdin=PIPE, stdout=PIPE, stderr=PIPE) - - for line in args.file: - match = GENERATED_INCLUDE_RE.match(line) - if match: - for d in args.generated_includes_dir: - try: - f = open(os.path.join(d, match.group(1))) - except IOError: - continue - else: - with f: - for generated_line in f: - iwyu.stdin.write(generated_line) - break - else: - raise IOError('Failed to find {0}'.format(match.group(1))) - else: - iwyu.stdin.write(line) - - iwyu.stdin.close() - - out = iwyu.stdout.read() - err = iwyu.stderr.read() - - ret = iwyu.wait() - - if ret != 2: - print('IWYU failed with exit code {0}:'.format(ret)) - print('{0} stdout {0}'.format('=' * ((80 - len(' stdout ')) // 2))) - print(out) - print('{0} stderr {0}'.format('=' * ((80 - len(' stderr ')) // 2))) - print(err) - return 1 - return 0 - - -if __name__ == '__main__': - raise SystemExit(main(sys.argv[1:])) -- cgit From c0d60526541a3cf977ae623471ae4a347b492af1 Mon Sep 17 00:00:00 2001 From: bfredl Date: Tue, 23 Aug 2022 09:33:08 +0200 Subject: perf(api): allow to use an arena for return values --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index bca3dd816f..23ed0e3f08 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -801,7 +801,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error')) + if a[0] not in ('void', 'Error', 'Arena')) if not fmt_vimhelp: c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) -- cgit From ddb762f4013ac2532ad45704466058d867e3a6ed Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Wed, 14 Sep 2022 11:08:31 +0200 Subject: docs(treesitter): clean up and update treesitter.txt (#20142) * add type annotations to code * clean up and expand static documentation * consistent use of tags for static and generated docs --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 23ed0e3f08..766c420c7d 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -260,7 +260,7 @@ CONFIG = { 'helptag_fmt': lambda name: ( '*lua-treesitter-core*' if name.lower() == 'treesitter' - else f'*treesitter-{name.lower()}*'), + else f'*lua-treesitter-{name.lower()}*'), 'fn_helptag_fmt': lambda fstem, name: ( f'*{name}()*' if name != 'new' -- cgit From 09b64d75bd92a95d89c4f39f9df7918760abe98d Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 27 Mar 2022 19:47:34 -0700 Subject: feat(docs): gen_help_html.lua MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: The :help docs HTML generated is driven by an old awk script `runtime/doc/makehtml.awk` that is hard to maintain (ad hoc parser and no one has touched it in decades) and has bugs like: - https://github.com/neovim/neovim.github.io/issues/96 - https://github.com/neovim/neovim.github.io/issues/97 Solution: Use Lua + treesitter (https://github.com/vigoux/tree-sitter-vimdoc) to generate :help docs HTML. Also validates tag links. fix https://github.com/neovim/neovim.github.io/issues/96 fix https://github.com/neovim/neovim.github.io/issues/97 TODO: - delete doc_html build task - delete runtime/doc/Makefile - delete makehtml.awk - delete maketags.awk OUTPUT: $ nvim -V1 -es --clean +"lua require('scripts.gen_help_html')" output dir: /…/neovim.github.io/_site/doc/ generated (207 errors): api.txt => api.html generated (122 errors): arabic.txt => arabic.html generated (285 errors): autocmd.txt => autocmd.html generated (641 errors): builtin.txt => builtin.html generated (623 errors): change.txt => change.html generated (65 errors): channel.txt => channel.html generated (353 errors): cmdline.txt => cmdline.html generated (3 errors): debug.txt => debug.html generated (28 errors): deprecated.txt => deprecated.html generated (193 errors): dev_style.txt => dev_style.html generated (460 errors): develop.txt => develop.html generated (19 errors): diagnostic.txt => diagnostic.html generated (57 errors): diff.txt => diff.html generated (818 errors): digraph.txt => digraph.html generated (330 errors): editing.txt => editing.html generated (368 errors): eval.txt => eval.html generated (184 errors): fold.txt => fold.html generated (61 errors): ft_ada.txt => ft_ada.html generated (0 errors): ft_ps1.txt => ft_ps1.html generated (20 errors): ft_raku.txt => ft_raku.html generated (5 errors): ft_rust.txt => ft_rust.html generated (41 errors): ft_sql.txt => ft_sql.html generated (110 errors): gui.txt => gui.html generated (79 errors): hebrew.txt => hebrew.html generated (17 errors): help.txt => index.html generated (104 errors): helphelp.txt => helphelp.html generated (0 errors): if_cscop.txt => if_cscop.html generated (23 errors): if_perl.txt => if_perl.html generated (16 errors): if_pyth.txt => if_pyth.html generated (9 errors): if_ruby.txt => if_ruby.html generated (216 errors): indent.txt => indent.html generated (634 errors): index.txt => vimindex.html generated (320 errors): insert.txt => insert.html generated (265 errors): intro.txt => intro.html generated (9 errors): job_control.txt => job_control.html generated (0 errors): lsp-extension.txt => lsp-extension.html generated (214 errors): lsp.txt => lsp.html generated (311 errors): lua.txt => lua.html generated (592 errors): luaref.txt => luaref.html generated (798 errors): luvref.txt => luvref.html generated (663 errors): map.txt => map.html generated (228 errors): mbyte.txt => mbyte.html generated (228 errors): message.txt => message.html generated (0 errors): mlang.txt => mlang.html generated (761 errors): motion.txt => motion.html generated (4 errors): nvim.txt => nvim.html generated (226 errors): nvim_terminal_emulator.txt => nvim_terminal_emulator.html generated (988 errors): options.txt => options.html generated (567 errors): pattern.txt => pattern.html generated (15 errors): pi_gzip.txt => pi_gzip.html generated (10 errors): pi_health.txt => pi_health.html generated (27 errors): pi_msgpack.txt => pi_msgpack.html generated (2177 errors): pi_netrw.txt => pi_netrw.html generated (41 errors): pi_paren.txt => pi_paren.html generated (9 errors): pi_spec.txt => pi_spec.html generated (218 errors): pi_tar.txt => pi_tar.html generated (0 errors): pi_tutor.txt => pi_tutor.html generated (235 errors): pi_zip.txt => pi_zip.html generated (265 errors): print.txt => print.html generated (31 errors): provider.txt => provider.html generated (335 errors): quickfix.txt => quickfix.html generated (572 errors): quickref.txt => quickref.html generated (109 errors): recover.txt => recover.html generated (14 errors): remote.txt => remote.html generated (14 errors): remote_plugin.txt => remote_plugin.html generated (351 errors): repeat.txt => repeat.html generated (23 errors): rileft.txt => rileft.html generated (12 errors): russian.txt => russian.html generated (6 errors): scroll.txt => scroll.html generated (106 errors): sign.txt => sign.html generated (347 errors): spell.txt => spell.html generated (784 errors): starting.txt => starting.html generated (1499 errors): syntax.txt => syntax.html generated (23 errors): tabpage.txt => tabpage.html generated (257 errors): tagsrch.txt => tagsrch.html generated (31 errors): term.txt => term.html generated (0 errors): testing.txt => testing.html generated (96 errors): tips.txt => tips.html generated (57 errors): treesitter.txt => treesitter.html generated (71 errors): uganda.txt => uganda.html generated (74 errors): ui.txt => ui.html generated (87 errors): undo.txt => undo.html generated (17 errors): userfunc.txt => userfunc.html generated (1 errors): usr_01.txt => usr_01.html generated (89 errors): usr_02.txt => usr_02.html generated (293 errors): usr_03.txt => usr_03.html generated (46 errors): usr_04.txt => usr_04.html generated (96 errors): usr_05.txt => usr_05.html generated (54 errors): usr_06.txt => usr_06.html generated (20 errors): usr_07.txt => usr_07.html generated (241 errors): usr_08.txt => usr_08.html generated (130 errors): usr_09.txt => usr_09.html generated (50 errors): usr_10.txt => usr_10.html generated (33 errors): usr_11.txt => usr_11.html generated (32 errors): usr_12.txt => usr_12.html generated (22 errors): usr_20.txt => usr_20.html generated (75 errors): usr_21.txt => usr_21.html generated (8 errors): usr_22.txt => usr_22.html generated (3 errors): usr_23.txt => usr_23.html generated (163 errors): usr_25.txt => usr_25.html generated (13 errors): usr_26.txt => usr_26.html generated (84 errors): usr_27.txt => usr_27.html generated (173 errors): usr_28.txt => usr_28.html generated (285 errors): usr_29.txt => usr_29.html generated (280 errors): usr_30.txt => usr_30.html generated (11 errors): usr_31.txt => usr_31.html generated (13 errors): usr_32.txt => usr_32.html generated (156 errors): usr_40.txt => usr_40.html generated (134 errors): usr_41.txt => usr_41.html generated (35 errors): usr_42.txt => usr_42.html generated (19 errors): usr_43.txt => usr_43.html generated (60 errors): usr_44.txt => usr_44.html generated (13 errors): usr_45.txt => usr_45.html generated (1 errors): usr_toc.txt => usr_toc.html generated (69 errors): various.txt => various.html generated (68 errors): vi_diff.txt => vi_diff.html generated (437 errors): vim_diff.txt => vim_diff.html generated (296 errors): visual.txt => visual.html generated (181 errors): windows.txt => windows.html generated 119 html pages total errors: 23862 invalid tags: 537 --- scripts/gen_help_html.lua | 830 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/gen_help_html.py | 389 ---------------------- 2 files changed, 830 insertions(+), 389 deletions(-) create mode 100644 scripts/gen_help_html.lua delete mode 100644 scripts/gen_help_html.py (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua new file mode 100644 index 0000000000..d79ece53f3 --- /dev/null +++ b/scripts/gen_help_html.lua @@ -0,0 +1,830 @@ +-- Converts Vim :help files to HTML. Validates |tag| links and document syntax (parser errors). +-- +-- USAGE (GENERATE HTML): +-- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" +-- - Read the docstring at gen(). +-- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 +-- 4. Visit http://localhost:4000/…/help.txt.html +-- +-- USAGE (VALIDATE): +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate()" +-- - validate() is 10x faster than gen(), so it is used in CI. +-- +-- SELF-TEST MODE: +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +-- +-- NOTES: +-- * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too +-- slow (~1 min) to run in per-commit CI. +-- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. +-- * visit_validate() is the core function used by validate(). +-- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. +-- +-- parser bugs: +-- * Should NOT be code_block: +-- tab:xy The 'x' is always used, then 'y' as many times as will +-- fit. Thus "tab:>-" displays: +-- > +-- >- +-- >-- +-- etc. +-- +-- tab:xyz The 'z' is always used, then 'x' is prepended, and +-- then 'y' is used as many times as will fit. Thus +-- "tab:<->" displays: +-- > +-- <> +-- <-> +-- <--> +-- etc. +-- * Should NOT be a "headline". Perhaps a "table" (or just "line"). +-- expr5 and expr6 *expr5* *expr6* +-- --------------- +-- expr6 + expr6 Number addition, |List| or |Blob| concatenation *expr-+* +-- expr6 - expr6 Number subtraction *expr--* +-- expr6 . expr6 String concatenation *expr-.* +-- expr6 .. expr6 String concatenation *expr-..* + +local tagmap = nil +local helpfiles = nil +local invalid_tags = {} + +local commit = '?' +local api = vim.api +local M = {} + +-- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). +-- All other files are "legacy" files which require fixed-width layout. +local new_layout = { + ['api.txt'] = true, + ['channel.txt'] = true, + ['develop.txt'] = true, + ['nvim.txt'] = true, + ['pi_health.txt'] = true, + ['provider.txt'] = true, + ['ui.txt'] = true, +} + +-- TODO: treesitter gets stuck on these files... +local exclude = { + ['filetype.txt'] = true, + ['usr_24.txt'] = true, +} + +local function tofile(fname, text) + local f = io.open(fname, 'w') + if not f then + error(('failed to write: %s'):format(f)) + else + f:write(text) + f:close() + end +end + +local function html_esc(s) + if s:find('', '>') +end + +local function url_encode(s) + -- Credit: tpope / vim-unimpaired + -- NOTE: these chars intentionally *not* escaped: ' ( ) + return vim.fn.substitute(vim.fn.iconv(s, 'latin1', 'utf-8'), + [=[[^A-Za-z0-9()'_.~-]]=], + [=[\="%".printf("%02X",char2nr(submatch(0)))]=], + 'g') +end + +-- Removes the ">" and "<" chars that delineate a codeblock in Vim :help files. +local function trim_gt_lt(s) + return s:gsub('^%s*>%s*\n', ''):gsub('\n<', '') +end + +local function expandtabs(s) + return s:gsub('\t', (' '):rep(8)) +end + +local function to_titlecase(s) + local text = '' + for w in vim.gsplit(s, '[ \t]+') do + text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2)) + end + return text +end + +local function to_heading_tag(text) + -- Prepend "_" to avoid conflicts with actual :help tags. + return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown' +end + +local function basename_noext(f) + return vim.fs.basename(f:gsub('%.txt', '')) +end + +local function is_blank(s) + return not not s:find('^%s*$') +end + +local function trim(s) + return vim.trim(s) +end + +local function trim_bullet(s) + return s:gsub('^%s*[-*•]%s', '') +end + +local function startswith_bullet(s) + return s:find('^%s*[-*•]%s') +end + +-- Checks if a given line is a "noise" line that doesn't look good in HTML form. +local function is_noise(line) + return ( + line:find('Type .*gO.* to see the table of contents') + -- Title line of traditional :help pages. + -- Example: "NVIM REFERENCE MANUAL by ..." + or line:find('^%s*N?VIM REFERENCE MANUAL') + -- First line of traditional :help pages. + -- Example: "*api.txt* Nvim" + or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') + -- modeline + -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" + or line:find('^%s*vi[m]%:.*ft=help') + or line:find('^%s*vi[m]%:.*filetype=help') + ) +end + +-- Creates a github issue URL at vigoux/tree-sitter-vimdoc with prefilled content. +local function get_bug_url_vimdoc(fname, to_fname, sample_text) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ('https://github.com/vigoux/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + ..vim.fs.basename(fname) + ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' + ..this_url + ..'%0D%0DContext%3A%0D%0D%60%60%60%0D' + ..url_encode(sample_text) + ..'%0D%60%60%60') + return bug_url +end + +-- Creates a github issue URL at neovim/neovim with prefilled content. +local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ('https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' + ..vim.fs.basename(fname) + ..'+&body=%60gen_help_html.lua%60+problem+at%3A+' + ..this_url + ..'%0D' + ..(token_name and '+unhandled+token%3A+%60'..token_name..'%60' or '') + ..'%0DContext%3A%0D%0D%60%60%60%0D' + ..url_encode(sample_text) + ..'%0D%60%60%60') + return bug_url +end + +-- Gets a "foo.html" name from a "foo.txt" helpfile name. +local function get_helppage(f) + if not f then + return nil + end + -- Special case: help.txt is the "main landing page" of :help files, not index.txt. + if f == 'index.txt' then + return 'vimindex.html' + elseif f == 'help.txt' then + return 'index.html' + end + + return f:gsub('%.txt$', '.html') +end + +-- Counts leading spaces (tab=8) to decide the indent size of multiline text. +-- +-- Blank lines (empty or whitespace-only) are ignored. +local function get_indent(s) + local min_indent = nil + for line in vim.gsplit(s, '\n') do + if line and not is_blank(line) then + local ws = expandtabs(line:match('^%s+') or '') + min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent + end + end + return min_indent or 0 +end + +-- Removes the common indent level, after expanding tabs to 8 spaces. +local function trim_indent(s) + local indent_size = get_indent(s) + local trimmed = '' + for line in vim.gsplit(s, '\n') do + line = expandtabs(line) + trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1)) + end + return trimmed:sub(1, -2) +end + +-- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. +local function getbuflinestr(node, bufnr, offset) + local line1, _, line2, _ = node:range() + line1 = line1 - offset + line2 = line2 + offset + local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1) + return table.concat(lines, '\n') +end + +-- Gets the whitespace just before `node` from the raw buffer text. +-- Needed for preformatted `old` lines. +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + +local function get_tagname(node, bufnr, link) + local node_name = (node.named and node:named()) and node:type() or nil + local node_text = vim.treesitter.get_node_text(node, bufnr) + local tag = ((node_name == 'option' and node_text) + or (link and node_text:gsub('^|', ''):gsub('|$', '') or node_text:gsub('^%*', ''):gsub('%*$', ''))) + local helpfile = tag and vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" + return helppage, tag +end + +-- Traverses the tree at `root` and checks that |tag| links point to valid helptags. +local function visit_validate(root, level, lang_tree, opt, stats) + level = level or 0 + local node_name = (root.named and root:named()) and root:type() or nil + local toplevel = level < 1 + + if root:child_count() > 0 then + for node, _ in root:iter_children() do + if node:named() then + visit_validate(node, level + 1, lang_tree, opt, stats) + end + end + end + + if node_name == 'ERROR' then + -- Store the raw text to give context to the bug report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + elseif node_name == 'hotlink' or node_name == 'option' then + local _, tagname = get_tagname(root, opt.buf, true) + if not root:has_error() and not tagmap[tagname] then + invalid_tags[tagname] = vim.fs.basename(opt.fname) + end + end +end + +-- Generates HTML from node `root` recursively. +local function visit_node(root, level, lang_tree, headings, opt, stats) + level = level or 0 + + local node_name = (root.named and root:named()) and root:type() or nil + -- Previous sibling kind (string). + local prev = root:prev_sibling() and (root:prev_sibling().named and root:prev_sibling():named()) and root:prev_sibling():type() or nil + -- Next sibling kind (string). + local next_ = root:next_sibling() and (root:next_sibling().named and root:next_sibling():named()) and root:next_sibling():type() or nil + -- Parent kind (string). + local parent = root:parent() and root:parent():type() or nil + local text = '' + local toplevel = level < 1 + local function node_text() + return vim.treesitter.get_node_text(root, opt.buf) + end + + if root:child_count() == 0 then + text = node_text() + else + -- Process children and join them with whitespace. + for node, _ in root:iter_children() do + if node:named() then + local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) + local ws = r == '' and '' or ((opt.old and (node:type() == 'word' or not node:named())) and getws(node, opt.buf) or ' ') + text = string.format('%s%s%s', text, ws, r) + end + end + end + local trimmed = trim(text) + + if node_name == 'help_file' then -- root node + return text + elseif node_name == 'word' or node_name == 'uppercase_name' then + if parent == 'headline' then + -- Start a new heading item, or update the current one. + local n = (prev == nil or #headings == 0) and #headings + 1 or #headings + headings[n] = string.format('%s%s', headings[n] and headings[n]..' ' or '', text) + end + + return html_esc(text) + elseif node_name == 'headline' then + return ('

%s

\n'):format(to_heading_tag(headings[#headings]), text) + elseif node_name == 'column_heading' or node_name == 'column_name' then + return ('

%s

\n'):format(trimmed) + elseif node_name == 'line' then + -- TODO: remove these "sibling inspection" hacks once the parser provides structured info + -- about paragraphs and listitems: https://github.com/vigoux/tree-sitter-vimdoc/issues/12 + local next_text = root:next_sibling() and vim.treesitter.get_node_text(root:next_sibling(), opt.buf) or '' + local li = startswith_bullet(text) -- Listitem? + local next_li = startswith_bullet(next_text) -- Next is listitem? + -- Close the paragraph/listitem if the next sibling is not a line. + local close = (next_ ~= 'line' or next_li or is_blank(next_text)) and '\n' or '' + + -- HACK: discard common "noise" lines. + if is_noise(text) then + table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + return (opt.old or prev ~= 'line') and '' or close + end + + if opt.old then + -- XXX: Treat old docs as preformatted. Until those docs are "fixed" or we get better info + -- from tree-sitter-vimdoc, this avoids broken layout for legacy docs. + return ('
%s
\n'):format(text) + end + + if li then + return string.format('
%s%s', trim_bullet(expandtabs(text)), close) + end + if prev ~= 'line' then -- Start a new paragraph. + return string.format('
%s%s', expandtabs(text), close) + end + + -- Continue in the current paragraph/listitem. + return string.format('%s%s', expandtabs(text), close) + elseif node_name == 'hotlink' or node_name == 'option' then + local helppage, tagname = get_tagname(root, opt.buf, true) + if not root:has_error() and not tagmap[tagname] then + invalid_tags[tagname] = vim.fs.basename(opt.fname) + end + return ('%s'):format(helppage, url_encode(tagname), html_esc(tagname)) + elseif node_name == 'backtick' then + return ('%s'):format(html_esc(text)) + elseif node_name == 'argument' then + return ('{%s}'):format(html_esc(trimmed)) + elseif node_name == 'code_block' then + return ('
\n%s
\n'):format(html_esc(trim_indent(trim_gt_lt(text)))) + elseif node_name == 'tag' then -- anchor + local _, tagname = get_tagname(root, opt.buf, false) + local s = ('%s'):format(url_encode(tagname), trimmed) + if parent == 'headline' and prev ~= 'tag' then + -- Start the container for tags in a heading. + -- This makes "justify-content:space-between" right-align the tags. + --

foo bartag1 tag2

+ return string.format('%s', s) + elseif parent == 'headline' and next_ == nil then + -- End the container for tags in a heading. + return string.format('%s', s) + end + return s + elseif node_name == 'ERROR' then + -- Store the raw text to give context to the bug report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + if prev == 'ERROR' then + -- Avoid trashing the text with cascading errors. + return trimmed, ('parse-error:"%s"'):format(node_text()) + end + return ('%s'):format( + get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) + else -- Unknown token. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + return ('%s'):format( + node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) + end +end + +local function get_helpfiles(include) + local dir = './build/runtime/doc' + local rv = {} + for f, type in vim.fs.dir(dir) do + if (vim.endswith(f, '.txt') + and type == 'file' + and (not include or vim.tbl_contains(include, f)) + and (not exclude[f])) then + local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') + table.insert(rv, fullpath) + end + end + return rv +end + +-- Populates the helptags map. +local function get_helptags(help_dir) + local m = {} + -- Load a random help file to convince taglist() to do its job. + vim.cmd(string.format('split %s/api.txt', help_dir)) + vim.cmd('lcd %:p:h') + for _, item in ipairs(vim.fn.taglist('.*')) do + if vim.endswith(item.filename, '.txt') then + m[item.name] = item.filename + end + end + vim.cmd('q!') + return m +end + +-- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents. +-- +-- @returns lang_tree, bufnr +local function parse_buf(fname) + local buf + if type(fname) == 'string' then + vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. + buf = api.nvim_get_current_buf() + else + buf = fname + vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. + end + -- vim.treesitter.require_language('help', './build/lib/nvim/parser/help.so') + local lang_tree = vim.treesitter.get_parser(buf, 'help') + return lang_tree, buf +end + +-- Validates one :help file `fname`: +-- - checks that |tag| links point to valid helptags. +-- - recursively counts parse errors ("ERROR" nodes) +-- +-- @returns { invalid_tags: number, parse_errors: number } +local function validate_one(fname) + local stats = { + invalid_tags = {}, + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname) + for _, tree in ipairs(lang_tree:trees()) do + visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname, }, stats) + end + lang_tree:destroy() + vim.cmd.close() + return { + invalid_tags = invalid_tags, + parse_errors = stats.parse_errors, + } +end + +-- Generates HTML from one :help file `fname` and writes the result to `to_fname`. +-- +-- @param fname Source :help file +-- @param to_fname Destination .html file +-- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) +-- +-- @returns html, stats +local function gen_one(fname, to_fname, old) + local stats = { + noise_lines = {}, + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname) + local headings = {} -- Headings (for ToC). + local title = to_titlecase(basename_noext(fname)) + + local html = ([[ + + + + + + + + + + + + %s - Neovim docs + + + ]]):format(title) + + local logo_svg = [[ + + Neovim + + + + + + + + + + + + + + + + + + + + + + + + + + ]] + + local main = ([[ +
+ +
+ +
+
+

%s

+

+ + Nvim help pages, updated automatically + from source. + Parsing by tree-sitter-vimdoc. + +

+ ]]):format(logo_svg, title, vim.fs.basename(fname)) + for _, tree in ipairs(lang_tree:trees()) do + main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + end + main = main .. '
\n' + + local toc = [[ +
+ + + +
+ ]] + for _, heading in ipairs(headings) do + toc = toc .. ('\n'):format(to_heading_tag(heading), heading) + end + toc = toc .. '
\n' + + local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil) + local bug_link = string.format('(report docs bug...)', bug_url) + + local footer = ([[ +
+
+
+ Generated on %s from {%s} +
+
+ parse_errors: %d %s | noise_lines: %d +
+
+
+ ]]):format( + os.date('%Y-%m-%d %H:%M:%S'), commit, #stats.parse_errors, bug_link, + html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines) + + html = ('%s%s%s
\n%s\n\n'):format( + html, main, toc, footer) + vim.cmd('q!') + lang_tree:destroy() + return html, stats +end + +local function gen_css(fname) + local css = [[ + @media (min-width: 40em) { + .toc { + position: fixed; + left: 67%; + } + } + .toc { + /* max-width: 12rem; */ + } + .toc > div { + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + } + html { + scroll-behavior: auto; + } + h1, h2, h3, h4 { + font-family: sans-serif; + } + .help-body { + padding-bottom: 2em; + } + .help-line { + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + } + .help-item { + display: list-item; + margin-left: 1.5rem; /* padding-left: 1rem; */ + } + .help-para { + padding-top: 10px; + padding-bottom: 10px; + } + .old-help-line { + /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */ + tab-size: 8; + white-space: pre; + font-size: .875em; + font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; + } + a.help-tag, a.help-tag:focus, a.help-tag:hover { + color: inherit; + text-decoration: none; + } + .help-tag { + color: gray; + } + h1 .help-tag, h2 .help-tag { + font-size: smaller; + } + .help-heading { + overflow: hidden; + white-space: nowrap; + display: flex; + justify-content: space-between; + } + /* The (right-aligned) "tags" part of a section heading. */ + .help-heading-tags { + margin-left: 10px; + } + .parse-error { + background-color: red; + } + .unknown-token { + color: black; + background-color: yellow; + } + pre { + /* Tabs are used in code_blocks only for indentation, not alignment, so we can aggressively shrink them. */ + tab-size: 2; + white-space: pre; + overflow: visible; + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + /* font-size: 14px; */ + /* border: 0px; */ + /* margin: 0px; */ + } + pre:hover, + .help-heading:hover { + overflow: visible; + } + .generator-stats { + color: gray; + font-size: smaller; + } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + ]] + tofile(fname, css) +end + +function M._test() + tagmap = get_helptags('./build/runtime/doc') + helpfiles = get_helpfiles() + + local function ok(cond, expected, actual) + assert((not expected and not actual) or (expected and actual), 'if "expected" is given, "actual" is also required') + if expected then + return assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual))) + else + return assert(cond) + end + end + local function eq(expected, actual) + return ok(expected == actual, expected, actual) + end + + eq(119, #helpfiles) + ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) + ok(vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), tagmap['vim.diagnostic.set()'], 'diagnostic.txt') + ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt') + ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]])) + ok(is_noise([[ VIM REFERENCE MANUAL by Abe Lincoln ]])) + ok(not is_noise([[vim:tw=78]])) + + eq(0, get_indent('a')) + eq(1, get_indent(' a')) + eq(2, get_indent(' a\n b\n c\n')) + eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) + eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + + print('all tests passed') +end + +--- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`. +--- +--- Example: +--- +--- gen('./build/runtime/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) +--- +--- @param help_dir string Source directory containing the :help files. Must run `make helptags` first. +--- @param to_dir string Target directory where the .html files will be written. +--- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} +--- +--- @returns info dict +function M.gen(help_dir, to_dir, include) + vim.validate{ + help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, + to_dir={to_dir, 's'}, + include={include, 't', true}, + } + + local err_count = 0 + tagmap = get_helptags(help_dir) + helpfiles = get_helpfiles(include) + + print(('output dir: %s'):format(to_dir)) + vim.fn.mkdir(to_dir, 'p') + gen_css(('%s/help.css'):format(to_dir)) + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) + local html, stats = gen_one(f, to_fname, not new_layout[helpfile]) + tofile(to_fname, html) + print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) + err_count = err_count + #stats.parse_errors + end + print(('generated %d html pages'):format(#helpfiles)) + print(('total errors: %d'):format(err_count)) + print(('invalid tags:\n%s'):format(vim.inspect(invalid_tags))) + + return { + helpfiles = helpfiles, + err_count = err_count, + invalid_tags = invalid_tags, + } +end + +-- Validates all :help files found in `help_dir`: +-- - checks that |tag| links point to valid helptags. +-- - recursively counts parse errors ("ERROR" nodes) +-- +-- This is 10x faster than gen(), for use in CI. +-- +-- @returns results dict +function M.validate(help_dir, include) + vim.validate{ + help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, + include={include, 't', true}, + } + local err_count = 0 + tagmap = get_helptags(help_dir) + helpfiles = get_helpfiles(include) + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local rv = validate_one(f) + print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) + err_count = err_count + #rv.parse_errors + end + + return { + helpfiles = helpfiles, + err_count = err_count, + invalid_tags = invalid_tags, + } +end + +return M diff --git a/scripts/gen_help_html.py b/scripts/gen_help_html.py deleted file mode 100644 index 0b8e77ac22..0000000000 --- a/scripts/gen_help_html.py +++ /dev/null @@ -1,389 +0,0 @@ -# Converts Vim/Nvim documentation to HTML. -# -# USAGE: -# 1. python3 scripts/gen_help_html.py runtime/doc/ ~/neovim.github.io/t/ -# 3. cd ~/neovim.github.io/ && jekyll serve --host 0.0.0.0 -# 2. Visit http://localhost:4000/t/help.txt.html -# -# Adapted from https://github.com/c4rlo/vimhelp/ -# License: MIT -# -# Copyright (c) 2016 Carlo Teubner -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import re -import urllib.parse -import datetime -import sys -from itertools import chain - -HEAD = """\ - - - - - -Nvim: {filename} -""" - -HEAD_END = '\n\n' - -INTRO = """ -

Nvim help files

-

-Nvim help pages{vers-note}. -Updated automatically -from the Nvim source. -

-""" - -VERSION_NOTE = ", current as of Nvim {version}" - -SITENAVI_LINKS = """ -Quick reference · -User manual · -Reference manual · -""" - -SITENAVI_LINKS_PLAIN = SITENAVI_LINKS.format(helptxt='help.txt.html') -SITENAVI_LINKS_WEB = SITENAVI_LINKS.format(helptxt='/') - -SITENAVI_PLAIN = '

' + SITENAVI_LINKS_PLAIN + '

' -SITENAVI_WEB = '

' + SITENAVI_LINKS_WEB + '

' - -SITENAVI_SEARCH = '
' + SITENAVI_LINKS_WEB + \ - '
' \ - '
' - -TEXTSTART = """ -
-
""" + (" " * 80) + """
-
-
-"""
-
-FOOTER = '
' - -FOOTER2 = """ - -
-
- - -""".format( - generated_date='{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()), - commit='?') - -RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)') - -PAT_WORDCHAR = '[!#-)+-{}~\xC0-\xFF]' - -PAT_HEADER = r'(^.*~$)' -PAT_GRAPHIC = r'(^.* `$)' -PAT_PIPEWORD = r'(?|.)?)' -PAT_SPECIAL = r'(<.+?>|\{.+?}|' \ - r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' \ - r'arg|arguments|ident|addr|group)]|' \ - r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])' -PAT_TITLE = r'(Vim version [0-9.a-z]+|VIM REFERENCE.*)' -PAT_NOTE = r'((? \t]+[a-zA-Z0-9/])' -PAT_WORD = r'((?$') -RE_EG_END = re.compile(r'\S') -RE_SECTION = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)') -RE_STARTAG = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)') -RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$') - - -class Link(object): - __slots__ = 'link_plain_same', 'link_pipe_same', \ - 'link_plain_foreign', 'link_pipe_foreign', \ - 'filename' - - def __init__(self, link_plain_same, link_plain_foreign, - link_pipe_same, link_pipe_foreign, filename): - self.link_plain_same = link_plain_same - self.link_plain_foreign = link_plain_foreign - self.link_pipe_same = link_pipe_same - self.link_pipe_foreign = link_pipe_foreign - self.filename = filename - - -class VimH2H(object): - def __init__(self, tags, version=None, is_web_version=True): - self._urls = {} - self._version = version - self._is_web_version = is_web_version - for line in RE_NEWLINE.split(tags): - m = RE_TAGLINE.match(line) - if m: - tag, filename = m.group(1, 2) - self.do_add_tag(filename, tag) - - def add_tags(self, filename, contents): - for match in RE_STARTAG.finditer(contents): - tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/') - self.do_add_tag(str(filename), tag) - - def do_add_tag(self, filename, tag): - tag_quoted = urllib.parse.quote_plus(tag) - - def mkpart1(doc): - return '' + html_escape[tag] + '' - - def mklinks(cssclass): - return (part1_same + cssclass + part2, - part1_foreign + cssclass + part2) - cssclass_plain = 'd' - m = RE_LINKWORD.match(tag) - if m: - opt, ctrl, special = m.groups() - if opt is not None: - cssclass_plain = 'o' - elif ctrl is not None: - cssclass_plain = 'k' - elif special is not None: - cssclass_plain = 's' - links_plain = mklinks(cssclass_plain) - links_pipe = mklinks('l') - self._urls[tag] = Link( - links_plain[0], links_plain[1], - links_pipe[0], links_pipe[1], - filename) - - def maplink(self, tag, curr_filename, css_class=None): - links = self._urls.get(tag) - if links is not None: - if links.filename == curr_filename: - if css_class == 'l': - return links.link_pipe_same - else: - return links.link_plain_same - else: - if css_class == 'l': - return links.link_pipe_foreign - else: - return links.link_plain_foreign - elif css_class is not None: - return '' + html_escape[tag] + \ - '' - else: - return html_escape[tag] - - def to_html(self, filename, contents, encoding): - out = [] - - inexample = 0 - filename = str(filename) - is_help_txt = (filename == 'help.txt') - last = '' - for line in RE_NEWLINE.split(contents): - line = line.rstrip('\r\n') - line_tabs = line - line = line.expandtabs() - if last == 'h1': - out.extend(('')) # XXX - out.extend(('

', line.rstrip(), '

\n')) - out.extend(('
'))
-                last = ''
-                continue
-            if RE_HRULE.match(line):
-                # out.extend(('', line, '\n'))
-                last = 'h1'
-                continue
-            if inexample == 2:
-                if RE_EG_END.match(line):
-                    inexample = 0
-                    if line[0] == '<':
-                        line = line[1:]
-                else:
-                    out.extend(('', html_escape[line],
-                                '\n'))
-                    continue
-            if RE_EG_START.match(line_tabs):
-                inexample = 1
-                line = line[0:-1]
-            if RE_SECTION.match(line_tabs):
-                m = RE_SECTION.match(line)
-                out.extend((r'', m.group(0), r''))
-                line = line[m.end():]
-            lastpos = 0
-            for match in RE_TAGWORD.finditer(line):
-                pos = match.start()
-                if pos > lastpos:
-                    out.append(html_escape[line[lastpos:pos]])
-                lastpos = match.end()
-                header, graphic, pipeword, starword, command, opt, ctrl, \
-                    special, title, note, url, word = match.groups()
-                if pipeword is not None:
-                    out.append(self.maplink(pipeword, filename, 'l'))
-                elif starword is not None:
-                    out.extend(('', html_escape[starword], ''))
-                elif command is not None:
-                    out.extend(('', html_escape[command],
-                                ''))
-                elif opt is not None:
-                    out.append(self.maplink(opt, filename, 'o'))
-                elif ctrl is not None:
-                    out.append(self.maplink(ctrl, filename, 'k'))
-                elif special is not None:
-                    out.append(self.maplink(special, filename, 's'))
-                elif title is not None:
-                    out.extend(('', html_escape[title],
-                                ''))
-                elif note is not None:
-                    out.extend(('', html_escape[note],
-                                ''))
-                elif header is not None:
-                    out.extend(('', html_escape[header[:-1]],
-                                ''))
-                elif graphic is not None:
-                    out.append(html_escape[graphic[:-2]])
-                elif url is not None:
-                    out.extend(('' +
-                                html_escape[url], ''))
-                elif word is not None:
-                    out.append(self.maplink(word, filename))
-            if lastpos < len(line):
-                out.append(html_escape[line[lastpos:]])
-            out.append('\n')
-            if inexample == 1:
-                inexample = 2
-
-        header = []
-        header.append(HEAD.format(encoding=encoding, filename=filename))
-        header.append(HEAD_END)
-        if self._is_web_version and is_help_txt:
-            vers_note = VERSION_NOTE.replace('{version}', self._version) \
-                if self._version else ''
-            header.append(INTRO.replace('{vers-note}', vers_note))
-        if self._is_web_version:
-            header.append(SITENAVI_SEARCH)
-            sitenavi_footer = SITENAVI_WEB
-        else:
-            header.append(SITENAVI_PLAIN)
-            sitenavi_footer = SITENAVI_PLAIN
-        header.append(TEXTSTART)
-        return ''.join(chain(header, out, (FOOTER, sitenavi_footer, FOOTER2)))
-
-
-class HtmlEscCache(dict):
-    def __missing__(self, key):
-        r = key.replace('&', '&') \
-               .replace('<', '<') \
-               .replace('>', '>')
-        self[key] = r
-        return r
-
-
-html_escape = HtmlEscCache()
-
-
-def slurp(filename):
-    try:
-        with open(filename, encoding='UTF-8') as f:
-            return f.read(), 'UTF-8'
-    except UnicodeError:
-        # 'ISO-8859-1' ?
-        with open(filename, encoding='latin-1') as f:
-            return f.read(), 'latin-1'
-
-
-def usage():
-    return "usage: " + sys.argv[0] + " IN_DIR OUT_DIR [BASENAMES...]"
-
-
-def main():
-    if len(sys.argv) < 3:
-        sys.exit(usage())
-
-    in_dir = sys.argv[1]
-    out_dir = sys.argv[2]
-    basenames = sys.argv[3:]
-
-    print("Processing tags...")
-    h2h = VimH2H(slurp(os.path.join(in_dir, 'tags'))[0], is_web_version=False)
-
-    if len(basenames) == 0:
-        basenames = os.listdir(in_dir)
-
-    for basename in basenames:
-        if os.path.splitext(basename)[1] != '.txt' and basename != 'tags':
-            print("Ignoring " + basename)
-            continue
-        print("Processing " + basename + "...")
-        path = os.path.join(in_dir, basename)
-        text, encoding = slurp(path)
-        outpath = os.path.join(out_dir, basename + '.html')
-        of = open(outpath, 'w')
-        of.write(h2h.to_html(basename, text, encoding))
-        of.close()
-
-
-main()
-- 
cgit 


From 1b60b5ec94001f18b70dbebf6c232c33209f11b5 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" 
Date: Sun, 25 Sep 2022 13:45:41 +0200
Subject: fix(gen_vimdoc.py): handle missing luajit

---
 scripts/gen_vimdoc.py  | 14 ++++++++++----
 scripts/lua2dox_filter |  7 ++++++-
 2 files changed, 16 insertions(+), 5 deletions(-)

(limited to 'scripts')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 766c420c7d..306857ca6c 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -295,14 +295,16 @@ annotation_map = {
 # or if `cond()` is callable and returns True.
 def debug_this(o, cond=True):
     name = ''
+    if cond is False:
+        return
     if not isinstance(o, str):
         try:
             name = o.nodeName
             o = o.toprettyxml(indent='  ', newl='\n')
         except Exception:
             pass
-    if ((callable(cond) and cond())
-            or (not callable(cond) and cond)
+    if (cond is True
+            or (callable(cond) and cond())
             or (not callable(cond) and cond in o)):
         raise RuntimeError('xxx: {}\n{}'.format(name, o))
 
@@ -887,7 +889,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
 def fmt_doxygen_xml_as_vimhelp(filename, target):
     """Entrypoint for generating Vim :help from from Doxygen XML.
 
-    Returns 3 items:
+    Returns 2 items:
       1. Vim help text for functions found in `filename`.
       2. Vim help text for deprecated functions.
     """
@@ -1094,7 +1096,11 @@ def main(config, args):
                         fn_map_full.update(fn_map)
 
         if len(sections) == 0:
-            fail(f'no sections for target: {target}')
+            if target == 'lua':
+                fail(f'no sections for target: {target} (this usually means'
+                     + ' "luajit" was not found by scripts/lua2dox_filter)')
+            else:
+                fail(f'no sections for target: {target}')
         if len(sections) > len(CONFIG[target]['section_order']):
             raise RuntimeError(
                 'found new modules "{}"; update the "section_order" map'.format(
diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter
index 22484a807f..0b9f59b6ac 100755
--- a/scripts/lua2dox_filter
+++ b/scripts/lua2dox_filter
@@ -36,6 +36,10 @@ test_executable(){
 
 ##! \brief sets the lua interpreter
 set_lua(){
+	if test -z "${EXE}"; then
+		test_executable '.deps/usr/bin/luajit'
+	fi
+
 	if test -z "${EXE}"; then
 		test_executable 'luajit'
 	fi
@@ -73,7 +77,8 @@ do_readlink(){
 set_lua
 if test -z "${EXE}"
 then
-	echo "no lua interpreter available"
+	echo "no lua interpreter found"
+	exit 1
 else
 	BASENAME=`basename "$0"`
 	do_readlink "$0"
-- 
cgit 


From 7b4c49888a97c21f346b8337330fbc8e196b9cf8 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" 
Date: Fri, 23 Sep 2022 14:26:59 +0200
Subject: feat(gen_help_html.lua): put commit-id in footer

---
 scripts/gen_help_html.lua | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'scripts')

diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua
index d79ece53f3..6e297814b8 100644
--- a/scripts/gen_help_html.lua
+++ b/scripts/gen_help_html.lua
@@ -16,7 +16,7 @@
 --
 -- NOTES:
 --   * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too
---   slow (~1 min) to run in per-commit CI.
+--     slow (~1 min) to run in per-commit CI.
 --   * visit_node() is the core function used by gen() to traverse the document tree and produce HTML.
 --   * visit_validate() is the core function used by validate().
 --   * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout.
@@ -50,7 +50,6 @@ local tagmap = nil
 local helpfiles = nil
 local invalid_tags = {}
 
-local commit = '?'
 local api = vim.api
 local M = {}
 
@@ -477,7 +476,7 @@ end
 -- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace)
 --
 -- @returns html, stats
-local function gen_one(fname, to_fname, old)
+local function gen_one(fname, to_fname, old, commit)
   local stats = {
     noise_lines = {},
     parse_errors = {},
@@ -604,7 +603,7 @@ local function gen_one(fname, to_fname, old)
   

     

       

-        Generated on %s from {%s}
+        Generated at %s from %s
       

       

       parse_errors: %d %s | noise_lines: %d
@@ -612,7 +611,7 @@ local function gen_one(fname, to_fname, old)
     

   

   ]]):format(
-    os.date('%Y-%m-%d %H:%M:%S'), commit, #stats.parse_errors, bug_link,
+    os.date('%Y-%m-%d %H:%M'), commit, commit:sub(1, 7), #stats.parse_errors, bug_link,
     html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines)
 
   html = ('%s%s%s
\n%s\n\n'):format( @@ -763,11 +762,12 @@ end --- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} --- --- @returns info dict -function M.gen(help_dir, to_dir, include) +function M.gen(help_dir, to_dir, include, commit) vim.validate{ help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, to_dir={to_dir, 's'}, include={include, 't', true}, + commit={commit, 's', true}, } local err_count = 0 @@ -781,7 +781,7 @@ function M.gen(help_dir, to_dir, include) for _, f in ipairs(helpfiles) do local helpfile = vim.fs.basename(f) local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) - local html, stats = gen_one(f, to_fname, not new_layout[helpfile]) + local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?') tofile(to_fname, html) print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) err_count = err_count + #stats.parse_errors -- cgit From 16336c486ecb5a60e85a870904316308c7d7fc3f Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 25 Sep 2022 02:20:47 +0200 Subject: feat(gen_help_html.lua): adapt to new parser - adapt to parser changes from https://github.com/vigoux/tree-sitter-vimdoc/pull/16 - numerous other generator improvements --- scripts/gen_help_html.lua | 467 +++++++++++++++++++++++++++++----------------- 1 file changed, 295 insertions(+), 172 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 6e297814b8..fdbf5f605a 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -1,5 +1,7 @@ -- Converts Vim :help files to HTML. Validates |tag| links and document syntax (parser errors). -- +-- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). +-- -- USAGE (GENERATE HTML): -- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). -- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" @@ -20,37 +22,19 @@ -- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. -- * visit_validate() is the core function used by validate(). -- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. --- --- parser bugs: --- * Should NOT be code_block: --- tab:xy The 'x' is always used, then 'y' as many times as will --- fit. Thus "tab:>-" displays: --- > --- >- --- >-- --- etc. --- --- tab:xyz The 'z' is always used, then 'x' is prepended, and --- then 'y' is used as many times as will fit. Thus --- "tab:<->" displays: --- > --- <> --- <-> --- <--> --- etc. --- * Should NOT be a "headline". Perhaps a "table" (or just "line"). --- expr5 and expr6 *expr5* *expr6* --- --------------- --- expr6 + expr6 Number addition, |List| or |Blob| concatenation *expr-+* --- expr6 - expr6 Number subtraction *expr--* --- expr6 . expr6 String concatenation *expr-.* --- expr6 .. expr6 String concatenation *expr-..* local tagmap = nil local helpfiles = nil -local invalid_tags = {} +local invalid_links = {} +local invalid_urls = {} +local invalid_spelling = {} +local spell_dict = { + Neovim = 'Nvim', + NeoVim = 'Nvim', + neovim = 'Nvim', + lua = 'Lua', +} -local api = vim.api local M = {} -- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). @@ -59,16 +43,31 @@ local new_layout = { ['api.txt'] = true, ['channel.txt'] = true, ['develop.txt'] = true, + ['luaref.txt'] = true, ['nvim.txt'] = true, ['pi_health.txt'] = true, ['provider.txt'] = true, ['ui.txt'] = true, } --- TODO: treesitter gets stuck on these files... -local exclude = { - ['filetype.txt'] = true, - ['usr_24.txt'] = true, +-- TODO: These known invalid |links| require an update to the relevant docs. +local exclude_invalid = { + ["'previewpopup'"] = "quickref.txt", + ["'pvp'"] = "quickref.txt", + ["'string'"] = "eval.txt", + Query = "treesitter.txt", + ["eq?"] = "treesitter.txt", + ["lsp-request"] = "lsp.txt", + matchit = "vim_diff.txt", + ["matchit.txt"] = "help.txt", + ["set!"] = "treesitter.txt", + ["v:_null_blob"] = "builtin.txt", + ["v:_null_dict"] = "builtin.txt", + ["v:_null_list"] = "builtin.txt", + ["v:_null_string"] = "builtin.txt", + ["vim.lsp.buf_request()"] = "lsp.txt", + ["vim.lsp.util.get_progress_messages()"] = "lsp.txt", + ["vim.treesitter.start()"] = "treesitter.txt" } local function tofile(fname, text) @@ -82,10 +81,6 @@ local function tofile(fname, text) end local function html_esc(s) - if s:find(']local%-additions[*<]') + ) then + -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + table.insert(noise_lines or {}, line) + return true + end + return false end --- Checks if a given line is a "noise" line that doesn't look good in HTML form. -local function is_noise(line) - return ( - line:find('Type .*gO.* to see the table of contents') - -- Title line of traditional :help pages. - -- Example: "NVIM REFERENCE MANUAL by ..." - or line:find('^%s*N?VIM REFERENCE MANUAL') - -- First line of traditional :help pages. - -- Example: "*api.txt* Nvim" - or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') - -- modeline - -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" - or line:find('^%s*vi[m]%:.*ft=help') - or line:find('^%s*vi[m]%:.*filetype=help') - ) -end - --- Creates a github issue URL at vigoux/tree-sitter-vimdoc with prefilled content. +-- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. local function get_bug_url_vimdoc(fname, to_fname, sample_text) local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ('https://github.com/vigoux/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + local bug_url = ('https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' ..vim.fs.basename(fname) ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' ..this_url @@ -237,31 +236,57 @@ local function getbuflinestr(node, bufnr, offset) return table.concat(lines, '\n') end --- Gets the whitespace just before `node` from the raw buffer text. --- Needed for preformatted `old` lines. -local function getws(node, bufnr) - local line1, c1, line2, _ = node:range() - local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] - local text_before = raw:sub(1, c1) - local leading_ws = text_before:match('%s+$') or '' - return leading_ws -end - -local function get_tagname(node, bufnr, link) - local node_name = (node.named and node:named()) and node:type() or nil +local function get_tagname(node, bufnr) local node_text = vim.treesitter.get_node_text(node, bufnr) - local tag = ((node_name == 'option' and node_text) - or (link and node_text:gsub('^|', ''):gsub('|$', '') or node_text:gsub('^%*', ''):gsub('%*$', ''))) - local helpfile = tag and vim.fs.basename(tagmap[tag]) or nil -- "api.txt" - local helppage = get_helppage(helpfile) -- "api.html" + local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(node_text) or node_text + local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" return helppage, tag end +-- Returns true if the given invalid tagname is a false positive. +local function ignore_invalid(s) + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + return not not (s:find('===') or exclude_invalid[s]) +end + +local function ignore_parse_error(s) + -- Ignore parse errors for unclosed codespan/optionlink/tag. + -- This is common in vimdocs and is treated as plaintext by :help. + return s:find("^[`'|*]") +end + +local function has_ancestor(node, ancestor_name) + local p = node + while true do + p = p:parent() + if not p or p:type() == 'help_file' then + break + elseif p:type() == ancestor_name then + return true + end + end + return false +end + +local function validate_link(node, bufnr, fname) + local helppage, tagname = get_tagname(node:child(1), bufnr, true) + if not has_ancestor(node, 'column_heading') and not node:has_error() and not tagmap[tagname] and not ignore_invalid(tagname) then + invalid_links[tagname] = vim.fs.basename(fname) + end + return helppage, tagname +end + -- Traverses the tree at `root` and checks that |tag| links point to valid helptags. local function visit_validate(root, level, lang_tree, opt, stats) level = level or 0 local node_name = (root.named and root:named()) and root:type() or nil local toplevel = level < 1 + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) + end + local text = trim(node_text()) if root:child_count() > 0 then for node, _ in root:iter_children() do @@ -272,14 +297,26 @@ local function visit_validate(root, level, lang_tree, opt, stats) end if node_name == 'ERROR' then + if ignore_parse_error(text) then + return + end -- Store the raw text to give context to the bug report. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) - elseif node_name == 'hotlink' or node_name == 'option' then - local _, tagname = get_tagname(root, opt.buf, true) - if not root:has_error() and not tagmap[tagname] then - invalid_tags[tagname] = vim.fs.basename(opt.fname) + elseif node_name == 'word' or node_name == 'uppercase_name' then + if spell_dict[text] then + if not invalid_spelling[text] then + invalid_spelling[text] = { vim.fs.basename(opt.fname) } + else + table.insert(invalid_spelling[text], vim.fs.basename(opt.fname)) + end end + elseif node_name == 'url' then + if text:find('http%:') then + invalid_urls[text] = vim.fs.basename(opt.fname) + end + elseif node_name == 'taglink' or node_name == 'optionlink' then + local _, _ = validate_link(root, opt.buf, opt.fname) end end @@ -296,19 +333,18 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local parent = root:parent() and root:parent():type() or nil local text = '' local toplevel = level < 1 - local function node_text() - return vim.treesitter.get_node_text(root, opt.buf) + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) end - if root:child_count() == 0 then + if root:child_count() == 0 or node_name == 'ERROR' then text = node_text() else -- Process children and join them with whitespace. for node, _ in root:iter_children() do if node:named() then local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) - local ws = r == '' and '' or ((opt.old and (node:type() == 'word' or not node:named())) and getws(node, opt.buf) or ' ') - text = string.format('%s%s%s', text, ws, r) + text = string.format('%s%s', text, r) end end end @@ -316,82 +352,112 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if node_name == 'help_file' then -- root node return text + elseif node_name == 'url' then + return ('%s'):format(trimmed, trimmed) elseif node_name == 'word' or node_name == 'uppercase_name' then - if parent == 'headline' then - -- Start a new heading item, or update the current one. - local n = (prev == nil or #headings == 0) and #headings + 1 or #headings - headings[n] = string.format('%s%s', headings[n] and headings[n]..' ' or '', text) - end - return html_esc(text) - elseif node_name == 'headline' then - return ('

%s

\n'):format(to_heading_tag(headings[#headings]), text) + elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then + if is_noise(text, stats.noise_lines) then + return '' -- Discard common "noise" lines. + end + -- Remove "===" and tags from ToC text. + local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) + if node_name == 'h1' or #headings == 0 then + table.insert(headings, { name = hname, subheadings = {}, }) + else + table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, }) + end + local el = node_name == 'h1' and 'h2' or 'h3' + return ('<%s class="help-heading">%s\n'):format(to_heading_tag(hname), el, text, el) elseif node_name == 'column_heading' or node_name == 'column_name' then - return ('

%s

\n'):format(trimmed) - elseif node_name == 'line' then - -- TODO: remove these "sibling inspection" hacks once the parser provides structured info - -- about paragraphs and listitems: https://github.com/vigoux/tree-sitter-vimdoc/issues/12 - local next_text = root:next_sibling() and vim.treesitter.get_node_text(root:next_sibling(), opt.buf) or '' - local li = startswith_bullet(text) -- Listitem? - local next_li = startswith_bullet(next_text) -- Next is listitem? - -- Close the paragraph/listitem if the next sibling is not a line. - local close = (next_ ~= 'line' or next_li or is_blank(next_text)) and '
\n' or '' - - -- HACK: discard common "noise" lines. - if is_noise(text) then - table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) - return (opt.old or prev ~= 'line') and '' or close + if root:has_error() then + return text + end + return ('
%s
'):format(trimmed) + elseif node_name == 'block' then + if is_blank(text) then + return '' end - if opt.old then - -- XXX: Treat old docs as preformatted. Until those docs are "fixed" or we get better info - -- from tree-sitter-vimdoc, this avoids broken layout for legacy docs. - return ('
%s
\n'):format(text) + -- XXX: Treat old docs as preformatted; random indentation is used for layout there. + return ('
%s
\n'):format(text) end - - if li then - return string.format('
%s%s', trim_bullet(expandtabs(text)), close) + return string.format('
\n%s\n
\n', text) + elseif node_name == 'line' then + local sib = root:prev_sibling() + local sib_last = sib and sib:named_child(sib:named_child_count() - 1) + local in_li = false + + -- XXX: parser bug: (codeblock) without terminating "<" consumes first char of the next (line). Recover it here. + local recovered = (sib_last and sib_last:type() == 'codeblock') and node_text(root:prev_sibling()):sub(-1) or '' + recovered = recovered == '<' and '' or html_esc(recovered) + + -- XXX: see if we are currently "in" a listitem. + while sib ~= nil and not in_li do + in_li = (sib:type() == 'line_li') + sib = sib:prev_sibling() end - if prev ~= 'line' then -- Start a new paragraph. - return string.format('
%s%s', expandtabs(text), close) + + -- Close the current listitem. + local close = (in_li and next_ ~= 'line') and '
' or '' + + if is_blank(text) or is_noise(text, stats.noise_lines) then + return close -- Discard common "noise" lines. end - -- Continue in the current paragraph/listitem. - return string.format('%s%s', expandtabs(text), close) - elseif node_name == 'hotlink' or node_name == 'option' then - local helppage, tagname = get_tagname(root, opt.buf, true) - if not root:has_error() and not tagmap[tagname] then - invalid_tags[tagname] = vim.fs.basename(opt.fname) + local div = (root:child(0) and root:child(0):type() == 'column_heading') or close ~= '' + return string.format('%s%s%s%s', recovered, div and trim(text) or text, div and '' or '\n', close) + elseif node_name == 'line_li' then + -- Close the listitem immediately if the next sibling is not a line. + local close = (next_ ~= 'line') and '
' or '' + return string.format('
%s %s', trim_bullet(text), close) + elseif node_name == 'taglink' or node_name == 'optionlink' then + if root:has_error() then + return text + end + local helppage, tagname = validate_link(root, opt.buf, opt.fname) + return (' %s'):format(helppage, url_encode(tagname), html_esc(tagname)) + elseif node_name == 'codespan' then + if root:has_error() then + return text end - return ('%s'):format(helppage, url_encode(tagname), html_esc(tagname)) - elseif node_name == 'backtick' then - return ('%s'):format(html_esc(text)) + return (' %s'):format(text) elseif node_name == 'argument' then - return ('{%s}'):format(html_esc(trimmed)) - elseif node_name == 'code_block' then - return ('
\n%s
\n'):format(html_esc(trim_indent(trim_gt_lt(text)))) + return (' {%s}'):format(trimmed) + elseif node_name == 'codeblock' then + return ('
%s
'):format(html_esc(trim_indent(trim_gt_lt(text)))) elseif node_name == 'tag' then -- anchor - local _, tagname = get_tagname(root, opt.buf, false) - local s = ('%s'):format(url_encode(tagname), trimmed) - if parent == 'headline' and prev ~= 'tag' then + if root:has_error() then + return text + end + local in_heading = (parent == 'h1' or parent == 'h2') + local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' + local tagname = node_text(root:child(1)) + if vim.tbl_count(stats.first_tags) < 2 then + -- First 2 tags in the doc will be anchored at the main heading. + table.insert(stats.first_tags, tagname) + return '' + end + local s = (' %s'):format(url_encode(tagname), cssclass, trimmed) + if in_heading and prev ~= 'tag' then -- Start the container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. --

foo bartag1 tag2

return string.format('%s', s) - elseif parent == 'headline' and next_ == nil then + elseif in_heading and next_ == nil then -- End the container for tags in a heading. return string.format('%s', s) end return s elseif node_name == 'ERROR' then + if ignore_parse_error(trimmed) then + return text + end + -- Store the raw text to give context to the bug report. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) - if prev == 'ERROR' then - -- Avoid trashing the text with cascading errors. - return trimmed, ('parse-error:"%s"'):format(node_text()) - end - return ('%s'):format( + return ('%s'):format( get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) else -- Unknown token. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' @@ -406,8 +472,7 @@ local function get_helpfiles(include) for f, type in vim.fs.dir(dir) do if (vim.endswith(f, '.txt') and type == 'file' - and (not include or vim.tbl_contains(include, f)) - and (not exclude[f])) then + and (not include or vim.tbl_contains(include, f))) then local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') table.insert(rv, fullpath) end @@ -430,6 +495,13 @@ local function get_helptags(help_dir) return m end +-- Use the help.so parser defined in the build, not whatever happens to be installed on the system. +local function ensure_runtimepath() + if not vim.o.runtimepath:find('build/lib/nvim/') then + vim.cmd[[set runtimepath^=./build/lib/nvim/]] + end +end + -- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents. -- -- @returns lang_tree, bufnr @@ -437,7 +509,7 @@ local function parse_buf(fname) local buf if type(fname) == 'string' then vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. - buf = api.nvim_get_current_buf() + buf = vim.api.nvim_get_current_buf() else buf = fname vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. @@ -451,10 +523,9 @@ end -- - checks that |tag| links point to valid helptags. -- - recursively counts parse errors ("ERROR" nodes) -- --- @returns { invalid_tags: number, parse_errors: number } +-- @returns { invalid_links: number, parse_errors: number } local function validate_one(fname) local stats = { - invalid_tags = {}, parse_errors = {}, } local lang_tree, buf = parse_buf(fname) @@ -463,10 +534,7 @@ local function validate_one(fname) end lang_tree:destroy() vim.cmd.close() - return { - invalid_tags = invalid_tags, - parse_errors = stats.parse_errors, - } + return stats end -- Generates HTML from one :help file `fname` and writes the result to `to_fname`. @@ -480,9 +548,10 @@ local function gen_one(fname, to_fname, old, commit) local stats = { noise_lines = {}, parse_errors = {}, + first_tags = {}, -- Track the first few tags in doc. } local lang_tree, buf = parse_buf(fname) - local headings = {} -- Headings (for ToC). + local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. local title = to_titlecase(basename_noext(fname)) local html = ([[ @@ -555,7 +624,12 @@ local function gen_one(fname, to_fname, old, commit) ]] - local main = ([[ + local main = '' + for _, tree in ipairs(lang_tree:trees()) do + main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + end + + main = ([[
' or '' - return string.format('
%s %s', trim_bullet(text), close) + return string.format('
%s
', margin, text) elseif node_name == 'taglink' or node_name == 'optionlink' then if root:has_error() then return text @@ -429,7 +438,10 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) elseif node_name == 'codeblock' then - return ('
%s
'):format(html_esc(trim_indent(trim_gt_lt(text)))) + if is_blank(text) then + return '' + end + return ('
%s
'):format(html_esc(trim(trim_indent(text), 2))) elseif node_name == 'tag' then -- anchor if root:has_error() then return text @@ -630,7 +642,9 @@ local function gen_one(fname, to_fname, old, commit) local main = '' for _, tree in ipairs(lang_tree:trees()) do - main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + main = main .. (visit_node(tree:root(), 0, tree, headings, + { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1, }, + stats)) end main = ([[ @@ -718,6 +732,17 @@ local function gen_css(fname) position: fixed; left: 67%; } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + } + @media (max-width: 40em) { + .golden-grid { + /* Disable grid for narrow viewport (mobile phone). */ + display: block; + } } @media (prefers-color-scheme: dark) { :root { @@ -831,11 +856,6 @@ local function gen_css(fname) color: gray; font-size: smaller; } - .golden-grid { - display: grid; - grid-template-columns: 65% auto; - grid-gap: 1em; - } ]] tofile(fname, css) end @@ -869,6 +889,22 @@ function M._test() eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + local fixed_url, removed_chars = fix_url('https://example.com).') + eq('https://example.com', fixed_url) + eq(').', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.)') + eq('https://example.com.', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.') + eq('https://example.com', fixed_url) + eq('.', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com)') + eq('https://example.com', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com') + eq('https://example.com', fixed_url) + eq('', removed_chars) + print('all tests passed') end -- cgit From 6abb48105135ce3ae7eda22334f8104c5ddf20ce Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 4 Oct 2022 15:15:06 +0200 Subject: fix(docs): missing "(" in :help HTML Problem: Since https://github.com/neovim/tree-sitter-vimdoc/commit/eba7b5b646546d9fed9b40b2c72b9cc0048f1dfa any opening paren and its leading whitespace " (" are missing in the generated HTML. Example: Use ":qa!" (careful, all changes are lost!). ^^missing Position the cursor on a tag (e.g. bars) and hit CTRL-]. ^^missing Solution: The main recursive loop only processes named children, so check named_child_count() instead of child_count(). Then anonymous nodes won't get lost. --- scripts/gen_help_html.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 3141ff3cdf..ccfa2e567e 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -350,7 +350,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return node_text():match('^%s+') or '' end - if root:child_count() == 0 or node_name == 'ERROR' then + if root:named_child_count() == 0 or node_name == 'ERROR' then text = node_text() else -- Process children and join them with whitespace. -- cgit From 18afacee1d98b9987391b8bdef08282fb156fa88 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Wed, 5 Oct 2022 08:15:55 -0400 Subject: feat(docs): format parameters as a list #20485 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: The {foo} parameters listed in `:help api` and similar generated docs, are intended to be a "list" but they aren't prefixed with a list symbol. This prevents parsers from understanding the list, which forces generators like `gen_help_html.lua` to use hard-wrapped/preformatted layout instead of a soft-wrapped "flow" layout. Solution: Modify gen_vimdoc.py to prefix {foo} parameters with a "•" symbol. --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 306857ca6c..2612260226 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -673,7 +673,7 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='', max_name_len = max_name(m.keys()) + 4 out = '' for name, desc in m.items(): - name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) out += '{}{}\n'.format(name, desc) return out.rstrip() -- cgit From f7b175e049db9262a45ee1c5eb41a38bd5b8ac38 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 6 Oct 2022 09:16:00 -0400 Subject: fix(docs-html): keycodes, taglinks, column_heading #20498 Problem: - Docs HTML: "foo ~" headings (column_heading) are not aligned with their table columns/contents because the leading whitespace is not emitted. - taglinks starting with hyphen like |-x| were not recognized. - keycodes like `` and `CTRL-x` were not recognized. - ToC is not scrollable. Solution: - Add ws() to the column_heading case. - Update help parser to latest version - supports `keycode` - fixes for taglink, argument - Update .toc CSS. https://github.com/neovim/neovim.github.io/issues/297 fix https://github.com/neovim/neovim.github.io/issues/297 --- scripts/gen_help_html.lua | 54 +++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 21 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index ccfa2e567e..15269ce175 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -208,7 +208,7 @@ local function get_helppage(f) return 'index.html' end - return f:gsub('%.txt$', '.html') + return (f:gsub('%.txt$', '.html')) end -- Counts leading spaces (tab=8) to decide the indent size of multiline text. @@ -255,9 +255,13 @@ end -- Returns true if the given invalid tagname is a false positive. local function ignore_invalid(s) - -- Strings like |~/====| appear in various places and the parser thinks they are links, but they - -- are just table borders. - return not not (s:find('===') or exclude_invalid[s]) + return not not ( + exclude_invalid[s] + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + or s:find('===') + or s:find('---') + ) end local function ignore_parse_error(s) @@ -281,10 +285,14 @@ end local function validate_link(node, bufnr, fname) local helppage, tagname = get_tagname(node:child(1), bufnr) - if not has_ancestor(node, 'column_heading') and not node:has_error() and not tagmap[tagname] and not ignore_invalid(tagname) then - invalid_links[tagname] = vim.fs.basename(fname) + local ignored = false + if not tagmap[tagname] then + ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname) + if not ignored then + invalid_links[tagname] = vim.fs.basename(fname) + end end - return helppage, tagname + return helppage, tagname, ignored end -- Traverses the tree at `root` and checks that |tag| links point to valid helptags. @@ -325,7 +333,7 @@ local function visit_validate(root, level, lang_tree, opt, stats) invalid_urls[text] = vim.fs.basename(opt.fname) end elseif node_name == 'taglink' or node_name == 'optionlink' then - local _, _ = validate_link(root, opt.buf, opt.fname) + local _, _, _ = validate_link(root, opt.buf, opt.fname) end end @@ -341,7 +349,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) -- Parent kind (string). local parent = root:parent() and root:parent():type() or nil local text = '' - local toplevel = level < 1 + local trimmed local function node_text(node) return vim.treesitter.get_node_text(node or root, opt.buf) end @@ -352,6 +360,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:named_child_count() == 0 or node_name == 'ERROR' then text = node_text() + trimmed = html_esc(trim(text)) + text = html_esc(text) else -- Process children and join them with whitespace. for node, _ in root:iter_children() do @@ -360,8 +370,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) text = string.format('%s%s', text, r) end end + trimmed = trim(text) end - local trimmed = trim(text) if node_name == 'help_file' then -- root node return text @@ -369,7 +379,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local fixed_url, removed_chars = fix_url(trimmed) return ('%s%s%s'):format(ws(), fixed_url, fixed_url, removed_chars) elseif node_name == 'word' or node_name == 'uppercase_name' then - return html_esc(text) + return text elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then if is_noise(text, stats.noise_lines) then return '' -- Discard common "noise" lines. @@ -387,7 +397,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - return ('
%s
'):format(trimmed) + return ('
%s%s
'):format(ws(), trimmed) elseif node_name == 'block' then if is_blank(text) then return '' @@ -425,28 +435,28 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return string.format('
%s
', margin, text) elseif node_name == 'taglink' or node_name == 'optionlink' then - if root:has_error() then + local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname) + if ignored then return text end - local helppage, tagname = validate_link(root, opt.buf, opt.fname) return ('%s%s'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) - elseif node_name == 'codespan' then + elseif vim.tbl_contains({'codespan', 'keycode'}, node_name) then if root:has_error() then return text end - return ('%s%s'):format(ws(), text) + return ('%s%s'):format(ws(), trimmed) elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) elseif node_name == 'codeblock' then if is_blank(text) then return '' end - return ('
%s
'):format(html_esc(trim(trim_indent(text), 2))) + return ('
%s
'):format(trim(trim_indent(text), 2)) elseif node_name == 'tag' then -- anchor if root:has_error() then return text end - local in_heading = (parent == 'h1' or parent == 'h2') + local in_heading = vim.tbl_count({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' local tagname = node_text(root:child(1)) if vim.tbl_count(stats.first_tags) < 2 then @@ -471,12 +481,12 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end -- Store the raw text to give context to the bug report. - local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) return ('%s'):format( get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) else -- Unknown token. - local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' return ('%s'):format( node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) end @@ -751,6 +761,8 @@ local function gen_css(fname) } .toc { /* max-width: 12rem; */ + height: 95%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ } .toc > div { text-overflow: ellipsis; @@ -809,7 +821,7 @@ local function gen_css(fname) .help-tag-right { color: var(--tag-color); } - h1 .help-tag, h2 .help-tag { + h1 .help-tag, h2 .help-tag, h3 .help-tag { font-size: smaller; } .help-heading { -- cgit From 453fffde16fe25b818a77eefc7e8e138a00aaf61 Mon Sep 17 00:00:00 2001 From: Folke Lemaitre Date: Wed, 28 Sep 2022 08:56:47 +0200 Subject: feat: added support for @generic to lua2dox.lua --- scripts/lua2dox.lua | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 6a206066b8..7d8e393cc9 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -387,10 +387,13 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local state = '' -- luacheck: ignore 231 variable is set but never accessed. local offset = 0 + local generic = {} + local l = 0 while not (inStream:eof()) do line = string_trim(inStream:getLine()) -- TCore_Debug_show_var('inStream',inStream) -- TCore_Debug_show_var('line',line ) + l = l + 1 if string.sub(line,1,2) == '--' then -- it's a comment -- Allow people to write style similar to EmmyLua (since they are basically the same) -- instead of silently skipping things that start with --- @@ -406,11 +409,25 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic_split = string_split(magic, ' ') + if magic_split[1] == "generic" then + local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") + if generic_type == "" then + generic_type = "any" + end + generic[generic_name] = generic_type + end + local type_index = 2 if magic_split[1] == 'param' then type_index = type_index + 1 end + if magic_split[type_index] then + for k, v in pairs(generic) do + magic_split[type_index] = magic_split[type_index]:gsub(k, v) + end + end + if magic_split[type_index] == 'number' or magic_split[type_index] == 'number|nil' or magic_split[type_index] == 'string' or @@ -426,8 +443,10 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) end magic = table.concat(magic_split, ' ') - outStream:writeln('/// @' .. magic) - fn_magic = checkComment4fn(fn_magic,magic) + if magic_split[1] ~= "generic" then + outStream:writeln('/// @' .. magic) + fn_magic = checkComment4fn(fn_magic,magic) + end elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment local comment = string.sub(line,4) outStream:writeln('/// '.. comment) @@ -467,6 +486,7 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) fn_magic = nil end elseif string.find(line, '^function') or string.find(line, '^local%s+function') then + generic = {} state = 'in_function' -- it's a function local pos_fn = string.find(line,'function') -- function -- cgit From 24a1c7f556bba35a9c31c2fdd19cf4b8c00a4395 Mon Sep 17 00:00:00 2001 From: Folke Lemaitre Date: Wed, 28 Sep 2022 09:11:21 +0200 Subject: feat: added support for optional params to lua2dox --- scripts/lua2dox.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 7d8e393cc9..d93bb6080f 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -410,7 +410,7 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic_split = string_split(magic, ' ') if magic_split[1] == "generic" then - local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") + local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") if generic_type == "" then generic_type = "any" end @@ -420,6 +420,10 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local type_index = 2 if magic_split[1] == 'param' then type_index = type_index + 1 + if magic_split[type_index] and magic_split[2]:find("%?$") then + magic_split[type_index] = magic_split[type_index] .. "|nil" + magic_split[2] = magic_split[2]:sub(1, -2) + end end if magic_split[type_index] then -- cgit From 1da7b4eb699fb04cc97dec389470fd0fbd64091d Mon Sep 17 00:00:00 2001 From: Folke Lemaitre Date: Wed, 28 Sep 2022 13:22:08 +0200 Subject: feat: added support for specifying types for lua2dox --- scripts/lua2dox.lua | 69 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 28 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index d93bb6080f..674b42a699 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -159,6 +159,7 @@ end --! \param Str --! \param Pattern --! \returns table of string fragments +---@return string[] local function string_split(Str, Pattern) local splitStr = {} local fpat = "(.-)" .. Pattern @@ -369,6 +370,9 @@ local function checkComment4fn(Fn_magic,MagicLines) return fn_magic end + +local types = {"number", "string", "table", "list", "boolean", "function"} + --! \brief run the filter function TLua2DoX_filter.readfile(this,AppStamp,Filename) local inStream = TStream_Read() @@ -408,6 +412,19 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic = string.sub(line, 4 + offset) local magic_split = string_split(magic, ' ') + if magic_split[1] == 'param' then + for _, type in ipairs(types) do + magic = magic:gsub("^param%s+([a-zA-Z_?]+)%s+.*%((" .. type .. ")%)", "param %1 %2" ) + magic = magic:gsub("^param%s+([a-zA-Z_?]+)%s+.*%((" .. type .. "|nil)%)", "param %1 %2" ) + end + magic_split = string_split(magic, ' ') + elseif magic_split[1] == 'return' then + for _, type in ipairs(types) do + magic = magic:gsub("^return%s+.*%((" .. type .. ")%)", "return %1" ) + magic = magic:gsub("^return%s+.*%((" .. type .. "|nil)%)", "return %1" ) + end + magic_split = string_split(magic, ' ') + end if magic_split[1] == "generic" then local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") @@ -415,39 +432,35 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) generic_type = "any" end generic[generic_name] = generic_type - end - - local type_index = 2 - if magic_split[1] == 'param' then - type_index = type_index + 1 - if magic_split[type_index] and magic_split[2]:find("%?$") then - magic_split[type_index] = magic_split[type_index] .. "|nil" - magic_split[2] = magic_split[2]:sub(1, -2) + else + local type_index = 2 + if magic_split[1] == 'param' then + type_index = type_index + 1 end - end - if magic_split[type_index] then - for k, v in pairs(generic) do - magic_split[type_index] = magic_split[type_index]:gsub(k, v) + if magic_split[type_index] then + -- fix optional parameters + if magic_split[type_index] and magic_split[2]:find("%?$") then + if not magic_split[type_index]:find("nil") then + magic_split[type_index] = magic_split[type_index] .. "|nil" + end + magic_split[2] = magic_split[2]:sub(1, -2) + end + -- replace generic types + if magic_split[type_index] then + for k, v in pairs(generic) do + magic_split[type_index] = magic_split[type_index]:gsub(k, v) + end + end + -- surround some types by () + for _, type in ipairs(types) do + magic_split[type_index] = magic_split[type_index]:gsub("^(" .. type .. "|nil):?$", "(%1)") + magic_split[type_index] = magic_split[type_index]:gsub("^(" .. type .. "):?$", "(%1)") + end end - end - if magic_split[type_index] == 'number' or - magic_split[type_index] == 'number|nil' or - magic_split[type_index] == 'string' or - magic_split[type_index] == 'string|nil' or - magic_split[type_index] == 'table' or - magic_split[type_index] == 'table|nil' or - magic_split[type_index] == 'boolean' or - magic_split[type_index] == 'boolean|nil' or - magic_split[type_index] == 'function' or - magic_split[type_index] == 'function|nil' - then - magic_split[type_index] = '(' .. magic_split[type_index] .. ')' - end - magic = table.concat(magic_split, ' ') + magic = table.concat(magic_split, ' ') - if magic_split[1] ~= "generic" then outStream:writeln('/// @' .. magic) fn_magic = checkComment4fn(fn_magic,magic) end -- cgit From 93117b358778487d85665035b6a95976eee70d9c Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Sat, 8 Oct 2022 17:49:09 +0200 Subject: docs(news): add news.txt and link from README (#20426) --- scripts/gen_help_html.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 15269ce175..540caa2ae3 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -44,6 +44,7 @@ local new_layout = { ['channel.txt'] = true, ['develop.txt'] = true, ['luaref.txt'] = true, + ['news.txt'] = true, ['nvim.txt'] = true, ['pi_health.txt'] = true, ['provider.txt'] = true, -- cgit From 09dffb9db7d16496e55e86f78ab60241533d86f6 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 9 Oct 2022 08:21:52 -0400 Subject: docs: various #12823 - increase python line-length limit from 88 => 100. - gen_help_html: fix bug in "tag" case (tbl_count => tbl_contains) ref #15632 fix #18215 fix #18479 fix #20527 fix #20532 Co-authored-by: Ben Weedon --- scripts/bump-deps.sh | 6 +++++- scripts/gen_help_html.lua | 14 +++++++++----- scripts/gen_vimdoc.py | 6 ++---- 3 files changed, 16 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/bump-deps.sh b/scripts/bump-deps.sh index 85c7f72700..e725608b39 100755 --- a/scripts/bump-deps.sh +++ b/scripts/bump-deps.sh @@ -17,9 +17,13 @@ BASENAME="$(basename "${0}")" readonly BASENAME usage() { - echo "Bump Neovim dependencies" + echo "Bump Nvim dependencies" echo echo "Usage: ${BASENAME} [ -h | --pr | --branch= | --dep= ]" + echo " Update a dependency:" + echo " ./scripts/bump-deps.sh --dep Luv --version 1.43.0-0" + echo " Create a PR:" + echo " ./scripts/bump-deps.sh --pr" echo echo "Options:" echo " -h show this message and exit." diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 540caa2ae3..13601b91f5 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -33,6 +33,7 @@ local spell_dict = { NeoVim = 'Nvim', neovim = 'Nvim', lua = 'Lua', + VimL = 'Vimscript', } local M = {} @@ -42,7 +43,9 @@ local M = {} local new_layout = { ['api.txt'] = true, ['channel.txt'] = true, + ['deprecated.txt'] = true, ['develop.txt'] = true, + ['lua.txt'] = true, ['luaref.txt'] = true, ['news.txt'] = true, ['nvim.txt'] = true, @@ -158,8 +161,8 @@ local function is_noise(line, noise_lines) or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') -- modeline -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" - or line:find('^%s*vi[m]%:.*ft=help') - or line:find('^%s*vi[m]%:.*filetype=help') + or line:find('^%s*vim?%:.*ft=help') + or line:find('^%s*vim?%:.*filetype=help') or line:find('[*>]local%-additions[*<]') ) then -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) @@ -457,7 +460,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - local in_heading = vim.tbl_count({'h1', 'h2', 'h3'}, parent) + local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' local tagname = node_text(root:child(1)) if vim.tbl_count(stats.first_tags) < 2 then @@ -465,7 +468,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) table.insert(stats.first_tags, tagname) return '' end - local s = ('%s%s'):format(ws(), url_encode(tagname), cssclass, trimmed) + local el = in_heading and 'span' or 'code' + local s = ('%s<%s class="%s">%s'):format(ws(), url_encode(tagname), el, cssclass, trimmed, el) if in_heading and prev ~= 'tag' then -- Start the container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. @@ -762,7 +766,7 @@ local function gen_css(fname) } .toc { /* max-width: 12rem; */ - height: 95%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ } .toc > div { diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 2612260226..3ee9d8b5dd 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -2,9 +2,7 @@ """Generates Nvim :help docs from C/Lua docstrings, using Doxygen. Also generates *.mpack files. To inspect the *.mpack structure: - - :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack'))) - + :new | put=v:lua.vim.inspect(v:lua.vim.mpack.unpack(readfile('runtime/doc/api.mpack','B'))) Flow: main @@ -287,7 +285,7 @@ annotation_map = { 'FUNC_API_FAST': '|api-fast|', 'FUNC_API_CHECK_TEXTLOCK': 'not allowed when |textlock| is active', 'FUNC_API_REMOTE_ONLY': '|RPC| only', - 'FUNC_API_LUA_ONLY': '|vim.api| only', + 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only', } -- cgit From a7a83bc4c25d63f3ae0a7a56e5211df1444699c4 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 9 Oct 2022 18:19:43 +0200 Subject: fix(docs-html): update parser - Improve generated HTML by updating parser which includes fixes for single "'" and single "|": https://github.com/neovim/tree-sitter-vimdoc/pull/31 - Updated parser also fixes the conceal issue for "help" highlight queries https://github.com/neovim/tree-sitter-vimdoc/issues/23 by NOT including whitespace in nodes. - But this means we need to restore the getws() function which scrapes leading whitespace from the original input (buffer). --- scripts/gen_help_html.lua | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 13601b91f5..c647484905 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -249,6 +249,16 @@ local function getbuflinestr(node, bufnr, offset) return table.concat(lines, '\n') end +-- Gets the whitespace just before `node` from the raw buffer text. +-- Needed for preformatted `old` lines. +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + local function get_tagname(node, bufnr) local text = vim.treesitter.get_node_text(node, bufnr) local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(text) or text @@ -354,12 +364,21 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local parent = root:parent() and root:parent():type() or nil local text = '' local trimmed - local function node_text(node) - return vim.treesitter.get_node_text(node or root, opt.buf) + -- Gets leading whitespace of `node`. + local function ws(node) + node = node or root + local ws_ = getws(node, opt.buf) + -- XXX: first node of a (line) includes whitespace, even after + -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ? + if ws_ == '' then + ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or '' + end + return ws_ end - -- Gets leading whitespace of the current node. - local function ws() - return node_text():match('^%s+') or '' + local function node_text(node, ws_) + node = node or root + ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or '' + return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) end if root:named_child_count() == 0 or node_name == 'ERROR' then @@ -401,7 +420,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - return ('
%s%s
'):format(ws(), trimmed) + return ('
%s
'):format(text) elseif node_name == 'block' then if is_blank(text) then return '' @@ -462,9 +481,9 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' - local tagname = node_text(root:child(1)) + local tagname = node_text(root:child(1), false) if vim.tbl_count(stats.first_tags) < 2 then - -- First 2 tags in the doc will be anchored at the main heading. + -- Force the first 2 tags in the doc to be anchored at the main heading. table.insert(stats.first_tags, tagname) return '' end -- cgit From 6b01e9bf872cc91530b41fcd97c6bf984776ade6 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Mon, 10 Oct 2022 00:40:17 +0200 Subject: feat(docs-html): try to use tags for ToC headings Problem: The generated ToC (table of contents) uses anchors derived from the heading title, e.g. the "Global Plugins" heading yields: https://neovim.io/doc/user/usr_05.html#_global-plugins- so if the heading title changes, then the old URL (anchor) is broken. Solution: :help tags change less often than heading titles, so if a heading contains a *tag*, use that as its anchor name instead. Example: https://neovim.io/doc/user/usr_05.html#standard-plugin --- scripts/gen_help_html.lua | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index c647484905..06ea1831b0 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -297,6 +297,16 @@ local function has_ancestor(node, ancestor_name) return false end +-- Gets the first matching child node matching `name`. +local function first(node, name) + for c, _ in node:iter_children() do + if c:named() and c:type() == name then + return c + end + end + return nil +end + local function validate_link(node, bufnr, fname) local helppage, tagname = get_tagname(node:child(1), bufnr) local ignored = false @@ -409,13 +419,18 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end -- Remove "===" and tags from ToC text. local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) + -- Use the first *tag* node as the heading anchor, if any. + local tagnode = first(root, 'tag') + local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname) if node_name == 'h1' or #headings == 0 then - table.insert(headings, { name = hname, subheadings = {}, }) + table.insert(headings, { name = hname, subheadings = {}, tag = tagname }) else - table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, }) + table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, tag = tagname }) end local el = node_name == 'h1' and 'h2' or 'h3' - return ('<%s class="help-heading">%s\n'):format(to_heading_tag(hname), el, text, el) + -- If we are re-using the *tag*, this heading anchor is redundant. + local a = tagnode and '' or (''):format(tagname) + return ('%s<%s class="help-heading">%s\n'):format(a, el, text, el) elseif node_name == 'column_heading' or node_name == 'column_name' then if root:has_error() then return text @@ -720,10 +735,10 @@ local function gen_one(fname, to_fname, old, commit) local n = 0 -- Count of all headings + subheadings. for _, h1 in ipairs(headings) do n = n + 1 + #h1.subheadings end for _, h1 in ipairs(headings) do - toc = toc .. ('
%s\n'):format(to_heading_tag(h1.name), h1.name) + toc = toc .. ('
%s\n'):format(h1.tag, h1.name) if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. for _, h2 in ipairs(h1.subheadings) do - toc = toc .. ('
%s
\n'):format(to_heading_tag(h2.name), h2.name) + toc = toc .. ('
%s
\n'):format(h2.tag, h2.name) end end toc = toc .. '
' -- cgit From a6d889eae1405b4a96c1938d3bab8f27c8c9a029 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 10 Oct 2022 12:10:57 +0100 Subject: refactor(lua2dox): format with stylua --- scripts/lua2dox.lua | 215 ++++++++++++++++++++++++++-------------------------- 1 file changed, 108 insertions(+), 107 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 674b42a699..562c3829c9 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -51,13 +51,13 @@ The effect is that you will get the function documented, but not with the parame ]] local function class(BaseClass, ClassInitialiser) - local newClass = {} -- a new class newClass + local newClass = {} -- a new class newClass if not ClassInitialiser and type(BaseClass) == 'function' then ClassInitialiser = BaseClass BaseClass = nil elseif type(BaseClass) == 'table' then -- our new class is a shallow copy of the base class! - for i,v in pairs(BaseClass) do + for i, v in pairs(BaseClass) do newClass[i] = v end newClass._base = BaseClass @@ -70,11 +70,11 @@ local function class(BaseClass, ClassInitialiser) local classMetatable = {} classMetatable.__call = function(class_tbl, ...) local newInstance = {} - setmetatable(newInstance,newClass) + setmetatable(newInstance, newClass) --if init then -- init(newInstance,...) if class_tbl.init then - class_tbl.init(newInstance,...) + class_tbl.init(newInstance, ...) else -- make sure that any stuff from the base class is initialized! if BaseClass and BaseClass.init then @@ -104,7 +104,7 @@ local TCore_Clock = class() --! \brief get the current time function TCore_Clock.GetTimeNow() - local gettimeofday = os.gettimeofday -- luacheck: ignore 143 Accessing an undefined field of a global variable. + local gettimeofday = os.gettimeofday -- luacheck: ignore 143 Accessing an undefined field of a global variable. if gettimeofday then return gettimeofday() else @@ -113,7 +113,7 @@ function TCore_Clock.GetTimeNow() end --! \brief constructor -function TCore_Clock.init(this,T0) +function TCore_Clock.init(this, T0) if T0 then this.t0 = T0 else @@ -122,36 +122,34 @@ function TCore_Clock.init(this,T0) end --! \brief get time string -function TCore_Clock.getTimeStamp(this,T0) +function TCore_Clock.getTimeStamp(this, T0) local t0 if T0 then t0 = T0 else t0 = this.t0 end - return os.date('%c %Z',t0) + return os.date('%c %Z', t0) end - --! \brief write to stdout local function TCore_IO_write(Str) - if (Str) then + if Str then io.write(Str) end end --! \brief write to stdout local function TCore_IO_writeln(Str) - if (Str) then + if Str then io.write(Str) end - io.write("\n") + io.write('\n') end - --! \brief trims a string local function string_trim(Str) - return Str:match("^%s*(.-)%s*$") + return Str:match('^%s*(.-)%s*$') end --! \brief split a string @@ -162,24 +160,23 @@ end ---@return string[] local function string_split(Str, Pattern) local splitStr = {} - local fpat = "(.-)" .. Pattern + local fpat = '(.-)' .. Pattern local last_end = 1 - local str, e, cap = string.find(Str,fpat, 1) + local str, e, cap = string.find(Str, fpat, 1) while str do - if str ~= 1 or cap ~= "" then - table.insert(splitStr,cap) + if str ~= 1 or cap ~= '' then + table.insert(splitStr, cap) end - last_end = e+1 - str, e, cap = string.find(Str,fpat, last_end) + last_end = e + 1 + str, e, cap = string.find(Str, fpat, last_end) end if last_end <= #Str then - cap = string.sub(Str,last_end) + cap = string.sub(Str, last_end) table.insert(splitStr, cap) end return splitStr end - --! \class TCore_Commandline --! \brief reads/parses commandline local TCore_Commandline = class() @@ -192,7 +189,7 @@ function TCore_Commandline.init(this) end --! \brief get value -function TCore_Commandline.getRaw(this,Key,Default) +function TCore_Commandline.getRaw(this, Key, Default) local val = this.argv[Key] if not val then val = Default @@ -209,14 +206,14 @@ local TStream_Read = class() --! \brief get contents of file --! --! \param Filename name of file to read (or nil == stdin) -function TStream_Read.getContents(this,Filename) +function TStream_Read.getContents(this, Filename) assert(Filename) -- get lines from file -- syphon lines to our table --TCore_Debug_show_var('Filename',Filename) - local filecontents={} + local filecontents = {} for line in io.lines(Filename) do - table.insert(filecontents,line) + table.insert(filecontents, line) end if filecontents then @@ -241,7 +238,7 @@ function TStream_Read.getLine(this) this.currentLine = nil else -- get line - if this.currentLineNo<=this.contentsLen then + if this.currentLineNo <= this.contentsLen then line = this.filecontents[this.currentLineNo] this.currentLineNo = this.currentLineNo + 1 else @@ -252,13 +249,13 @@ function TStream_Read.getLine(this) end --! \brief save line fragment -function TStream_Read.ungetLine(this,LineFrag) +function TStream_Read.ungetLine(this, LineFrag) this.currentLine = LineFrag end --! \brief is it eof? function TStream_Read.eof(this) - if this.currentLine or this.currentLineNo<=this.contentsLen then + if this.currentLine or this.currentLineNo <= this.contentsLen then return false end return true @@ -273,32 +270,32 @@ function TStream_Write.init(this) end --! \brief write immediately -function TStream_Write.write(_,Str) +function TStream_Write.write(_, Str) TCore_IO_write(Str) end --! \brief write immediately -function TStream_Write.writeln(_,Str) +function TStream_Write.writeln(_, Str) TCore_IO_writeln(Str) end --! \brief write immediately -function TStream_Write.writelnComment(_,Str) +function TStream_Write.writelnComment(_, Str) TCore_IO_write('// ZZ: ') TCore_IO_writeln(Str) end --! \brief write to tail -function TStream_Write.writelnTail(this,Line) +function TStream_Write.writelnTail(this, Line) if not Line then Line = '' end - table.insert(this.tailLine,Line) + table.insert(this.tailLine, Line) end --! \brief output tail lines function TStream_Write.write_tailLines(this) - for _,line in ipairs(this.tailLine) do + for _, line in ipairs(this.tailLine) do TCore_IO_writeln(line) end TCore_IO_write('// Lua2DoX new eof') @@ -308,9 +305,9 @@ end local TLua2DoX_filter = class() --! \brief allow us to do errormessages -function TLua2DoX_filter.warning(this,Line,LineNo,Legend) +function TLua2DoX_filter.warning(this, Line, LineNo, Legend) this.outStream:writelnTail( - '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"' + '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"' ) end @@ -319,47 +316,47 @@ end --! If the string has a comment on the end, this trims it off. --! local function TString_removeCommentFromLine(Line) - local pos_comment = string.find(Line,'%-%-') + local pos_comment = string.find(Line, '%-%-') local tailComment if pos_comment then - Line = string.sub(Line,1,pos_comment-1) - tailComment = string.sub(Line,pos_comment) + Line = string.sub(Line, 1, pos_comment - 1) + tailComment = string.sub(Line, pos_comment) end - return Line,tailComment + return Line, tailComment end --! \brief get directive from magic local function getMagicDirective(Line) - local macro,tail + local macro, tail local macroStr = '[\\@]' - local pos_macro = string.find(Line,macroStr) + local pos_macro = string.find(Line, macroStr) if pos_macro then --! ....\\ macro...stuff --! ....\@ macro...stuff - local line = string.sub(Line,pos_macro+1) - local space = string.find(line,'%s+') + local line = string.sub(Line, pos_macro + 1) + local space = string.find(line, '%s+') if space then - macro = string.sub(line,1,space-1) - tail = string_trim(string.sub(line,space+1)) + macro = string.sub(line, 1, space - 1) + tail = string_trim(string.sub(line, space + 1)) else macro = line - tail = '' + tail = '' end end - return macro,tail + return macro, tail end --! \brief check comment for fn -local function checkComment4fn(Fn_magic,MagicLines) +local function checkComment4fn(Fn_magic, MagicLines) local fn_magic = Fn_magic -- TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"') - local magicLines = string_split(MagicLines,'\n') + local magicLines = string_split(MagicLines, '\n') - local macro,tail + local macro, tail for _, line in ipairs(magicLines) do - macro,tail = getMagicDirective(line) + macro, tail = getMagicDirective(line) if macro == 'fn' then fn_magic = tail -- TCore_IO_writeln('// found fn "' .. fn_magic .. '"') @@ -371,15 +368,15 @@ local function checkComment4fn(Fn_magic,MagicLines) return fn_magic end -local types = {"number", "string", "table", "list", "boolean", "function"} +local types = { 'number', 'string', 'table', 'list', 'boolean', 'function' } --! \brief run the filter -function TLua2DoX_filter.readfile(this,AppStamp,Filename) +function TLua2DoX_filter.readfile(this, AppStamp, Filename) local inStream = TStream_Read() local outStream = TStream_Write() this.outStream = outStream -- save to this obj - if (inStream:getContents(Filename)) then + if inStream:getContents(Filename) then -- output the file local line local fn_magic -- function name/def from magic comment @@ -389,7 +386,7 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) outStream:writelnTail('// #######################') outStream:writelnTail() - local state = '' -- luacheck: ignore 231 variable is set but never accessed. + local state = '' -- luacheck: ignore 231 variable is set but never accessed. local offset = 0 local generic = {} local l = 0 @@ -398,7 +395,7 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) -- TCore_Debug_show_var('inStream',inStream) -- TCore_Debug_show_var('line',line ) l = l + 1 - if string.sub(line,1,2) == '--' then -- it's a comment + if string.sub(line, 1, 2) == '--' then -- it's a comment -- Allow people to write style similar to EmmyLua (since they are basically the same) -- instead of silently skipping things that start with --- if string.sub(line, 3, 3) == '@' then -- it's a magic comment @@ -414,22 +411,23 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic_split = string_split(magic, ' ') if magic_split[1] == 'param' then for _, type in ipairs(types) do - magic = magic:gsub("^param%s+([a-zA-Z_?]+)%s+.*%((" .. type .. ")%)", "param %1 %2" ) - magic = magic:gsub("^param%s+([a-zA-Z_?]+)%s+.*%((" .. type .. "|nil)%)", "param %1 %2" ) + magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2') + magic = + magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') end magic_split = string_split(magic, ' ') elseif magic_split[1] == 'return' then for _, type in ipairs(types) do - magic = magic:gsub("^return%s+.*%((" .. type .. ")%)", "return %1" ) - magic = magic:gsub("^return%s+.*%((" .. type .. "|nil)%)", "return %1" ) + magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1') + magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1') end magic_split = string_split(magic, ' ') end - if magic_split[1] == "generic" then - local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") - if generic_type == "" then - generic_type = "any" + if magic_split[1] == 'generic' then + local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)') + if generic_type == '' then + generic_type = 'any' end generic[generic_name] = generic_type else @@ -440,9 +438,9 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) if magic_split[type_index] then -- fix optional parameters - if magic_split[type_index] and magic_split[2]:find("%?$") then - if not magic_split[type_index]:find("nil") then - magic_split[type_index] = magic_split[type_index] .. "|nil" + if magic_split[type_index] and magic_split[2]:find('%?$') then + if not magic_split[type_index]:find('nil') then + magic_split[type_index] = magic_split[type_index] .. '|nil' end magic_split[2] = magic_split[2]:sub(1, -2) end @@ -454,42 +452,44 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) end -- surround some types by () for _, type in ipairs(types) do - magic_split[type_index] = magic_split[type_index]:gsub("^(" .. type .. "|nil):?$", "(%1)") - magic_split[type_index] = magic_split[type_index]:gsub("^(" .. type .. "):?$", "(%1)") + magic_split[type_index] = + magic_split[type_index]:gsub('^(' .. type .. '|nil):?$', '(%1)') + magic_split[type_index] = + magic_split[type_index]:gsub('^(' .. type .. '):?$', '(%1)') end end magic = table.concat(magic_split, ' ') outStream:writeln('/// @' .. magic) - fn_magic = checkComment4fn(fn_magic,magic) + fn_magic = checkComment4fn(fn_magic, magic) end - elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment - local comment = string.sub(line,4) - outStream:writeln('/// '.. comment) - elseif string.sub(line,3,4)=='[[' then -- it's a long comment - line = string.sub(line,5) -- nibble head + elseif string.sub(line, 3, 3) == '-' then -- it's a nonmagic doc comment + local comment = string.sub(line, 4) + outStream:writeln('/// ' .. comment) + elseif string.sub(line, 3, 4) == '[[' then -- it's a long comment + line = string.sub(line, 5) -- nibble head local comment = '' - local closeSquare,hitend,thisComment - while (not hitend) and (not inStream:eof()) do - closeSquare = string.find(line,']]') + local closeSquare, hitend, thisComment + while not hitend and (not inStream:eof()) do + closeSquare = string.find(line, ']]') if not closeSquare then -- need to look on another line thisComment = line .. '\n' line = inStream:getLine() else - thisComment = string.sub(line,1,closeSquare-1) + thisComment = string.sub(line, 1, closeSquare - 1) hitend = true -- unget the tail of the line -- in most cases it's empty. This may make us less efficient but -- easier to program - inStream:ungetLine(string_trim(string.sub(line,closeSquare+2))) + inStream:ungetLine(string_trim(string.sub(line, closeSquare + 2))) end comment = comment .. thisComment end - if string.sub(comment,1,1)=='@' then -- it's a long magic comment + if string.sub(comment, 1, 1) == '@' then -- it's a long magic comment outStream:write('/*' .. comment .. '*/ ') - fn_magic = checkComment4fn(fn_magic,comment) + fn_magic = checkComment4fn(fn_magic, comment) else -- discard outStream:write('/* zz:' .. comment .. '*/ ') fn_magic = nil @@ -504,64 +504,67 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) end elseif string.find(line, '^function') or string.find(line, '^local%s+function') then generic = {} - state = 'in_function' -- it's a function - local pos_fn = string.find(line,'function') + state = 'in_function' -- it's a function + local pos_fn = string.find(line, 'function') -- function -- ....v... if pos_fn then -- we've got a function local fn_type - if string.find(line,'^local%s+') then - fn_type = ''--'static ' -- static functions seem to be excluded + if string.find(line, '^local%s+') then + fn_type = '' --'static ' -- static functions seem to be excluded else fn_type = '' end - local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8))) + local fn = TString_removeCommentFromLine(string_trim(string.sub(line, pos_fn + 8))) if fn_magic then fn = fn_magic end - if string.sub(fn,1,1)=='(' then + if string.sub(fn, 1, 1) == '(' then -- it's an anonymous function outStream:writelnComment(line) else -- fn has a name, so is interesting -- want to fix for iffy declarations - local open_paren = string.find(fn,'[%({]') + local open_paren = string.find(fn, '[%({]') if open_paren then -- we might have a missing close paren - if not string.find(fn,'%)') then + if not string.find(fn, '%)') then fn = fn .. ' ___MissingCloseParenHere___)' end end -- Big hax - if string.find(fn, ":") then + if string.find(fn, ':') then -- TODO: We need to add a first parameter of "SELF" here -- local colon_place = string.find(fn, ":") -- local name = string.sub(fn, 1, colon_place) - fn = fn:gsub(":", ".", 1) - outStream:writeln("/// @param self") + fn = fn:gsub(':', '.', 1) + outStream:writeln('/// @param self') - local paren_start = string.find(fn, "(", 1, true) - local paren_finish = string.find(fn, ")", 1, true) + local paren_start = string.find(fn, '(', 1, true) + local paren_finish = string.find(fn, ')', 1, true) -- Nothing in between the parens local comma if paren_finish == paren_start + 1 then - comma = "" + comma = '' else - comma = ", " + comma = ', ' end - fn = string.sub(fn, 1, paren_start) .. "self" .. comma .. string.sub(fn, paren_start + 1) + fn = string.sub(fn, 1, paren_start) + .. 'self' + .. comma + .. string.sub(fn, paren_start + 1) end -- add vanilla function outStream:writeln(fn_type .. 'function ' .. fn .. '{}') end else - this:warning(inStream:getLineNo(),'something weird here') + this:warning(inStream:getLineNo(), 'something weird here') end fn_magic = nil -- mustn't indavertently use it again @@ -572,8 +575,8 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) -- fn_magic = nil -- mustn't indavertently use it again else - state = '' -- unknown - if #line>0 then -- we don't know what this line means, so just comment it out + state = '' -- unknown + if #line > 0 then -- we don't know what this line means, so just comment it out outStream:writeln('// zz: ' .. line) else outStream:writeln() -- keep this line blank @@ -601,8 +604,7 @@ function TApp.init(this) end function TApp.getRunStamp(this) - return this.name .. ' (' .. this.version .. ') ' - .. this.timestamp + return this.name .. ' (' .. this.version .. ') ' .. this.timestamp end function TApp.getVersion(this) @@ -639,8 +641,7 @@ else local filename = argv1 local filter = TLua2DoX_filter() - filter:readfile(appStamp,filename) + filter:readfile(appStamp, filename) end - --eof -- cgit From c401b06fe978167a7408375e769be38db4146935 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 10 Oct 2022 12:21:19 +0100 Subject: refactor(lua2dox): tidy --- scripts/lua2dox.lua | 96 ++++++++--------------------------------------------- 1 file changed, 13 insertions(+), 83 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 562c3829c9..86afe97a7e 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -50,88 +50,28 @@ However I have put in a hack that will insert the "missing" close paren. The effect is that you will get the function documented, but not with the parameter list you might expect. ]] -local function class(BaseClass, ClassInitialiser) +local function class() local newClass = {} -- a new class newClass - if not ClassInitialiser and type(BaseClass) == 'function' then - ClassInitialiser = BaseClass - BaseClass = nil - elseif type(BaseClass) == 'table' then - -- our new class is a shallow copy of the base class! - for i, v in pairs(BaseClass) do - newClass[i] = v - end - newClass._base = BaseClass - end -- the class will be the metatable for all its newInstanceects, -- and they will look up their methods in it. newClass.__index = newClass -- expose a constructor which can be called by () - local classMetatable = {} - classMetatable.__call = function(class_tbl, ...) - local newInstance = {} - setmetatable(newInstance, newClass) - --if init then - -- init(newInstance,...) - if class_tbl.init then - class_tbl.init(newInstance, ...) - else - -- make sure that any stuff from the base class is initialized! - if BaseClass and BaseClass.init then - BaseClass.init(newInstance, ...) - end - end - return newInstance - end - newClass.init = ClassInitialiser - newClass.is_a = function(this, klass) - local thisMetatable = getmetatable(this) - while thisMetatable do - if thisMetatable == klass then - return true + setmetatable(newClass, { + __call = function(class_tbl, ...) + local newInstance = {} + setmetatable(newInstance, newClass) + --if init then + -- init(newInstance,...) + if class_tbl.init then + class_tbl.init(newInstance, ...) end - thisMetatable = thisMetatable._base + return newInstance end - return false - end - setmetatable(newClass, classMetatable) + }) return newClass end ---! \class TCore_Clock ---! \brief a clock -local TCore_Clock = class() - ---! \brief get the current time -function TCore_Clock.GetTimeNow() - local gettimeofday = os.gettimeofday -- luacheck: ignore 143 Accessing an undefined field of a global variable. - if gettimeofday then - return gettimeofday() - else - return os.time() - end -end - ---! \brief constructor -function TCore_Clock.init(this, T0) - if T0 then - this.t0 = T0 - else - this.t0 = TCore_Clock.GetTimeNow() - end -end - ---! \brief get time string -function TCore_Clock.getTimeStamp(this, T0) - local t0 - if T0 then - t0 = T0 - else - t0 = this.t0 - end - return os.date('%c %Z', t0) -end - --! \brief write to stdout local function TCore_IO_write(Str) if Str then @@ -210,7 +150,6 @@ function TStream_Read.getContents(this, Filename) assert(Filename) -- get lines from file -- syphon lines to our table - --TCore_Debug_show_var('Filename',Filename) local filecontents = {} for line in io.lines(Filename) do table.insert(filecontents, line) @@ -392,8 +331,6 @@ function TLua2DoX_filter.readfile(this, AppStamp, Filename) local l = 0 while not (inStream:eof()) do line = string_trim(inStream:getLine()) - -- TCore_Debug_show_var('inStream',inStream) - -- TCore_Debug_show_var('line',line ) l = l + 1 if string.sub(line, 1, 2) == '--' then -- it's a comment -- Allow people to write style similar to EmmyLua (since they are basically the same) @@ -510,12 +447,6 @@ function TLua2DoX_filter.readfile(this, AppStamp, Filename) -- ....v... if pos_fn then -- we've got a function - local fn_type - if string.find(line, '^local%s+') then - fn_type = '' --'static ' -- static functions seem to be excluded - else - fn_type = '' - end local fn = TString_removeCommentFromLine(string_trim(string.sub(line, pos_fn + 8))) if fn_magic then fn = fn_magic @@ -561,7 +492,7 @@ function TLua2DoX_filter.readfile(this, AppStamp, Filename) end -- add vanilla function - outStream:writeln(fn_type .. 'function ' .. fn .. '{}') + outStream:writeln('function ' .. fn .. '{}') end else this:warning(inStream:getLineNo(), 'something weird here') @@ -596,8 +527,7 @@ local TApp = class() --! \brief constructor function TApp.init(this) - local t0 = TCore_Clock() - this.timestamp = t0:getTimeStamp() + this.timestamp = os.date('%c %Z', os.time()) this.name = 'Lua2DoX' this.version = '0.2 20130128' this.copyright = 'Copyright (c) Simon Dales 2012-13' -- cgit From 3b6c7f9c7f2df493e24990c47a338ef8b46dfe8a Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 10 Oct 2022 12:21:40 +0100 Subject: refactor(lua2dox_filter): format --- scripts/lua2dox_filter | 105 ++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 54 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter index 0b9f59b6ac..e3fa95d0cf 100755 --- a/scripts/lua2dox_filter +++ b/scripts/lua2dox_filter @@ -22,72 +22,69 @@ LANG="" ##! \brief test executable to see if it exists -test_executable(){ - P_EXE="$1" - ######### - WHICH=`which ${P_EXE}` - if test -z "${WHICH}" - then - echo "not found \"${P_EXE}\"" - else - EXE="${P_EXE}" - fi - } +test_executable() { + P_EXE="$1" + ######### + WHICH=$(which "$P_EXE") + if test -z "${WHICH}"; then + echo "not found \"${P_EXE}\"" + else + EXE="${P_EXE}" + fi +} ##! \brief sets the lua interpreter -set_lua(){ - if test -z "${EXE}"; then - test_executable '.deps/usr/bin/luajit' - fi +set_lua() { + if test -z "${EXE}"; then + test_executable '.deps/usr/bin/luajit' + fi - if test -z "${EXE}"; then - test_executable 'luajit' - fi + if test -z "${EXE}"; then + test_executable 'luajit' + fi - if test -z "${EXE}"; then - test_executable 'lua' - fi + if test -z "${EXE}"; then + test_executable 'lua' + fi } ##! \brief makes canonical name of file -##! +##! ##! Note that "readlink -f" doesn't work in MacOSX -##! -do_readlink(){ - pushd . > /dev/null - TARGET_FILE=$1 - - cd `dirname $TARGET_FILE` - TARGET_FILE=`basename $TARGET_FILE` - - # Iterate down a (possible) chain of symlinks - while [ -L "$TARGET_FILE" ] - do - TARGET_FILE=`readlink $TARGET_FILE` - cd `dirname $TARGET_FILE` - TARGET_FILE=`basename $TARGET_FILE` - done - - PHYS_DIR=`pwd -P` - RESULT=$PHYS_DIR - popd > /dev/null - } +##! +do_readlink() { + pushd . > /dev/null + TARGET_FILE=$1 + + cd "$(dirname $TARGET_FILE)" + TARGET_FILE=$(basename "$TARGET_FILE") + + # Iterate down a (possible) chain of symlinks + while [ -L "$TARGET_FILE" ]; do + TARGET_FILE=$(readlink "$TARGET_FILE") + cd $(dirname "$TARGET_FILE") + TARGET_FILE=$(basename "$TARGET_FILE") + done + + PHYS_DIR=$(pwd -P) + RESULT=$PHYS_DIR + popd > /dev/null +} ##main set_lua -if test -z "${EXE}" -then - echo "no lua interpreter found" - exit 1 +if test -z "${EXE}"; then + echo "no lua interpreter found" + exit 1 else - BASENAME=`basename "$0"` - do_readlink "$0" - DIRNAME="${RESULT}" - - LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}" - #echo "lua[${LUASCRIPT}]" + BASENAME=$(basename "$0") + do_readlink "$0" + DIRNAME="${RESULT}" + + LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}" + #echo "lua[${LUASCRIPT}]" - ${EXE} ${LUASCRIPT} $@ + ${EXE} ${LUASCRIPT} $@ fi -# + ##eof -- cgit