From 0c0a426e40692eabb3e1800cc93135eb70a28334 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 23 Aug 2022 14:18:05 +0100 Subject: doc(CONTRIBUTING): update #19891 - add section for managing includes via IWYU - recommend clangd over ctags - tidy up links - remove ./scripts/check-includes.py --- scripts/check-includes.py | 64 ----------------------------------------------- 1 file changed, 64 deletions(-) delete mode 100755 scripts/check-includes.py (limited to 'scripts') diff --git a/scripts/check-includes.py b/scripts/check-includes.py deleted file mode 100755 index ed1fe407c5..0000000000 --- a/scripts/check-includes.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python - -import sys -import re -import os - -from subprocess import Popen, PIPE -from argparse import ArgumentParser - - -GENERATED_INCLUDE_RE = re.compile( - r'^\s*#\s*include\s*"([/a-z_0-9.]+\.generated\.h)"(\s+//.*)?$') - - -def main(argv): - argparser = ArgumentParser() - argparser.add_argument('--generated-includes-dir', action='append', - help='Directory where generated includes are located.') - argparser.add_argument('--file', type=open, help='File to check.') - argparser.add_argument('iwyu_args', nargs='*', - help='IWYU arguments, must go after --.') - args = argparser.parse_args(argv) - - with args.file: - iwyu = Popen(['include-what-you-use', '-xc'] + args.iwyu_args + ['/dev/stdin'], - stdin=PIPE, stdout=PIPE, stderr=PIPE) - - for line in args.file: - match = GENERATED_INCLUDE_RE.match(line) - if match: - for d in args.generated_includes_dir: - try: - f = open(os.path.join(d, match.group(1))) - except IOError: - continue - else: - with f: - for generated_line in f: - iwyu.stdin.write(generated_line) - break - else: - raise IOError('Failed to find {0}'.format(match.group(1))) - else: - iwyu.stdin.write(line) - - iwyu.stdin.close() - - out = iwyu.stdout.read() - err = iwyu.stderr.read() - - ret = iwyu.wait() - - if ret != 2: - print('IWYU failed with exit code {0}:'.format(ret)) - print('{0} stdout {0}'.format('=' * ((80 - len(' stdout ')) // 2))) - print(out) - print('{0} stderr {0}'.format('=' * ((80 - len(' stderr ')) // 2))) - print(err) - return 1 - return 0 - - -if __name__ == '__main__': - raise SystemExit(main(sys.argv[1:])) -- cgit From c0d60526541a3cf977ae623471ae4a347b492af1 Mon Sep 17 00:00:00 2001 From: bfredl Date: Tue, 23 Aug 2022 09:33:08 +0200 Subject: perf(api): allow to use an arena for return values --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index bca3dd816f..23ed0e3f08 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -801,7 +801,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error')) + if a[0] not in ('void', 'Error', 'Arena')) if not fmt_vimhelp: c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) -- cgit From ddb762f4013ac2532ad45704466058d867e3a6ed Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Wed, 14 Sep 2022 11:08:31 +0200 Subject: docs(treesitter): clean up and update treesitter.txt (#20142) * add type annotations to code * clean up and expand static documentation * consistent use of tags for static and generated docs --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 23ed0e3f08..766c420c7d 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -260,7 +260,7 @@ CONFIG = { 'helptag_fmt': lambda name: ( '*lua-treesitter-core*' if name.lower() == 'treesitter' - else f'*treesitter-{name.lower()}*'), + else f'*lua-treesitter-{name.lower()}*'), 'fn_helptag_fmt': lambda fstem, name: ( f'*{name}()*' if name != 'new' -- cgit From 09b64d75bd92a95d89c4f39f9df7918760abe98d Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 27 Mar 2022 19:47:34 -0700 Subject: feat(docs): gen_help_html.lua MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: The :help docs HTML generated is driven by an old awk script `runtime/doc/makehtml.awk` that is hard to maintain (ad hoc parser and no one has touched it in decades) and has bugs like: - https://github.com/neovim/neovim.github.io/issues/96 - https://github.com/neovim/neovim.github.io/issues/97 Solution: Use Lua + treesitter (https://github.com/vigoux/tree-sitter-vimdoc) to generate :help docs HTML. Also validates tag links. fix https://github.com/neovim/neovim.github.io/issues/96 fix https://github.com/neovim/neovim.github.io/issues/97 TODO: - delete doc_html build task - delete runtime/doc/Makefile - delete makehtml.awk - delete maketags.awk OUTPUT: $ nvim -V1 -es --clean +"lua require('scripts.gen_help_html')" output dir: /…/neovim.github.io/_site/doc/ generated (207 errors): api.txt => api.html generated (122 errors): arabic.txt => arabic.html generated (285 errors): autocmd.txt => autocmd.html generated (641 errors): builtin.txt => builtin.html generated (623 errors): change.txt => change.html generated (65 errors): channel.txt => channel.html generated (353 errors): cmdline.txt => cmdline.html generated (3 errors): debug.txt => debug.html generated (28 errors): deprecated.txt => deprecated.html generated (193 errors): dev_style.txt => dev_style.html generated (460 errors): develop.txt => develop.html generated (19 errors): diagnostic.txt => diagnostic.html generated (57 errors): diff.txt => diff.html generated (818 errors): digraph.txt => digraph.html generated (330 errors): editing.txt => editing.html generated (368 errors): eval.txt => eval.html generated (184 errors): fold.txt => fold.html generated (61 errors): ft_ada.txt => ft_ada.html generated (0 errors): ft_ps1.txt => ft_ps1.html generated (20 errors): ft_raku.txt => ft_raku.html generated (5 errors): ft_rust.txt => ft_rust.html generated (41 errors): ft_sql.txt => ft_sql.html generated (110 errors): gui.txt => gui.html generated (79 errors): hebrew.txt => hebrew.html generated (17 errors): help.txt => index.html generated (104 errors): helphelp.txt => helphelp.html generated (0 errors): if_cscop.txt => if_cscop.html generated (23 errors): if_perl.txt => if_perl.html generated (16 errors): if_pyth.txt => if_pyth.html generated (9 errors): if_ruby.txt => if_ruby.html generated (216 errors): indent.txt => indent.html generated (634 errors): index.txt => vimindex.html generated (320 errors): insert.txt => insert.html generated (265 errors): intro.txt => intro.html generated (9 errors): job_control.txt => job_control.html generated (0 errors): lsp-extension.txt => lsp-extension.html generated (214 errors): lsp.txt => lsp.html generated (311 errors): lua.txt => lua.html generated (592 errors): luaref.txt => luaref.html generated (798 errors): luvref.txt => luvref.html generated (663 errors): map.txt => map.html generated (228 errors): mbyte.txt => mbyte.html generated (228 errors): message.txt => message.html generated (0 errors): mlang.txt => mlang.html generated (761 errors): motion.txt => motion.html generated (4 errors): nvim.txt => nvim.html generated (226 errors): nvim_terminal_emulator.txt => nvim_terminal_emulator.html generated (988 errors): options.txt => options.html generated (567 errors): pattern.txt => pattern.html generated (15 errors): pi_gzip.txt => pi_gzip.html generated (10 errors): pi_health.txt => pi_health.html generated (27 errors): pi_msgpack.txt => pi_msgpack.html generated (2177 errors): pi_netrw.txt => pi_netrw.html generated (41 errors): pi_paren.txt => pi_paren.html generated (9 errors): pi_spec.txt => pi_spec.html generated (218 errors): pi_tar.txt => pi_tar.html generated (0 errors): pi_tutor.txt => pi_tutor.html generated (235 errors): pi_zip.txt => pi_zip.html generated (265 errors): print.txt => print.html generated (31 errors): provider.txt => provider.html generated (335 errors): quickfix.txt => quickfix.html generated (572 errors): quickref.txt => quickref.html generated (109 errors): recover.txt => recover.html generated (14 errors): remote.txt => remote.html generated (14 errors): remote_plugin.txt => remote_plugin.html generated (351 errors): repeat.txt => repeat.html generated (23 errors): rileft.txt => rileft.html generated (12 errors): russian.txt => russian.html generated (6 errors): scroll.txt => scroll.html generated (106 errors): sign.txt => sign.html generated (347 errors): spell.txt => spell.html generated (784 errors): starting.txt => starting.html generated (1499 errors): syntax.txt => syntax.html generated (23 errors): tabpage.txt => tabpage.html generated (257 errors): tagsrch.txt => tagsrch.html generated (31 errors): term.txt => term.html generated (0 errors): testing.txt => testing.html generated (96 errors): tips.txt => tips.html generated (57 errors): treesitter.txt => treesitter.html generated (71 errors): uganda.txt => uganda.html generated (74 errors): ui.txt => ui.html generated (87 errors): undo.txt => undo.html generated (17 errors): userfunc.txt => userfunc.html generated (1 errors): usr_01.txt => usr_01.html generated (89 errors): usr_02.txt => usr_02.html generated (293 errors): usr_03.txt => usr_03.html generated (46 errors): usr_04.txt => usr_04.html generated (96 errors): usr_05.txt => usr_05.html generated (54 errors): usr_06.txt => usr_06.html generated (20 errors): usr_07.txt => usr_07.html generated (241 errors): usr_08.txt => usr_08.html generated (130 errors): usr_09.txt => usr_09.html generated (50 errors): usr_10.txt => usr_10.html generated (33 errors): usr_11.txt => usr_11.html generated (32 errors): usr_12.txt => usr_12.html generated (22 errors): usr_20.txt => usr_20.html generated (75 errors): usr_21.txt => usr_21.html generated (8 errors): usr_22.txt => usr_22.html generated (3 errors): usr_23.txt => usr_23.html generated (163 errors): usr_25.txt => usr_25.html generated (13 errors): usr_26.txt => usr_26.html generated (84 errors): usr_27.txt => usr_27.html generated (173 errors): usr_28.txt => usr_28.html generated (285 errors): usr_29.txt => usr_29.html generated (280 errors): usr_30.txt => usr_30.html generated (11 errors): usr_31.txt => usr_31.html generated (13 errors): usr_32.txt => usr_32.html generated (156 errors): usr_40.txt => usr_40.html generated (134 errors): usr_41.txt => usr_41.html generated (35 errors): usr_42.txt => usr_42.html generated (19 errors): usr_43.txt => usr_43.html generated (60 errors): usr_44.txt => usr_44.html generated (13 errors): usr_45.txt => usr_45.html generated (1 errors): usr_toc.txt => usr_toc.html generated (69 errors): various.txt => various.html generated (68 errors): vi_diff.txt => vi_diff.html generated (437 errors): vim_diff.txt => vim_diff.html generated (296 errors): visual.txt => visual.html generated (181 errors): windows.txt => windows.html generated 119 html pages total errors: 23862 invalid tags: 537 --- scripts/gen_help_html.lua | 830 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/gen_help_html.py | 389 ---------------------- 2 files changed, 830 insertions(+), 389 deletions(-) create mode 100644 scripts/gen_help_html.lua delete mode 100644 scripts/gen_help_html.py (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua new file mode 100644 index 0000000000..d79ece53f3 --- /dev/null +++ b/scripts/gen_help_html.lua @@ -0,0 +1,830 @@ +-- Converts Vim :help files to HTML. Validates |tag| links and document syntax (parser errors). +-- +-- USAGE (GENERATE HTML): +-- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). +-- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" +-- - Read the docstring at gen(). +-- 3. cd target/dir/ && jekyll serve --host 0.0.0.0 +-- 4. Visit http://localhost:4000/…/help.txt.html +-- +-- USAGE (VALIDATE): +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html').validate()" +-- - validate() is 10x faster than gen(), so it is used in CI. +-- +-- SELF-TEST MODE: +-- 1. nvim -V1 -es +"lua require('scripts.gen_help_html')._test()" +-- +-- NOTES: +-- * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too +-- slow (~1 min) to run in per-commit CI. +-- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. +-- * visit_validate() is the core function used by validate(). +-- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. +-- +-- parser bugs: +-- * Should NOT be code_block: +-- tab:xy The 'x' is always used, then 'y' as many times as will +-- fit. Thus "tab:>-" displays: +-- > +-- >- +-- >-- +-- etc. +-- +-- tab:xyz The 'z' is always used, then 'x' is prepended, and +-- then 'y' is used as many times as will fit. Thus +-- "tab:<->" displays: +-- > +-- <> +-- <-> +-- <--> +-- etc. +-- * Should NOT be a "headline". Perhaps a "table" (or just "line"). +-- expr5 and expr6 *expr5* *expr6* +-- --------------- +-- expr6 + expr6 Number addition, |List| or |Blob| concatenation *expr-+* +-- expr6 - expr6 Number subtraction *expr--* +-- expr6 . expr6 String concatenation *expr-.* +-- expr6 .. expr6 String concatenation *expr-..* + +local tagmap = nil +local helpfiles = nil +local invalid_tags = {} + +local commit = '?' +local api = vim.api +local M = {} + +-- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). +-- All other files are "legacy" files which require fixed-width layout. +local new_layout = { + ['api.txt'] = true, + ['channel.txt'] = true, + ['develop.txt'] = true, + ['nvim.txt'] = true, + ['pi_health.txt'] = true, + ['provider.txt'] = true, + ['ui.txt'] = true, +} + +-- TODO: treesitter gets stuck on these files... +local exclude = { + ['filetype.txt'] = true, + ['usr_24.txt'] = true, +} + +local function tofile(fname, text) + local f = io.open(fname, 'w') + if not f then + error(('failed to write: %s'):format(f)) + else + f:write(text) + f:close() + end +end + +local function html_esc(s) + if s:find('', '>') +end + +local function url_encode(s) + -- Credit: tpope / vim-unimpaired + -- NOTE: these chars intentionally *not* escaped: ' ( ) + return vim.fn.substitute(vim.fn.iconv(s, 'latin1', 'utf-8'), + [=[[^A-Za-z0-9()'_.~-]]=], + [=[\="%".printf("%02X",char2nr(submatch(0)))]=], + 'g') +end + +-- Removes the ">" and "<" chars that delineate a codeblock in Vim :help files. +local function trim_gt_lt(s) + return s:gsub('^%s*>%s*\n', ''):gsub('\n<', '') +end + +local function expandtabs(s) + return s:gsub('\t', (' '):rep(8)) +end + +local function to_titlecase(s) + local text = '' + for w in vim.gsplit(s, '[ \t]+') do + text = ('%s %s%s'):format(text, vim.fn.toupper(w:sub(1, 1)), w:sub(2)) + end + return text +end + +local function to_heading_tag(text) + -- Prepend "_" to avoid conflicts with actual :help tags. + return text and string.format('_%s', vim.fn.tolower((text:gsub('%s+', '-')))) or 'unknown' +end + +local function basename_noext(f) + return vim.fs.basename(f:gsub('%.txt', '')) +end + +local function is_blank(s) + return not not s:find('^%s*$') +end + +local function trim(s) + return vim.trim(s) +end + +local function trim_bullet(s) + return s:gsub('^%s*[-*•]%s', '') +end + +local function startswith_bullet(s) + return s:find('^%s*[-*•]%s') +end + +-- Checks if a given line is a "noise" line that doesn't look good in HTML form. +local function is_noise(line) + return ( + line:find('Type .*gO.* to see the table of contents') + -- Title line of traditional :help pages. + -- Example: "NVIM REFERENCE MANUAL by ..." + or line:find('^%s*N?VIM REFERENCE MANUAL') + -- First line of traditional :help pages. + -- Example: "*api.txt* Nvim" + or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') + -- modeline + -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" + or line:find('^%s*vi[m]%:.*ft=help') + or line:find('^%s*vi[m]%:.*filetype=help') + ) +end + +-- Creates a github issue URL at vigoux/tree-sitter-vimdoc with prefilled content. +local function get_bug_url_vimdoc(fname, to_fname, sample_text) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ('https://github.com/vigoux/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + ..vim.fs.basename(fname) + ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' + ..this_url + ..'%0D%0DContext%3A%0D%0D%60%60%60%0D' + ..url_encode(sample_text) + ..'%0D%60%60%60') + return bug_url +end + +-- Creates a github issue URL at neovim/neovim with prefilled content. +local function get_bug_url_nvim(fname, to_fname, sample_text, token_name) + local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) + local bug_url = ('https://github.com/neovim/neovim/issues/new?labels=bug&title=user+docs+HTML%3A+' + ..vim.fs.basename(fname) + ..'+&body=%60gen_help_html.lua%60+problem+at%3A+' + ..this_url + ..'%0D' + ..(token_name and '+unhandled+token%3A+%60'..token_name..'%60' or '') + ..'%0DContext%3A%0D%0D%60%60%60%0D' + ..url_encode(sample_text) + ..'%0D%60%60%60') + return bug_url +end + +-- Gets a "foo.html" name from a "foo.txt" helpfile name. +local function get_helppage(f) + if not f then + return nil + end + -- Special case: help.txt is the "main landing page" of :help files, not index.txt. + if f == 'index.txt' then + return 'vimindex.html' + elseif f == 'help.txt' then + return 'index.html' + end + + return f:gsub('%.txt$', '.html') +end + +-- Counts leading spaces (tab=8) to decide the indent size of multiline text. +-- +-- Blank lines (empty or whitespace-only) are ignored. +local function get_indent(s) + local min_indent = nil + for line in vim.gsplit(s, '\n') do + if line and not is_blank(line) then + local ws = expandtabs(line:match('^%s+') or '') + min_indent = (not min_indent or ws:len() < min_indent) and ws:len() or min_indent + end + end + return min_indent or 0 +end + +-- Removes the common indent level, after expanding tabs to 8 spaces. +local function trim_indent(s) + local indent_size = get_indent(s) + local trimmed = '' + for line in vim.gsplit(s, '\n') do + line = expandtabs(line) + trimmed = ('%s%s\n'):format(trimmed, line:sub(indent_size + 1)) + end + return trimmed:sub(1, -2) +end + +-- Gets raw buffer text in the node's range (+/- an offset), as a newline-delimited string. +local function getbuflinestr(node, bufnr, offset) + local line1, _, line2, _ = node:range() + line1 = line1 - offset + line2 = line2 + offset + local lines = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1) + return table.concat(lines, '\n') +end + +-- Gets the whitespace just before `node` from the raw buffer text. +-- Needed for preformatted `old` lines. +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + +local function get_tagname(node, bufnr, link) + local node_name = (node.named and node:named()) and node:type() or nil + local node_text = vim.treesitter.get_node_text(node, bufnr) + local tag = ((node_name == 'option' and node_text) + or (link and node_text:gsub('^|', ''):gsub('|$', '') or node_text:gsub('^%*', ''):gsub('%*$', ''))) + local helpfile = tag and vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" + return helppage, tag +end + +-- Traverses the tree at `root` and checks that |tag| links point to valid helptags. +local function visit_validate(root, level, lang_tree, opt, stats) + level = level or 0 + local node_name = (root.named and root:named()) and root:type() or nil + local toplevel = level < 1 + + if root:child_count() > 0 then + for node, _ in root:iter_children() do + if node:named() then + visit_validate(node, level + 1, lang_tree, opt, stats) + end + end + end + + if node_name == 'ERROR' then + -- Store the raw text to give context to the bug report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + elseif node_name == 'hotlink' or node_name == 'option' then + local _, tagname = get_tagname(root, opt.buf, true) + if not root:has_error() and not tagmap[tagname] then + invalid_tags[tagname] = vim.fs.basename(opt.fname) + end + end +end + +-- Generates HTML from node `root` recursively. +local function visit_node(root, level, lang_tree, headings, opt, stats) + level = level or 0 + + local node_name = (root.named and root:named()) and root:type() or nil + -- Previous sibling kind (string). + local prev = root:prev_sibling() and (root:prev_sibling().named and root:prev_sibling():named()) and root:prev_sibling():type() or nil + -- Next sibling kind (string). + local next_ = root:next_sibling() and (root:next_sibling().named and root:next_sibling():named()) and root:next_sibling():type() or nil + -- Parent kind (string). + local parent = root:parent() and root:parent():type() or nil + local text = '' + local toplevel = level < 1 + local function node_text() + return vim.treesitter.get_node_text(root, opt.buf) + end + + if root:child_count() == 0 then + text = node_text() + else + -- Process children and join them with whitespace. + for node, _ in root:iter_children() do + if node:named() then + local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) + local ws = r == '' and '' or ((opt.old and (node:type() == 'word' or not node:named())) and getws(node, opt.buf) or ' ') + text = string.format('%s%s%s', text, ws, r) + end + end + end + local trimmed = trim(text) + + if node_name == 'help_file' then -- root node + return text + elseif node_name == 'word' or node_name == 'uppercase_name' then + if parent == 'headline' then + -- Start a new heading item, or update the current one. + local n = (prev == nil or #headings == 0) and #headings + 1 or #headings + headings[n] = string.format('%s%s', headings[n] and headings[n]..' ' or '', text) + end + + return html_esc(text) + elseif node_name == 'headline' then + return ('

%s

\n'):format(to_heading_tag(headings[#headings]), text) + elseif node_name == 'column_heading' or node_name == 'column_name' then + return ('

%s

\n'):format(trimmed) + elseif node_name == 'line' then + -- TODO: remove these "sibling inspection" hacks once the parser provides structured info + -- about paragraphs and listitems: https://github.com/vigoux/tree-sitter-vimdoc/issues/12 + local next_text = root:next_sibling() and vim.treesitter.get_node_text(root:next_sibling(), opt.buf) or '' + local li = startswith_bullet(text) -- Listitem? + local next_li = startswith_bullet(next_text) -- Next is listitem? + -- Close the paragraph/listitem if the next sibling is not a line. + local close = (next_ ~= 'line' or next_li or is_blank(next_text)) and '\n' or '' + + -- HACK: discard common "noise" lines. + if is_noise(text) then + table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + return (opt.old or prev ~= 'line') and '' or close + end + + if opt.old then + -- XXX: Treat old docs as preformatted. Until those docs are "fixed" or we get better info + -- from tree-sitter-vimdoc, this avoids broken layout for legacy docs. + return ('
%s
\n'):format(text) + end + + if li then + return string.format('
%s%s', trim_bullet(expandtabs(text)), close) + end + if prev ~= 'line' then -- Start a new paragraph. + return string.format('
%s%s', expandtabs(text), close) + end + + -- Continue in the current paragraph/listitem. + return string.format('%s%s', expandtabs(text), close) + elseif node_name == 'hotlink' or node_name == 'option' then + local helppage, tagname = get_tagname(root, opt.buf, true) + if not root:has_error() and not tagmap[tagname] then + invalid_tags[tagname] = vim.fs.basename(opt.fname) + end + return ('%s'):format(helppage, url_encode(tagname), html_esc(tagname)) + elseif node_name == 'backtick' then + return ('%s'):format(html_esc(text)) + elseif node_name == 'argument' then + return ('{%s}'):format(html_esc(trimmed)) + elseif node_name == 'code_block' then + return ('
\n%s
\n'):format(html_esc(trim_indent(trim_gt_lt(text)))) + elseif node_name == 'tag' then -- anchor + local _, tagname = get_tagname(root, opt.buf, false) + local s = ('%s'):format(url_encode(tagname), trimmed) + if parent == 'headline' and prev ~= 'tag' then + -- Start the container for tags in a heading. + -- This makes "justify-content:space-between" right-align the tags. + --

foo bartag1 tag2

+ return string.format('%s', s) + elseif parent == 'headline' and next_ == nil then + -- End the container for tags in a heading. + return string.format('%s', s) + end + return s + elseif node_name == 'ERROR' then + -- Store the raw text to give context to the bug report. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + table.insert(stats.parse_errors, sample_text) + if prev == 'ERROR' then + -- Avoid trashing the text with cascading errors. + return trimmed, ('parse-error:"%s"'):format(node_text()) + end + return ('%s'):format( + get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) + else -- Unknown token. + local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + return ('%s'):format( + node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) + end +end + +local function get_helpfiles(include) + local dir = './build/runtime/doc' + local rv = {} + for f, type in vim.fs.dir(dir) do + if (vim.endswith(f, '.txt') + and type == 'file' + and (not include or vim.tbl_contains(include, f)) + and (not exclude[f])) then + local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') + table.insert(rv, fullpath) + end + end + return rv +end + +-- Populates the helptags map. +local function get_helptags(help_dir) + local m = {} + -- Load a random help file to convince taglist() to do its job. + vim.cmd(string.format('split %s/api.txt', help_dir)) + vim.cmd('lcd %:p:h') + for _, item in ipairs(vim.fn.taglist('.*')) do + if vim.endswith(item.filename, '.txt') then + m[item.name] = item.filename + end + end + vim.cmd('q!') + return m +end + +-- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents. +-- +-- @returns lang_tree, bufnr +local function parse_buf(fname) + local buf + if type(fname) == 'string' then + vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. + buf = api.nvim_get_current_buf() + else + buf = fname + vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. + end + -- vim.treesitter.require_language('help', './build/lib/nvim/parser/help.so') + local lang_tree = vim.treesitter.get_parser(buf, 'help') + return lang_tree, buf +end + +-- Validates one :help file `fname`: +-- - checks that |tag| links point to valid helptags. +-- - recursively counts parse errors ("ERROR" nodes) +-- +-- @returns { invalid_tags: number, parse_errors: number } +local function validate_one(fname) + local stats = { + invalid_tags = {}, + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname) + for _, tree in ipairs(lang_tree:trees()) do + visit_validate(tree:root(), 0, tree, { buf = buf, fname = fname, }, stats) + end + lang_tree:destroy() + vim.cmd.close() + return { + invalid_tags = invalid_tags, + parse_errors = stats.parse_errors, + } +end + +-- Generates HTML from one :help file `fname` and writes the result to `to_fname`. +-- +-- @param fname Source :help file +-- @param to_fname Destination .html file +-- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace) +-- +-- @returns html, stats +local function gen_one(fname, to_fname, old) + local stats = { + noise_lines = {}, + parse_errors = {}, + } + local lang_tree, buf = parse_buf(fname) + local headings = {} -- Headings (for ToC). + local title = to_titlecase(basename_noext(fname)) + + local html = ([[ + + + + + + + + + + + + %s - Neovim docs + + + ]]):format(title) + + local logo_svg = [[ + + Neovim + + + + + + + + + + + + + + + + + + + + + + + + + + ]] + + local main = ([[ +
+ +
+ +
+
+

%s

+

+ + Nvim help pages, updated automatically + from source. + Parsing by tree-sitter-vimdoc. + +

+ ]]):format(logo_svg, title, vim.fs.basename(fname)) + for _, tree in ipairs(lang_tree:trees()) do + main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + end + main = main .. '
\n' + + local toc = [[ +
+ + + +
+ ]] + for _, heading in ipairs(headings) do + toc = toc .. ('\n'):format(to_heading_tag(heading), heading) + end + toc = toc .. '
\n' + + local bug_url = get_bug_url_nvim(fname, to_fname, 'TODO', nil) + local bug_link = string.format('(report docs bug...)', bug_url) + + local footer = ([[ +
+
+
+ Generated on %s from {%s} +
+
+ parse_errors: %d %s | noise_lines: %d +
+
+
+ ]]):format( + os.date('%Y-%m-%d %H:%M:%S'), commit, #stats.parse_errors, bug_link, + html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines) + + html = ('%s%s%s
\n%s\n\n'):format( + html, main, toc, footer) + vim.cmd('q!') + lang_tree:destroy() + return html, stats +end + +local function gen_css(fname) + local css = [[ + @media (min-width: 40em) { + .toc { + position: fixed; + left: 67%; + } + } + .toc { + /* max-width: 12rem; */ + } + .toc > div { + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + } + html { + scroll-behavior: auto; + } + h1, h2, h3, h4 { + font-family: sans-serif; + } + .help-body { + padding-bottom: 2em; + } + .help-line { + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + } + .help-item { + display: list-item; + margin-left: 1.5rem; /* padding-left: 1rem; */ + } + .help-para { + padding-top: 10px; + padding-bottom: 10px; + } + .old-help-line { + /* Tabs are used for alignment in old docs, so we must match Vim's 8-char expectation. */ + tab-size: 8; + white-space: pre; + font-size: .875em; + font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; + } + a.help-tag, a.help-tag:focus, a.help-tag:hover { + color: inherit; + text-decoration: none; + } + .help-tag { + color: gray; + } + h1 .help-tag, h2 .help-tag { + font-size: smaller; + } + .help-heading { + overflow: hidden; + white-space: nowrap; + display: flex; + justify-content: space-between; + } + /* The (right-aligned) "tags" part of a section heading. */ + .help-heading-tags { + margin-left: 10px; + } + .parse-error { + background-color: red; + } + .unknown-token { + color: black; + background-color: yellow; + } + pre { + /* Tabs are used in code_blocks only for indentation, not alignment, so we can aggressively shrink them. */ + tab-size: 2; + white-space: pre; + overflow: visible; + /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ + /* font-size: 14px; */ + /* border: 0px; */ + /* margin: 0px; */ + } + pre:hover, + .help-heading:hover { + overflow: visible; + } + .generator-stats { + color: gray; + font-size: smaller; + } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + ]] + tofile(fname, css) +end + +function M._test() + tagmap = get_helptags('./build/runtime/doc') + helpfiles = get_helpfiles() + + local function ok(cond, expected, actual) + assert((not expected and not actual) or (expected and actual), 'if "expected" is given, "actual" is also required') + if expected then + return assert(cond, ('expected %s, got: %s'):format(vim.inspect(expected), vim.inspect(actual))) + else + return assert(cond) + end + end + local function eq(expected, actual) + return ok(expected == actual, expected, actual) + end + + eq(119, #helpfiles) + ok(vim.tbl_count(tagmap) > 3000, '>3000', vim.tbl_count(tagmap)) + ok(vim.endswith(tagmap['vim.diagnostic.set()'], 'diagnostic.txt'), tagmap['vim.diagnostic.set()'], 'diagnostic.txt') + ok(vim.endswith(tagmap['%:s'], 'cmdline.txt'), tagmap['%:s'], 'cmdline.txt') + ok(is_noise([[vim:tw=78:isk=!-~,^*,^\|,^\":ts=8:noet:ft=help:norl:]])) + ok(is_noise([[ VIM REFERENCE MANUAL by Abe Lincoln ]])) + ok(not is_noise([[vim:tw=78]])) + + eq(0, get_indent('a')) + eq(1, get_indent(' a')) + eq(2, get_indent(' a\n b\n c\n')) + eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) + eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + + print('all tests passed') +end + +--- Generates HTML from :help docs located in `help_dir` and writes the result in `to_dir`. +--- +--- Example: +--- +--- gen('./build/runtime/doc', '/path/to/neovim.github.io/_site/doc/', {'api.txt', 'autocmd.txt', 'channel.txt'}, nil) +--- +--- @param help_dir string Source directory containing the :help files. Must run `make helptags` first. +--- @param to_dir string Target directory where the .html files will be written. +--- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} +--- +--- @returns info dict +function M.gen(help_dir, to_dir, include) + vim.validate{ + help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, + to_dir={to_dir, 's'}, + include={include, 't', true}, + } + + local err_count = 0 + tagmap = get_helptags(help_dir) + helpfiles = get_helpfiles(include) + + print(('output dir: %s'):format(to_dir)) + vim.fn.mkdir(to_dir, 'p') + gen_css(('%s/help.css'):format(to_dir)) + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) + local html, stats = gen_one(f, to_fname, not new_layout[helpfile]) + tofile(to_fname, html) + print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) + err_count = err_count + #stats.parse_errors + end + print(('generated %d html pages'):format(#helpfiles)) + print(('total errors: %d'):format(err_count)) + print(('invalid tags:\n%s'):format(vim.inspect(invalid_tags))) + + return { + helpfiles = helpfiles, + err_count = err_count, + invalid_tags = invalid_tags, + } +end + +-- Validates all :help files found in `help_dir`: +-- - checks that |tag| links point to valid helptags. +-- - recursively counts parse errors ("ERROR" nodes) +-- +-- This is 10x faster than gen(), for use in CI. +-- +-- @returns results dict +function M.validate(help_dir, include) + vim.validate{ + help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, + include={include, 't', true}, + } + local err_count = 0 + tagmap = get_helptags(help_dir) + helpfiles = get_helpfiles(include) + + for _, f in ipairs(helpfiles) do + local helpfile = vim.fs.basename(f) + local rv = validate_one(f) + print(('validated (%-4s errors): %s'):format(#rv.parse_errors, helpfile)) + err_count = err_count + #rv.parse_errors + end + + return { + helpfiles = helpfiles, + err_count = err_count, + invalid_tags = invalid_tags, + } +end + +return M diff --git a/scripts/gen_help_html.py b/scripts/gen_help_html.py deleted file mode 100644 index 0b8e77ac22..0000000000 --- a/scripts/gen_help_html.py +++ /dev/null @@ -1,389 +0,0 @@ -# Converts Vim/Nvim documentation to HTML. -# -# USAGE: -# 1. python3 scripts/gen_help_html.py runtime/doc/ ~/neovim.github.io/t/ -# 3. cd ~/neovim.github.io/ && jekyll serve --host 0.0.0.0 -# 2. Visit http://localhost:4000/t/help.txt.html -# -# Adapted from https://github.com/c4rlo/vimhelp/ -# License: MIT -# -# Copyright (c) 2016 Carlo Teubner -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import re -import urllib.parse -import datetime -import sys -from itertools import chain - -HEAD = """\ - - - - - -Nvim: {filename} -""" - -HEAD_END = '\n\n' - -INTRO = """ -

Nvim help files

-

-Nvim help pages{vers-note}. -Updated automatically -from the Nvim source. -

-""" - -VERSION_NOTE = ", current as of Nvim {version}" - -SITENAVI_LINKS = """ -Quick reference · -User manual · -Reference manual · -""" - -SITENAVI_LINKS_PLAIN = SITENAVI_LINKS.format(helptxt='help.txt.html') -SITENAVI_LINKS_WEB = SITENAVI_LINKS.format(helptxt='/') - -SITENAVI_PLAIN = '

' + SITENAVI_LINKS_PLAIN + '

' -SITENAVI_WEB = '

' + SITENAVI_LINKS_WEB + '

' - -SITENAVI_SEARCH = '
' + SITENAVI_LINKS_WEB + \ - '
' \ - '
' - -TEXTSTART = """ -
-
""" + (" " * 80) + """
-
-
-"""
-
-FOOTER = '
' - -FOOTER2 = """ - -
-
- - -""".format( - generated_date='{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()), - commit='?') - -RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)') - -PAT_WORDCHAR = '[!#-)+-{}~\xC0-\xFF]' - -PAT_HEADER = r'(^.*~$)' -PAT_GRAPHIC = r'(^.* `$)' -PAT_PIPEWORD = r'(?|.)?)' -PAT_SPECIAL = r'(<.+?>|\{.+?}|' \ - r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' \ - r'arg|arguments|ident|addr|group)]|' \ - r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])' -PAT_TITLE = r'(Vim version [0-9.a-z]+|VIM REFERENCE.*)' -PAT_NOTE = r'((? \t]+[a-zA-Z0-9/])' -PAT_WORD = r'((?$') -RE_EG_END = re.compile(r'\S') -RE_SECTION = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)') -RE_STARTAG = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)') -RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$') - - -class Link(object): - __slots__ = 'link_plain_same', 'link_pipe_same', \ - 'link_plain_foreign', 'link_pipe_foreign', \ - 'filename' - - def __init__(self, link_plain_same, link_plain_foreign, - link_pipe_same, link_pipe_foreign, filename): - self.link_plain_same = link_plain_same - self.link_plain_foreign = link_plain_foreign - self.link_pipe_same = link_pipe_same - self.link_pipe_foreign = link_pipe_foreign - self.filename = filename - - -class VimH2H(object): - def __init__(self, tags, version=None, is_web_version=True): - self._urls = {} - self._version = version - self._is_web_version = is_web_version - for line in RE_NEWLINE.split(tags): - m = RE_TAGLINE.match(line) - if m: - tag, filename = m.group(1, 2) - self.do_add_tag(filename, tag) - - def add_tags(self, filename, contents): - for match in RE_STARTAG.finditer(contents): - tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/') - self.do_add_tag(str(filename), tag) - - def do_add_tag(self, filename, tag): - tag_quoted = urllib.parse.quote_plus(tag) - - def mkpart1(doc): - return '' + html_escape[tag] + '' - - def mklinks(cssclass): - return (part1_same + cssclass + part2, - part1_foreign + cssclass + part2) - cssclass_plain = 'd' - m = RE_LINKWORD.match(tag) - if m: - opt, ctrl, special = m.groups() - if opt is not None: - cssclass_plain = 'o' - elif ctrl is not None: - cssclass_plain = 'k' - elif special is not None: - cssclass_plain = 's' - links_plain = mklinks(cssclass_plain) - links_pipe = mklinks('l') - self._urls[tag] = Link( - links_plain[0], links_plain[1], - links_pipe[0], links_pipe[1], - filename) - - def maplink(self, tag, curr_filename, css_class=None): - links = self._urls.get(tag) - if links is not None: - if links.filename == curr_filename: - if css_class == 'l': - return links.link_pipe_same - else: - return links.link_plain_same - else: - if css_class == 'l': - return links.link_pipe_foreign - else: - return links.link_plain_foreign - elif css_class is not None: - return '' + html_escape[tag] + \ - '' - else: - return html_escape[tag] - - def to_html(self, filename, contents, encoding): - out = [] - - inexample = 0 - filename = str(filename) - is_help_txt = (filename == 'help.txt') - last = '' - for line in RE_NEWLINE.split(contents): - line = line.rstrip('\r\n') - line_tabs = line - line = line.expandtabs() - if last == 'h1': - out.extend(('')) # XXX - out.extend(('

', line.rstrip(), '

\n')) - out.extend(('
'))
-                last = ''
-                continue
-            if RE_HRULE.match(line):
-                # out.extend(('', line, '\n'))
-                last = 'h1'
-                continue
-            if inexample == 2:
-                if RE_EG_END.match(line):
-                    inexample = 0
-                    if line[0] == '<':
-                        line = line[1:]
-                else:
-                    out.extend(('', html_escape[line],
-                                '\n'))
-                    continue
-            if RE_EG_START.match(line_tabs):
-                inexample = 1
-                line = line[0:-1]
-            if RE_SECTION.match(line_tabs):
-                m = RE_SECTION.match(line)
-                out.extend((r'', m.group(0), r''))
-                line = line[m.end():]
-            lastpos = 0
-            for match in RE_TAGWORD.finditer(line):
-                pos = match.start()
-                if pos > lastpos:
-                    out.append(html_escape[line[lastpos:pos]])
-                lastpos = match.end()
-                header, graphic, pipeword, starword, command, opt, ctrl, \
-                    special, title, note, url, word = match.groups()
-                if pipeword is not None:
-                    out.append(self.maplink(pipeword, filename, 'l'))
-                elif starword is not None:
-                    out.extend(('', html_escape[starword], ''))
-                elif command is not None:
-                    out.extend(('', html_escape[command],
-                                ''))
-                elif opt is not None:
-                    out.append(self.maplink(opt, filename, 'o'))
-                elif ctrl is not None:
-                    out.append(self.maplink(ctrl, filename, 'k'))
-                elif special is not None:
-                    out.append(self.maplink(special, filename, 's'))
-                elif title is not None:
-                    out.extend(('', html_escape[title],
-                                ''))
-                elif note is not None:
-                    out.extend(('', html_escape[note],
-                                ''))
-                elif header is not None:
-                    out.extend(('', html_escape[header[:-1]],
-                                ''))
-                elif graphic is not None:
-                    out.append(html_escape[graphic[:-2]])
-                elif url is not None:
-                    out.extend(('' +
-                                html_escape[url], ''))
-                elif word is not None:
-                    out.append(self.maplink(word, filename))
-            if lastpos < len(line):
-                out.append(html_escape[line[lastpos:]])
-            out.append('\n')
-            if inexample == 1:
-                inexample = 2
-
-        header = []
-        header.append(HEAD.format(encoding=encoding, filename=filename))
-        header.append(HEAD_END)
-        if self._is_web_version and is_help_txt:
-            vers_note = VERSION_NOTE.replace('{version}', self._version) \
-                if self._version else ''
-            header.append(INTRO.replace('{vers-note}', vers_note))
-        if self._is_web_version:
-            header.append(SITENAVI_SEARCH)
-            sitenavi_footer = SITENAVI_WEB
-        else:
-            header.append(SITENAVI_PLAIN)
-            sitenavi_footer = SITENAVI_PLAIN
-        header.append(TEXTSTART)
-        return ''.join(chain(header, out, (FOOTER, sitenavi_footer, FOOTER2)))
-
-
-class HtmlEscCache(dict):
-    def __missing__(self, key):
-        r = key.replace('&', '&') \
-               .replace('<', '<') \
-               .replace('>', '>')
-        self[key] = r
-        return r
-
-
-html_escape = HtmlEscCache()
-
-
-def slurp(filename):
-    try:
-        with open(filename, encoding='UTF-8') as f:
-            return f.read(), 'UTF-8'
-    except UnicodeError:
-        # 'ISO-8859-1' ?
-        with open(filename, encoding='latin-1') as f:
-            return f.read(), 'latin-1'
-
-
-def usage():
-    return "usage: " + sys.argv[0] + " IN_DIR OUT_DIR [BASENAMES...]"
-
-
-def main():
-    if len(sys.argv) < 3:
-        sys.exit(usage())
-
-    in_dir = sys.argv[1]
-    out_dir = sys.argv[2]
-    basenames = sys.argv[3:]
-
-    print("Processing tags...")
-    h2h = VimH2H(slurp(os.path.join(in_dir, 'tags'))[0], is_web_version=False)
-
-    if len(basenames) == 0:
-        basenames = os.listdir(in_dir)
-
-    for basename in basenames:
-        if os.path.splitext(basename)[1] != '.txt' and basename != 'tags':
-            print("Ignoring " + basename)
-            continue
-        print("Processing " + basename + "...")
-        path = os.path.join(in_dir, basename)
-        text, encoding = slurp(path)
-        outpath = os.path.join(out_dir, basename + '.html')
-        of = open(outpath, 'w')
-        of.write(h2h.to_html(basename, text, encoding))
-        of.close()
-
-
-main()
-- 
cgit 


From 1b60b5ec94001f18b70dbebf6c232c33209f11b5 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" 
Date: Sun, 25 Sep 2022 13:45:41 +0200
Subject: fix(gen_vimdoc.py): handle missing luajit

---
 scripts/gen_vimdoc.py  | 14 ++++++++++----
 scripts/lua2dox_filter |  7 ++++++-
 2 files changed, 16 insertions(+), 5 deletions(-)

(limited to 'scripts')

diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index 766c420c7d..306857ca6c 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -295,14 +295,16 @@ annotation_map = {
 # or if `cond()` is callable and returns True.
 def debug_this(o, cond=True):
     name = ''
+    if cond is False:
+        return
     if not isinstance(o, str):
         try:
             name = o.nodeName
             o = o.toprettyxml(indent='  ', newl='\n')
         except Exception:
             pass
-    if ((callable(cond) and cond())
-            or (not callable(cond) and cond)
+    if (cond is True
+            or (callable(cond) and cond())
             or (not callable(cond) and cond in o)):
         raise RuntimeError('xxx: {}\n{}'.format(name, o))
 
@@ -887,7 +889,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp):
 def fmt_doxygen_xml_as_vimhelp(filename, target):
     """Entrypoint for generating Vim :help from from Doxygen XML.
 
-    Returns 3 items:
+    Returns 2 items:
       1. Vim help text for functions found in `filename`.
       2. Vim help text for deprecated functions.
     """
@@ -1094,7 +1096,11 @@ def main(config, args):
                         fn_map_full.update(fn_map)
 
         if len(sections) == 0:
-            fail(f'no sections for target: {target}')
+            if target == 'lua':
+                fail(f'no sections for target: {target} (this usually means'
+                     + ' "luajit" was not found by scripts/lua2dox_filter)')
+            else:
+                fail(f'no sections for target: {target}')
         if len(sections) > len(CONFIG[target]['section_order']):
             raise RuntimeError(
                 'found new modules "{}"; update the "section_order" map'.format(
diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter
index 22484a807f..0b9f59b6ac 100755
--- a/scripts/lua2dox_filter
+++ b/scripts/lua2dox_filter
@@ -36,6 +36,10 @@ test_executable(){
 
 ##! \brief sets the lua interpreter
 set_lua(){
+	if test -z "${EXE}"; then
+		test_executable '.deps/usr/bin/luajit'
+	fi
+
 	if test -z "${EXE}"; then
 		test_executable 'luajit'
 	fi
@@ -73,7 +77,8 @@ do_readlink(){
 set_lua
 if test -z "${EXE}"
 then
-	echo "no lua interpreter available"
+	echo "no lua interpreter found"
+	exit 1
 else
 	BASENAME=`basename "$0"`
 	do_readlink "$0"
-- 
cgit 


From 7b4c49888a97c21f346b8337330fbc8e196b9cf8 Mon Sep 17 00:00:00 2001
From: "Justin M. Keyes" 
Date: Fri, 23 Sep 2022 14:26:59 +0200
Subject: feat(gen_help_html.lua): put commit-id in footer

---
 scripts/gen_help_html.lua | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'scripts')

diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua
index d79ece53f3..6e297814b8 100644
--- a/scripts/gen_help_html.lua
+++ b/scripts/gen_help_html.lua
@@ -16,7 +16,7 @@
 --
 -- NOTES:
 --   * gen() and validate() are the primary entrypoints. validate() only exists because gen() is too
---   slow (~1 min) to run in per-commit CI.
+--     slow (~1 min) to run in per-commit CI.
 --   * visit_node() is the core function used by gen() to traverse the document tree and produce HTML.
 --   * visit_validate() is the core function used by validate().
 --   * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout.
@@ -50,7 +50,6 @@ local tagmap = nil
 local helpfiles = nil
 local invalid_tags = {}
 
-local commit = '?'
 local api = vim.api
 local M = {}
 
@@ -477,7 +476,7 @@ end
 -- @param old boolean Preformat paragraphs (for old :help files which are full of arbitrary whitespace)
 --
 -- @returns html, stats
-local function gen_one(fname, to_fname, old)
+local function gen_one(fname, to_fname, old, commit)
   local stats = {
     noise_lines = {},
     parse_errors = {},
@@ -604,7 +603,7 @@ local function gen_one(fname, to_fname, old)
   

     

       

-        Generated on %s from {%s}
+        Generated at %s from %s
       

       

       parse_errors: %d %s | noise_lines: %d
@@ -612,7 +611,7 @@ local function gen_one(fname, to_fname, old)
     

   

   ]]):format(
-    os.date('%Y-%m-%d %H:%M:%S'), commit, #stats.parse_errors, bug_link,
+    os.date('%Y-%m-%d %H:%M'), commit, commit:sub(1, 7), #stats.parse_errors, bug_link,
     html_esc(table.concat(stats.noise_lines, '\n')), #stats.noise_lines)
 
   html = ('%s%s%s
\n%s\n\n'):format( @@ -763,11 +762,12 @@ end --- @param include table|nil Process only these filenames. Example: {'api.txt', 'autocmd.txt', 'channel.txt'} --- --- @returns info dict -function M.gen(help_dir, to_dir, include) +function M.gen(help_dir, to_dir, include, commit) vim.validate{ help_dir={help_dir, function(d) return vim.fn.isdirectory(d) == 1 end, 'valid directory'}, to_dir={to_dir, 's'}, include={include, 't', true}, + commit={commit, 's', true}, } local err_count = 0 @@ -781,7 +781,7 @@ function M.gen(help_dir, to_dir, include) for _, f in ipairs(helpfiles) do local helpfile = vim.fs.basename(f) local to_fname = ('%s/%s'):format(to_dir, get_helppage(helpfile)) - local html, stats = gen_one(f, to_fname, not new_layout[helpfile]) + local html, stats = gen_one(f, to_fname, not new_layout[helpfile], commit or '?') tofile(to_fname, html) print(('generated (%-4s errors): %-15s => %s'):format(#stats.parse_errors, helpfile, vim.fs.basename(to_fname))) err_count = err_count + #stats.parse_errors -- cgit From 16336c486ecb5a60e85a870904316308c7d7fc3f Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 25 Sep 2022 02:20:47 +0200 Subject: feat(gen_help_html.lua): adapt to new parser - adapt to parser changes from https://github.com/vigoux/tree-sitter-vimdoc/pull/16 - numerous other generator improvements --- scripts/gen_help_html.lua | 467 +++++++++++++++++++++++++++++----------------- 1 file changed, 295 insertions(+), 172 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 6e297814b8..fdbf5f605a 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -1,5 +1,7 @@ -- Converts Vim :help files to HTML. Validates |tag| links and document syntax (parser errors). -- +-- NOTE: :helptags checks for duplicate tags, whereas this script checks _links_ (to tags). +-- -- USAGE (GENERATE HTML): -- 1. Run `make helptags` first; this script depends on vim.fn.taglist(). -- 2. nvim -V1 -es --clean +"lua require('scripts.gen_help_html').gen('./build/runtime/doc/', 'target/dir/')" @@ -20,37 +22,19 @@ -- * visit_node() is the core function used by gen() to traverse the document tree and produce HTML. -- * visit_validate() is the core function used by validate(). -- * Files in `new_layout` will be generated with a "flow" layout instead of preformatted/fixed-width layout. --- --- parser bugs: --- * Should NOT be code_block: --- tab:xy The 'x' is always used, then 'y' as many times as will --- fit. Thus "tab:>-" displays: --- > --- >- --- >-- --- etc. --- --- tab:xyz The 'z' is always used, then 'x' is prepended, and --- then 'y' is used as many times as will fit. Thus --- "tab:<->" displays: --- > --- <> --- <-> --- <--> --- etc. --- * Should NOT be a "headline". Perhaps a "table" (or just "line"). --- expr5 and expr6 *expr5* *expr6* --- --------------- --- expr6 + expr6 Number addition, |List| or |Blob| concatenation *expr-+* --- expr6 - expr6 Number subtraction *expr--* --- expr6 . expr6 String concatenation *expr-.* --- expr6 .. expr6 String concatenation *expr-..* local tagmap = nil local helpfiles = nil -local invalid_tags = {} +local invalid_links = {} +local invalid_urls = {} +local invalid_spelling = {} +local spell_dict = { + Neovim = 'Nvim', + NeoVim = 'Nvim', + neovim = 'Nvim', + lua = 'Lua', +} -local api = vim.api local M = {} -- These files are generated with "flow" layout (non fixed-width, wrapped text paragraphs). @@ -59,16 +43,31 @@ local new_layout = { ['api.txt'] = true, ['channel.txt'] = true, ['develop.txt'] = true, + ['luaref.txt'] = true, ['nvim.txt'] = true, ['pi_health.txt'] = true, ['provider.txt'] = true, ['ui.txt'] = true, } --- TODO: treesitter gets stuck on these files... -local exclude = { - ['filetype.txt'] = true, - ['usr_24.txt'] = true, +-- TODO: These known invalid |links| require an update to the relevant docs. +local exclude_invalid = { + ["'previewpopup'"] = "quickref.txt", + ["'pvp'"] = "quickref.txt", + ["'string'"] = "eval.txt", + Query = "treesitter.txt", + ["eq?"] = "treesitter.txt", + ["lsp-request"] = "lsp.txt", + matchit = "vim_diff.txt", + ["matchit.txt"] = "help.txt", + ["set!"] = "treesitter.txt", + ["v:_null_blob"] = "builtin.txt", + ["v:_null_dict"] = "builtin.txt", + ["v:_null_list"] = "builtin.txt", + ["v:_null_string"] = "builtin.txt", + ["vim.lsp.buf_request()"] = "lsp.txt", + ["vim.lsp.util.get_progress_messages()"] = "lsp.txt", + ["vim.treesitter.start()"] = "treesitter.txt" } local function tofile(fname, text) @@ -82,10 +81,6 @@ local function tofile(fname, text) end local function html_esc(s) - if s:find(']local%-additions[*<]') + ) then + -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) + table.insert(noise_lines or {}, line) + return true + end + return false end --- Checks if a given line is a "noise" line that doesn't look good in HTML form. -local function is_noise(line) - return ( - line:find('Type .*gO.* to see the table of contents') - -- Title line of traditional :help pages. - -- Example: "NVIM REFERENCE MANUAL by ..." - or line:find('^%s*N?VIM REFERENCE MANUAL') - -- First line of traditional :help pages. - -- Example: "*api.txt* Nvim" - or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') - -- modeline - -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" - or line:find('^%s*vi[m]%:.*ft=help') - or line:find('^%s*vi[m]%:.*filetype=help') - ) -end - --- Creates a github issue URL at vigoux/tree-sitter-vimdoc with prefilled content. +-- Creates a github issue URL at neovim/tree-sitter-vimdoc with prefilled content. local function get_bug_url_vimdoc(fname, to_fname, sample_text) local this_url = string.format('https://neovim.io/doc/user/%s', vim.fs.basename(to_fname)) - local bug_url = ('https://github.com/vigoux/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' + local bug_url = ('https://github.com/neovim/tree-sitter-vimdoc/issues/new?labels=bug&title=parse+error%3A+' ..vim.fs.basename(fname) ..'+&body=Found+%60tree-sitter-vimdoc%60+parse+error+at%3A+' ..this_url @@ -237,31 +236,57 @@ local function getbuflinestr(node, bufnr, offset) return table.concat(lines, '\n') end --- Gets the whitespace just before `node` from the raw buffer text. --- Needed for preformatted `old` lines. -local function getws(node, bufnr) - local line1, c1, line2, _ = node:range() - local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] - local text_before = raw:sub(1, c1) - local leading_ws = text_before:match('%s+$') or '' - return leading_ws -end - -local function get_tagname(node, bufnr, link) - local node_name = (node.named and node:named()) and node:type() or nil +local function get_tagname(node, bufnr) local node_text = vim.treesitter.get_node_text(node, bufnr) - local tag = ((node_name == 'option' and node_text) - or (link and node_text:gsub('^|', ''):gsub('|$', '') or node_text:gsub('^%*', ''):gsub('%*$', ''))) - local helpfile = tag and vim.fs.basename(tagmap[tag]) or nil -- "api.txt" - local helppage = get_helppage(helpfile) -- "api.html" + local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(node_text) or node_text + local helpfile = vim.fs.basename(tagmap[tag]) or nil -- "api.txt" + local helppage = get_helppage(helpfile) -- "api.html" return helppage, tag end +-- Returns true if the given invalid tagname is a false positive. +local function ignore_invalid(s) + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + return not not (s:find('===') or exclude_invalid[s]) +end + +local function ignore_parse_error(s) + -- Ignore parse errors for unclosed codespan/optionlink/tag. + -- This is common in vimdocs and is treated as plaintext by :help. + return s:find("^[`'|*]") +end + +local function has_ancestor(node, ancestor_name) + local p = node + while true do + p = p:parent() + if not p or p:type() == 'help_file' then + break + elseif p:type() == ancestor_name then + return true + end + end + return false +end + +local function validate_link(node, bufnr, fname) + local helppage, tagname = get_tagname(node:child(1), bufnr, true) + if not has_ancestor(node, 'column_heading') and not node:has_error() and not tagmap[tagname] and not ignore_invalid(tagname) then + invalid_links[tagname] = vim.fs.basename(fname) + end + return helppage, tagname +end + -- Traverses the tree at `root` and checks that |tag| links point to valid helptags. local function visit_validate(root, level, lang_tree, opt, stats) level = level or 0 local node_name = (root.named and root:named()) and root:type() or nil local toplevel = level < 1 + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) + end + local text = trim(node_text()) if root:child_count() > 0 then for node, _ in root:iter_children() do @@ -272,14 +297,26 @@ local function visit_validate(root, level, lang_tree, opt, stats) end if node_name == 'ERROR' then + if ignore_parse_error(text) then + return + end -- Store the raw text to give context to the bug report. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) - elseif node_name == 'hotlink' or node_name == 'option' then - local _, tagname = get_tagname(root, opt.buf, true) - if not root:has_error() and not tagmap[tagname] then - invalid_tags[tagname] = vim.fs.basename(opt.fname) + elseif node_name == 'word' or node_name == 'uppercase_name' then + if spell_dict[text] then + if not invalid_spelling[text] then + invalid_spelling[text] = { vim.fs.basename(opt.fname) } + else + table.insert(invalid_spelling[text], vim.fs.basename(opt.fname)) + end end + elseif node_name == 'url' then + if text:find('http%:') then + invalid_urls[text] = vim.fs.basename(opt.fname) + end + elseif node_name == 'taglink' or node_name == 'optionlink' then + local _, _ = validate_link(root, opt.buf, opt.fname) end end @@ -296,19 +333,18 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local parent = root:parent() and root:parent():type() or nil local text = '' local toplevel = level < 1 - local function node_text() - return vim.treesitter.get_node_text(root, opt.buf) + local function node_text(node) + return vim.treesitter.get_node_text(node or root, opt.buf) end - if root:child_count() == 0 then + if root:child_count() == 0 or node_name == 'ERROR' then text = node_text() else -- Process children and join them with whitespace. for node, _ in root:iter_children() do if node:named() then local r = visit_node(node, level + 1, lang_tree, headings, opt, stats) - local ws = r == '' and '' or ((opt.old and (node:type() == 'word' or not node:named())) and getws(node, opt.buf) or ' ') - text = string.format('%s%s%s', text, ws, r) + text = string.format('%s%s', text, r) end end end @@ -316,82 +352,112 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if node_name == 'help_file' then -- root node return text + elseif node_name == 'url' then + return ('%s'):format(trimmed, trimmed) elseif node_name == 'word' or node_name == 'uppercase_name' then - if parent == 'headline' then - -- Start a new heading item, or update the current one. - local n = (prev == nil or #headings == 0) and #headings + 1 or #headings - headings[n] = string.format('%s%s', headings[n] and headings[n]..' ' or '', text) - end - return html_esc(text) - elseif node_name == 'headline' then - return ('

%s

\n'):format(to_heading_tag(headings[#headings]), text) + elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then + if is_noise(text, stats.noise_lines) then + return '' -- Discard common "noise" lines. + end + -- Remove "===" and tags from ToC text. + local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) + if node_name == 'h1' or #headings == 0 then + table.insert(headings, { name = hname, subheadings = {}, }) + else + table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, }) + end + local el = node_name == 'h1' and 'h2' or 'h3' + return ('<%s class="help-heading">%s\n'):format(to_heading_tag(hname), el, text, el) elseif node_name == 'column_heading' or node_name == 'column_name' then - return ('

%s

\n'):format(trimmed) - elseif node_name == 'line' then - -- TODO: remove these "sibling inspection" hacks once the parser provides structured info - -- about paragraphs and listitems: https://github.com/vigoux/tree-sitter-vimdoc/issues/12 - local next_text = root:next_sibling() and vim.treesitter.get_node_text(root:next_sibling(), opt.buf) or '' - local li = startswith_bullet(text) -- Listitem? - local next_li = startswith_bullet(next_text) -- Next is listitem? - -- Close the paragraph/listitem if the next sibling is not a line. - local close = (next_ ~= 'line' or next_li or is_blank(next_text)) and '
\n' or '' - - -- HACK: discard common "noise" lines. - if is_noise(text) then - table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) - return (opt.old or prev ~= 'line') and '' or close + if root:has_error() then + return text + end + return ('
%s
'):format(trimmed) + elseif node_name == 'block' then + if is_blank(text) then + return '' end - if opt.old then - -- XXX: Treat old docs as preformatted. Until those docs are "fixed" or we get better info - -- from tree-sitter-vimdoc, this avoids broken layout for legacy docs. - return ('
%s
\n'):format(text) + -- XXX: Treat old docs as preformatted; random indentation is used for layout there. + return ('
%s
\n'):format(text) end - - if li then - return string.format('
%s%s', trim_bullet(expandtabs(text)), close) + return string.format('
\n%s\n
\n', text) + elseif node_name == 'line' then + local sib = root:prev_sibling() + local sib_last = sib and sib:named_child(sib:named_child_count() - 1) + local in_li = false + + -- XXX: parser bug: (codeblock) without terminating "<" consumes first char of the next (line). Recover it here. + local recovered = (sib_last and sib_last:type() == 'codeblock') and node_text(root:prev_sibling()):sub(-1) or '' + recovered = recovered == '<' and '' or html_esc(recovered) + + -- XXX: see if we are currently "in" a listitem. + while sib ~= nil and not in_li do + in_li = (sib:type() == 'line_li') + sib = sib:prev_sibling() end - if prev ~= 'line' then -- Start a new paragraph. - return string.format('
%s%s', expandtabs(text), close) + + -- Close the current listitem. + local close = (in_li and next_ ~= 'line') and '
' or '' + + if is_blank(text) or is_noise(text, stats.noise_lines) then + return close -- Discard common "noise" lines. end - -- Continue in the current paragraph/listitem. - return string.format('%s%s', expandtabs(text), close) - elseif node_name == 'hotlink' or node_name == 'option' then - local helppage, tagname = get_tagname(root, opt.buf, true) - if not root:has_error() and not tagmap[tagname] then - invalid_tags[tagname] = vim.fs.basename(opt.fname) + local div = (root:child(0) and root:child(0):type() == 'column_heading') or close ~= '' + return string.format('%s%s%s%s', recovered, div and trim(text) or text, div and '' or '\n', close) + elseif node_name == 'line_li' then + -- Close the listitem immediately if the next sibling is not a line. + local close = (next_ ~= 'line') and '
' or '' + return string.format('
%s %s', trim_bullet(text), close) + elseif node_name == 'taglink' or node_name == 'optionlink' then + if root:has_error() then + return text + end + local helppage, tagname = validate_link(root, opt.buf, opt.fname) + return (' %s'):format(helppage, url_encode(tagname), html_esc(tagname)) + elseif node_name == 'codespan' then + if root:has_error() then + return text end - return ('%s'):format(helppage, url_encode(tagname), html_esc(tagname)) - elseif node_name == 'backtick' then - return ('%s'):format(html_esc(text)) + return (' %s'):format(text) elseif node_name == 'argument' then - return ('{%s}'):format(html_esc(trimmed)) - elseif node_name == 'code_block' then - return ('
\n%s
\n'):format(html_esc(trim_indent(trim_gt_lt(text)))) + return (' {%s}'):format(trimmed) + elseif node_name == 'codeblock' then + return ('
%s
'):format(html_esc(trim_indent(trim_gt_lt(text)))) elseif node_name == 'tag' then -- anchor - local _, tagname = get_tagname(root, opt.buf, false) - local s = ('%s'):format(url_encode(tagname), trimmed) - if parent == 'headline' and prev ~= 'tag' then + if root:has_error() then + return text + end + local in_heading = (parent == 'h1' or parent == 'h2') + local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' + local tagname = node_text(root:child(1)) + if vim.tbl_count(stats.first_tags) < 2 then + -- First 2 tags in the doc will be anchored at the main heading. + table.insert(stats.first_tags, tagname) + return '' + end + local s = (' %s'):format(url_encode(tagname), cssclass, trimmed) + if in_heading and prev ~= 'tag' then -- Start the container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. --

foo bartag1 tag2

return string.format('%s', s) - elseif parent == 'headline' and next_ == nil then + elseif in_heading and next_ == nil then -- End the container for tags in a heading. return string.format('%s', s) end return s elseif node_name == 'ERROR' then + if ignore_parse_error(trimmed) then + return text + end + -- Store the raw text to give context to the bug report. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) - if prev == 'ERROR' then - -- Avoid trashing the text with cascading errors. - return trimmed, ('parse-error:"%s"'):format(node_text()) - end - return ('%s'):format( + return ('%s'):format( get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) else -- Unknown token. local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' @@ -406,8 +472,7 @@ local function get_helpfiles(include) for f, type in vim.fs.dir(dir) do if (vim.endswith(f, '.txt') and type == 'file' - and (not include or vim.tbl_contains(include, f)) - and (not exclude[f])) then + and (not include or vim.tbl_contains(include, f))) then local fullpath = vim.fn.fnamemodify(('%s/%s'):format(dir, f), ':p') table.insert(rv, fullpath) end @@ -430,6 +495,13 @@ local function get_helptags(help_dir) return m end +-- Use the help.so parser defined in the build, not whatever happens to be installed on the system. +local function ensure_runtimepath() + if not vim.o.runtimepath:find('build/lib/nvim/') then + vim.cmd[[set runtimepath^=./build/lib/nvim/]] + end +end + -- Opens `fname` in a buffer and gets a treesitter parser for the buffer contents. -- -- @returns lang_tree, bufnr @@ -437,7 +509,7 @@ local function parse_buf(fname) local buf if type(fname) == 'string' then vim.cmd('split '..vim.fn.fnameescape(fname)) -- Filename. - buf = api.nvim_get_current_buf() + buf = vim.api.nvim_get_current_buf() else buf = fname vim.cmd('sbuffer '..tostring(fname)) -- Buffer number. @@ -451,10 +523,9 @@ end -- - checks that |tag| links point to valid helptags. -- - recursively counts parse errors ("ERROR" nodes) -- --- @returns { invalid_tags: number, parse_errors: number } +-- @returns { invalid_links: number, parse_errors: number } local function validate_one(fname) local stats = { - invalid_tags = {}, parse_errors = {}, } local lang_tree, buf = parse_buf(fname) @@ -463,10 +534,7 @@ local function validate_one(fname) end lang_tree:destroy() vim.cmd.close() - return { - invalid_tags = invalid_tags, - parse_errors = stats.parse_errors, - } + return stats end -- Generates HTML from one :help file `fname` and writes the result to `to_fname`. @@ -480,9 +548,10 @@ local function gen_one(fname, to_fname, old, commit) local stats = { noise_lines = {}, parse_errors = {}, + first_tags = {}, -- Track the first few tags in doc. } local lang_tree, buf = parse_buf(fname) - local headings = {} -- Headings (for ToC). + local headings = {} -- Headings (for ToC). 2-dimensional: h1 contains h2/h3. local title = to_titlecase(basename_noext(fname)) local html = ([[ @@ -555,7 +624,12 @@ local function gen_one(fname, to_fname, old, commit) ]] - local main = ([[ + local main = '' + for _, tree in ipairs(lang_tree:trees()) do + main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + end + + main = ([[
' or '' - return string.format('
%s %s', trim_bullet(text), close) + return string.format('
%s
', margin, text) elseif node_name == 'taglink' or node_name == 'optionlink' then if root:has_error() then return text @@ -429,7 +438,10 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) elseif node_name == 'codeblock' then - return ('
%s
'):format(html_esc(trim_indent(trim_gt_lt(text)))) + if is_blank(text) then + return '' + end + return ('
%s
'):format(html_esc(trim(trim_indent(text), 2))) elseif node_name == 'tag' then -- anchor if root:has_error() then return text @@ -630,7 +642,9 @@ local function gen_one(fname, to_fname, old, commit) local main = '' for _, tree in ipairs(lang_tree:trees()) do - main = main .. (visit_node(tree:root(), 0, tree, headings, { buf = buf, old = old, fname = fname, to_fname = to_fname }, stats)) + main = main .. (visit_node(tree:root(), 0, tree, headings, + { buf = buf, old = old, fname = fname, to_fname = to_fname, indent = 1, }, + stats)) end main = ([[ @@ -718,6 +732,17 @@ local function gen_css(fname) position: fixed; left: 67%; } + .golden-grid { + display: grid; + grid-template-columns: 65% auto; + grid-gap: 1em; + } + } + @media (max-width: 40em) { + .golden-grid { + /* Disable grid for narrow viewport (mobile phone). */ + display: block; + } } @media (prefers-color-scheme: dark) { :root { @@ -831,11 +856,6 @@ local function gen_css(fname) color: gray; font-size: smaller; } - .golden-grid { - display: grid; - grid-template-columns: 65% auto; - grid-gap: 1em; - } ]] tofile(fname, css) end @@ -869,6 +889,22 @@ function M._test() eq(5, get_indent(' a\n \n b\n c\n d\n e\n')) eq('a\n \n b\n c\n d\n e\n', trim_indent(' a\n \n b\n c\n d\n e\n')) + local fixed_url, removed_chars = fix_url('https://example.com).') + eq('https://example.com', fixed_url) + eq(').', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.)') + eq('https://example.com.', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com.') + eq('https://example.com', fixed_url) + eq('.', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com)') + eq('https://example.com', fixed_url) + eq(')', removed_chars) + fixed_url, removed_chars = fix_url('https://example.com') + eq('https://example.com', fixed_url) + eq('', removed_chars) + print('all tests passed') end -- cgit From 6abb48105135ce3ae7eda22334f8104c5ddf20ce Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 4 Oct 2022 15:15:06 +0200 Subject: fix(docs): missing "(" in :help HTML Problem: Since https://github.com/neovim/tree-sitter-vimdoc/commit/eba7b5b646546d9fed9b40b2c72b9cc0048f1dfa any opening paren and its leading whitespace " (" are missing in the generated HTML. Example: Use ":qa!" (careful, all changes are lost!). ^^missing Position the cursor on a tag (e.g. bars) and hit CTRL-]. ^^missing Solution: The main recursive loop only processes named children, so check named_child_count() instead of child_count(). Then anonymous nodes won't get lost. --- scripts/gen_help_html.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 3141ff3cdf..ccfa2e567e 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -350,7 +350,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return node_text():match('^%s+') or '' end - if root:child_count() == 0 or node_name == 'ERROR' then + if root:named_child_count() == 0 or node_name == 'ERROR' then text = node_text() else -- Process children and join them with whitespace. -- cgit From 18afacee1d98b9987391b8bdef08282fb156fa88 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Wed, 5 Oct 2022 08:15:55 -0400 Subject: feat(docs): format parameters as a list #20485 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: The {foo} parameters listed in `:help api` and similar generated docs, are intended to be a "list" but they aren't prefixed with a list symbol. This prevents parsers from understanding the list, which forces generators like `gen_help_html.lua` to use hard-wrapped/preformatted layout instead of a soft-wrapped "flow" layout. Solution: Modify gen_vimdoc.py to prefix {foo} parameters with a "•" symbol. --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 306857ca6c..2612260226 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -673,7 +673,7 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='', max_name_len = max_name(m.keys()) + 4 out = '' for name, desc in m.items(): - name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) out += '{}{}\n'.format(name, desc) return out.rstrip() -- cgit From f7b175e049db9262a45ee1c5eb41a38bd5b8ac38 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 6 Oct 2022 09:16:00 -0400 Subject: fix(docs-html): keycodes, taglinks, column_heading #20498 Problem: - Docs HTML: "foo ~" headings (column_heading) are not aligned with their table columns/contents because the leading whitespace is not emitted. - taglinks starting with hyphen like |-x| were not recognized. - keycodes like `` and `CTRL-x` were not recognized. - ToC is not scrollable. Solution: - Add ws() to the column_heading case. - Update help parser to latest version - supports `keycode` - fixes for taglink, argument - Update .toc CSS. https://github.com/neovim/neovim.github.io/issues/297 fix https://github.com/neovim/neovim.github.io/issues/297 --- scripts/gen_help_html.lua | 54 +++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 21 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index ccfa2e567e..15269ce175 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -208,7 +208,7 @@ local function get_helppage(f) return 'index.html' end - return f:gsub('%.txt$', '.html') + return (f:gsub('%.txt$', '.html')) end -- Counts leading spaces (tab=8) to decide the indent size of multiline text. @@ -255,9 +255,13 @@ end -- Returns true if the given invalid tagname is a false positive. local function ignore_invalid(s) - -- Strings like |~/====| appear in various places and the parser thinks they are links, but they - -- are just table borders. - return not not (s:find('===') or exclude_invalid[s]) + return not not ( + exclude_invalid[s] + -- Strings like |~/====| appear in various places and the parser thinks they are links, but they + -- are just table borders. + or s:find('===') + or s:find('---') + ) end local function ignore_parse_error(s) @@ -281,10 +285,14 @@ end local function validate_link(node, bufnr, fname) local helppage, tagname = get_tagname(node:child(1), bufnr) - if not has_ancestor(node, 'column_heading') and not node:has_error() and not tagmap[tagname] and not ignore_invalid(tagname) then - invalid_links[tagname] = vim.fs.basename(fname) + local ignored = false + if not tagmap[tagname] then + ignored = has_ancestor(node, 'column_heading') or node:has_error() or ignore_invalid(tagname) + if not ignored then + invalid_links[tagname] = vim.fs.basename(fname) + end end - return helppage, tagname + return helppage, tagname, ignored end -- Traverses the tree at `root` and checks that |tag| links point to valid helptags. @@ -325,7 +333,7 @@ local function visit_validate(root, level, lang_tree, opt, stats) invalid_urls[text] = vim.fs.basename(opt.fname) end elseif node_name == 'taglink' or node_name == 'optionlink' then - local _, _ = validate_link(root, opt.buf, opt.fname) + local _, _, _ = validate_link(root, opt.buf, opt.fname) end end @@ -341,7 +349,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) -- Parent kind (string). local parent = root:parent() and root:parent():type() or nil local text = '' - local toplevel = level < 1 + local trimmed local function node_text(node) return vim.treesitter.get_node_text(node or root, opt.buf) end @@ -352,6 +360,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:named_child_count() == 0 or node_name == 'ERROR' then text = node_text() + trimmed = html_esc(trim(text)) + text = html_esc(text) else -- Process children and join them with whitespace. for node, _ in root:iter_children() do @@ -360,8 +370,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) text = string.format('%s%s', text, r) end end + trimmed = trim(text) end - local trimmed = trim(text) if node_name == 'help_file' then -- root node return text @@ -369,7 +379,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local fixed_url, removed_chars = fix_url(trimmed) return ('%s%s%s'):format(ws(), fixed_url, fixed_url, removed_chars) elseif node_name == 'word' or node_name == 'uppercase_name' then - return html_esc(text) + return text elseif node_name == 'h1' or node_name == 'h2' or node_name == 'h3' then if is_noise(text, stats.noise_lines) then return '' -- Discard common "noise" lines. @@ -387,7 +397,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - return ('
%s
'):format(trimmed) + return ('
%s%s
'):format(ws(), trimmed) elseif node_name == 'block' then if is_blank(text) then return '' @@ -425,28 +435,28 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return string.format('
%s
', margin, text) elseif node_name == 'taglink' or node_name == 'optionlink' then - if root:has_error() then + local helppage, tagname, ignored = validate_link(root, opt.buf, opt.fname) + if ignored then return text end - local helppage, tagname = validate_link(root, opt.buf, opt.fname) return ('%s%s'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) - elseif node_name == 'codespan' then + elseif vim.tbl_contains({'codespan', 'keycode'}, node_name) then if root:has_error() then return text end - return ('%s%s'):format(ws(), text) + return ('%s%s'):format(ws(), trimmed) elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) elseif node_name == 'codeblock' then if is_blank(text) then return '' end - return ('
%s
'):format(html_esc(trim(trim_indent(text), 2))) + return ('
%s
'):format(trim(trim_indent(text), 2)) elseif node_name == 'tag' then -- anchor if root:has_error() then return text end - local in_heading = (parent == 'h1' or parent == 'h2') + local in_heading = vim.tbl_count({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' local tagname = node_text(root:child(1)) if vim.tbl_count(stats.first_tags) < 2 then @@ -471,12 +481,12 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end -- Store the raw text to give context to the bug report. - local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' table.insert(stats.parse_errors, sample_text) return ('%s'):format( get_bug_url_vimdoc(opt.fname, opt.to_fname, sample_text), trimmed) else -- Unknown token. - local sample_text = not toplevel and getbuflinestr(root, opt.buf, 3) or '[top level!]' + local sample_text = level > 0 and getbuflinestr(root, opt.buf, 3) or '[top level!]' return ('%s'):format( node_name, get_bug_url_nvim(opt.fname, opt.to_fname, sample_text, node_name), trimmed), ('unknown-token:"%s"'):format(node_name) end @@ -751,6 +761,8 @@ local function gen_css(fname) } .toc { /* max-width: 12rem; */ + height: 95%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ } .toc > div { text-overflow: ellipsis; @@ -809,7 +821,7 @@ local function gen_css(fname) .help-tag-right { color: var(--tag-color); } - h1 .help-tag, h2 .help-tag { + h1 .help-tag, h2 .help-tag, h3 .help-tag { font-size: smaller; } .help-heading { -- cgit From 453fffde16fe25b818a77eefc7e8e138a00aaf61 Mon Sep 17 00:00:00 2001 From: Folke Lemaitre Date: Wed, 28 Sep 2022 08:56:47 +0200 Subject: feat: added support for @generic to lua2dox.lua --- scripts/lua2dox.lua | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 6a206066b8..7d8e393cc9 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -387,10 +387,13 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local state = '' -- luacheck: ignore 231 variable is set but never accessed. local offset = 0 + local generic = {} + local l = 0 while not (inStream:eof()) do line = string_trim(inStream:getLine()) -- TCore_Debug_show_var('inStream',inStream) -- TCore_Debug_show_var('line',line ) + l = l + 1 if string.sub(line,1,2) == '--' then -- it's a comment -- Allow people to write style similar to EmmyLua (since they are basically the same) -- instead of silently skipping things that start with --- @@ -406,11 +409,25 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic_split = string_split(magic, ' ') + if magic_split[1] == "generic" then + local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") + if generic_type == "" then + generic_type = "any" + end + generic[generic_name] = generic_type + end + local type_index = 2 if magic_split[1] == 'param' then type_index = type_index + 1 end + if magic_split[type_index] then + for k, v in pairs(generic) do + magic_split[type_index] = magic_split[type_index]:gsub(k, v) + end + end + if magic_split[type_index] == 'number' or magic_split[type_index] == 'number|nil' or magic_split[type_index] == 'string' or @@ -426,8 +443,10 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) end magic = table.concat(magic_split, ' ') - outStream:writeln('/// @' .. magic) - fn_magic = checkComment4fn(fn_magic,magic) + if magic_split[1] ~= "generic" then + outStream:writeln('/// @' .. magic) + fn_magic = checkComment4fn(fn_magic,magic) + end elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment local comment = string.sub(line,4) outStream:writeln('/// '.. comment) @@ -467,6 +486,7 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) fn_magic = nil end elseif string.find(line, '^function') or string.find(line, '^local%s+function') then + generic = {} state = 'in_function' -- it's a function local pos_fn = string.find(line,'function') -- function -- cgit From 24a1c7f556bba35a9c31c2fdd19cf4b8c00a4395 Mon Sep 17 00:00:00 2001 From: Folke Lemaitre Date: Wed, 28 Sep 2022 09:11:21 +0200 Subject: feat: added support for optional params to lua2dox --- scripts/lua2dox.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 7d8e393cc9..d93bb6080f 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -410,7 +410,7 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic_split = string_split(magic, ' ') if magic_split[1] == "generic" then - local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") + local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") if generic_type == "" then generic_type = "any" end @@ -420,6 +420,10 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local type_index = 2 if magic_split[1] == 'param' then type_index = type_index + 1 + if magic_split[type_index] and magic_split[2]:find("%?$") then + magic_split[type_index] = magic_split[type_index] .. "|nil" + magic_split[2] = magic_split[2]:sub(1, -2) + end end if magic_split[type_index] then -- cgit From 1da7b4eb699fb04cc97dec389470fd0fbd64091d Mon Sep 17 00:00:00 2001 From: Folke Lemaitre Date: Wed, 28 Sep 2022 13:22:08 +0200 Subject: feat: added support for specifying types for lua2dox --- scripts/lua2dox.lua | 69 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 28 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index d93bb6080f..674b42a699 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -159,6 +159,7 @@ end --! \param Str --! \param Pattern --! \returns table of string fragments +---@return string[] local function string_split(Str, Pattern) local splitStr = {} local fpat = "(.-)" .. Pattern @@ -369,6 +370,9 @@ local function checkComment4fn(Fn_magic,MagicLines) return fn_magic end + +local types = {"number", "string", "table", "list", "boolean", "function"} + --! \brief run the filter function TLua2DoX_filter.readfile(this,AppStamp,Filename) local inStream = TStream_Read() @@ -408,6 +412,19 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic = string.sub(line, 4 + offset) local magic_split = string_split(magic, ' ') + if magic_split[1] == 'param' then + for _, type in ipairs(types) do + magic = magic:gsub("^param%s+([a-zA-Z_?]+)%s+.*%((" .. type .. ")%)", "param %1 %2" ) + magic = magic:gsub("^param%s+([a-zA-Z_?]+)%s+.*%((" .. type .. "|nil)%)", "param %1 %2" ) + end + magic_split = string_split(magic, ' ') + elseif magic_split[1] == 'return' then + for _, type in ipairs(types) do + magic = magic:gsub("^return%s+.*%((" .. type .. ")%)", "return %1" ) + magic = magic:gsub("^return%s+.*%((" .. type .. "|nil)%)", "return %1" ) + end + magic_split = string_split(magic, ' ') + end if magic_split[1] == "generic" then local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") @@ -415,39 +432,35 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) generic_type = "any" end generic[generic_name] = generic_type - end - - local type_index = 2 - if magic_split[1] == 'param' then - type_index = type_index + 1 - if magic_split[type_index] and magic_split[2]:find("%?$") then - magic_split[type_index] = magic_split[type_index] .. "|nil" - magic_split[2] = magic_split[2]:sub(1, -2) + else + local type_index = 2 + if magic_split[1] == 'param' then + type_index = type_index + 1 end - end - if magic_split[type_index] then - for k, v in pairs(generic) do - magic_split[type_index] = magic_split[type_index]:gsub(k, v) + if magic_split[type_index] then + -- fix optional parameters + if magic_split[type_index] and magic_split[2]:find("%?$") then + if not magic_split[type_index]:find("nil") then + magic_split[type_index] = magic_split[type_index] .. "|nil" + end + magic_split[2] = magic_split[2]:sub(1, -2) + end + -- replace generic types + if magic_split[type_index] then + for k, v in pairs(generic) do + magic_split[type_index] = magic_split[type_index]:gsub(k, v) + end + end + -- surround some types by () + for _, type in ipairs(types) do + magic_split[type_index] = magic_split[type_index]:gsub("^(" .. type .. "|nil):?$", "(%1)") + magic_split[type_index] = magic_split[type_index]:gsub("^(" .. type .. "):?$", "(%1)") + end end - end - if magic_split[type_index] == 'number' or - magic_split[type_index] == 'number|nil' or - magic_split[type_index] == 'string' or - magic_split[type_index] == 'string|nil' or - magic_split[type_index] == 'table' or - magic_split[type_index] == 'table|nil' or - magic_split[type_index] == 'boolean' or - magic_split[type_index] == 'boolean|nil' or - magic_split[type_index] == 'function' or - magic_split[type_index] == 'function|nil' - then - magic_split[type_index] = '(' .. magic_split[type_index] .. ')' - end - magic = table.concat(magic_split, ' ') + magic = table.concat(magic_split, ' ') - if magic_split[1] ~= "generic" then outStream:writeln('/// @' .. magic) fn_magic = checkComment4fn(fn_magic,magic) end -- cgit From 93117b358778487d85665035b6a95976eee70d9c Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Sat, 8 Oct 2022 17:49:09 +0200 Subject: docs(news): add news.txt and link from README (#20426) --- scripts/gen_help_html.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 15269ce175..540caa2ae3 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -44,6 +44,7 @@ local new_layout = { ['channel.txt'] = true, ['develop.txt'] = true, ['luaref.txt'] = true, + ['news.txt'] = true, ['nvim.txt'] = true, ['pi_health.txt'] = true, ['provider.txt'] = true, -- cgit From 09dffb9db7d16496e55e86f78ab60241533d86f6 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 9 Oct 2022 08:21:52 -0400 Subject: docs: various #12823 - increase python line-length limit from 88 => 100. - gen_help_html: fix bug in "tag" case (tbl_count => tbl_contains) ref #15632 fix #18215 fix #18479 fix #20527 fix #20532 Co-authored-by: Ben Weedon --- scripts/bump-deps.sh | 6 +++++- scripts/gen_help_html.lua | 14 +++++++++----- scripts/gen_vimdoc.py | 6 ++---- 3 files changed, 16 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/bump-deps.sh b/scripts/bump-deps.sh index 85c7f72700..e725608b39 100755 --- a/scripts/bump-deps.sh +++ b/scripts/bump-deps.sh @@ -17,9 +17,13 @@ BASENAME="$(basename "${0}")" readonly BASENAME usage() { - echo "Bump Neovim dependencies" + echo "Bump Nvim dependencies" echo echo "Usage: ${BASENAME} [ -h | --pr | --branch= | --dep= ]" + echo " Update a dependency:" + echo " ./scripts/bump-deps.sh --dep Luv --version 1.43.0-0" + echo " Create a PR:" + echo " ./scripts/bump-deps.sh --pr" echo echo "Options:" echo " -h show this message and exit." diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 540caa2ae3..13601b91f5 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -33,6 +33,7 @@ local spell_dict = { NeoVim = 'Nvim', neovim = 'Nvim', lua = 'Lua', + VimL = 'Vimscript', } local M = {} @@ -42,7 +43,9 @@ local M = {} local new_layout = { ['api.txt'] = true, ['channel.txt'] = true, + ['deprecated.txt'] = true, ['develop.txt'] = true, + ['lua.txt'] = true, ['luaref.txt'] = true, ['news.txt'] = true, ['nvim.txt'] = true, @@ -158,8 +161,8 @@ local function is_noise(line, noise_lines) or line:find('%s*%*?[a-zA-Z]+%.txt%*?%s+N?[vV]im%s*$') -- modeline -- Example: "vim:tw=78:ts=8:sw=4:sts=4:et:ft=help:norl:" - or line:find('^%s*vi[m]%:.*ft=help') - or line:find('^%s*vi[m]%:.*filetype=help') + or line:find('^%s*vim?%:.*ft=help') + or line:find('^%s*vim?%:.*filetype=help') or line:find('[*>]local%-additions[*<]') ) then -- table.insert(stats.noise_lines, getbuflinestr(root, opt.buf, 0)) @@ -457,7 +460,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - local in_heading = vim.tbl_count({'h1', 'h2', 'h3'}, parent) + local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' local tagname = node_text(root:child(1)) if vim.tbl_count(stats.first_tags) < 2 then @@ -465,7 +468,8 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) table.insert(stats.first_tags, tagname) return '' end - local s = ('%s%s'):format(ws(), url_encode(tagname), cssclass, trimmed) + local el = in_heading and 'span' or 'code' + local s = ('%s<%s class="%s">%s'):format(ws(), url_encode(tagname), el, cssclass, trimmed, el) if in_heading and prev ~= 'tag' then -- Start the container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. @@ -762,7 +766,7 @@ local function gen_css(fname) } .toc { /* max-width: 12rem; */ - height: 95%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ + height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ overflow: auto; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ } .toc > div { diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 2612260226..3ee9d8b5dd 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -2,9 +2,7 @@ """Generates Nvim :help docs from C/Lua docstrings, using Doxygen. Also generates *.mpack files. To inspect the *.mpack structure: - - :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack'))) - + :new | put=v:lua.vim.inspect(v:lua.vim.mpack.unpack(readfile('runtime/doc/api.mpack','B'))) Flow: main @@ -287,7 +285,7 @@ annotation_map = { 'FUNC_API_FAST': '|api-fast|', 'FUNC_API_CHECK_TEXTLOCK': 'not allowed when |textlock| is active', 'FUNC_API_REMOTE_ONLY': '|RPC| only', - 'FUNC_API_LUA_ONLY': '|vim.api| only', + 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only', } -- cgit From a7a83bc4c25d63f3ae0a7a56e5211df1444699c4 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 9 Oct 2022 18:19:43 +0200 Subject: fix(docs-html): update parser - Improve generated HTML by updating parser which includes fixes for single "'" and single "|": https://github.com/neovim/tree-sitter-vimdoc/pull/31 - Updated parser also fixes the conceal issue for "help" highlight queries https://github.com/neovim/tree-sitter-vimdoc/issues/23 by NOT including whitespace in nodes. - But this means we need to restore the getws() function which scrapes leading whitespace from the original input (buffer). --- scripts/gen_help_html.lua | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 13601b91f5..c647484905 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -249,6 +249,16 @@ local function getbuflinestr(node, bufnr, offset) return table.concat(lines, '\n') end +-- Gets the whitespace just before `node` from the raw buffer text. +-- Needed for preformatted `old` lines. +local function getws(node, bufnr) + local line1, c1, line2, _ = node:range() + local raw = vim.fn.getbufline(bufnr, line1 + 1, line2 + 1)[1] + local text_before = raw:sub(1, c1) + local leading_ws = text_before:match('%s+$') or '' + return leading_ws +end + local function get_tagname(node, bufnr) local text = vim.treesitter.get_node_text(node, bufnr) local tag = (node:type() == 'optionlink' or node:parent():type() == 'optionlink') and ("'%s'"):format(text) or text @@ -354,12 +364,21 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) local parent = root:parent() and root:parent():type() or nil local text = '' local trimmed - local function node_text(node) - return vim.treesitter.get_node_text(node or root, opt.buf) + -- Gets leading whitespace of `node`. + local function ws(node) + node = node or root + local ws_ = getws(node, opt.buf) + -- XXX: first node of a (line) includes whitespace, even after + -- https://github.com/neovim/tree-sitter-vimdoc/pull/31 ? + if ws_ == '' then + ws_ = vim.treesitter.get_node_text(node, opt.buf):match('^%s+') or '' + end + return ws_ end - -- Gets leading whitespace of the current node. - local function ws() - return node_text():match('^%s+') or '' + local function node_text(node, ws_) + node = node or root + ws_ = (ws_ == nil or ws_ == true) and getws(node, opt.buf) or '' + return string.format('%s%s', ws_, vim.treesitter.get_node_text(node, opt.buf)) end if root:named_child_count() == 0 or node_name == 'ERROR' then @@ -401,7 +420,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if root:has_error() then return text end - return ('
%s%s
'):format(ws(), trimmed) + return ('
%s
'):format(text) elseif node_name == 'block' then if is_blank(text) then return '' @@ -462,9 +481,9 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end local in_heading = vim.tbl_contains({'h1', 'h2', 'h3'}, parent) local cssclass = (not in_heading and get_indent(node_text()) > 8) and 'help-tag-right' or 'help-tag' - local tagname = node_text(root:child(1)) + local tagname = node_text(root:child(1), false) if vim.tbl_count(stats.first_tags) < 2 then - -- First 2 tags in the doc will be anchored at the main heading. + -- Force the first 2 tags in the doc to be anchored at the main heading. table.insert(stats.first_tags, tagname) return '' end -- cgit From 6b01e9bf872cc91530b41fcd97c6bf984776ade6 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Mon, 10 Oct 2022 00:40:17 +0200 Subject: feat(docs-html): try to use tags for ToC headings Problem: The generated ToC (table of contents) uses anchors derived from the heading title, e.g. the "Global Plugins" heading yields: https://neovim.io/doc/user/usr_05.html#_global-plugins- so if the heading title changes, then the old URL (anchor) is broken. Solution: :help tags change less often than heading titles, so if a heading contains a *tag*, use that as its anchor name instead. Example: https://neovim.io/doc/user/usr_05.html#standard-plugin --- scripts/gen_help_html.lua | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index c647484905..06ea1831b0 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -297,6 +297,16 @@ local function has_ancestor(node, ancestor_name) return false end +-- Gets the first matching child node matching `name`. +local function first(node, name) + for c, _ in node:iter_children() do + if c:named() and c:type() == name then + return c + end + end + return nil +end + local function validate_link(node, bufnr, fname) local helppage, tagname = get_tagname(node:child(1), bufnr) local ignored = false @@ -409,13 +419,18 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end -- Remove "===" and tags from ToC text. local hname = (node_text():gsub('%-%-%-%-+', ''):gsub('%=%=%=%=+', ''):gsub('%*.*%*', '')) + -- Use the first *tag* node as the heading anchor, if any. + local tagnode = first(root, 'tag') + local tagname = tagnode and url_encode(node_text(tagnode:child(1), false)) or to_heading_tag(hname) if node_name == 'h1' or #headings == 0 then - table.insert(headings, { name = hname, subheadings = {}, }) + table.insert(headings, { name = hname, subheadings = {}, tag = tagname }) else - table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, }) + table.insert(headings[#headings].subheadings, { name = hname, subheadings = {}, tag = tagname }) end local el = node_name == 'h1' and 'h2' or 'h3' - return ('<%s class="help-heading">%s\n'):format(to_heading_tag(hname), el, text, el) + -- If we are re-using the *tag*, this heading anchor is redundant. + local a = tagnode and '' or (''):format(tagname) + return ('%s<%s class="help-heading">%s\n'):format(a, el, text, el) elseif node_name == 'column_heading' or node_name == 'column_name' then if root:has_error() then return text @@ -720,10 +735,10 @@ local function gen_one(fname, to_fname, old, commit) local n = 0 -- Count of all headings + subheadings. for _, h1 in ipairs(headings) do n = n + 1 + #h1.subheadings end for _, h1 in ipairs(headings) do - toc = toc .. ('
%s\n'):format(to_heading_tag(h1.name), h1.name) + toc = toc .. ('
%s\n'):format(h1.tag, h1.name) if n < 30 or #headings < 10 then -- Show subheadings only if there aren't too many. for _, h2 in ipairs(h1.subheadings) do - toc = toc .. ('
%s
\n'):format(to_heading_tag(h2.name), h2.name) + toc = toc .. ('
%s
\n'):format(h2.tag, h2.name) end end toc = toc .. '
' -- cgit From a6d889eae1405b4a96c1938d3bab8f27c8c9a029 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 10 Oct 2022 12:10:57 +0100 Subject: refactor(lua2dox): format with stylua --- scripts/lua2dox.lua | 215 ++++++++++++++++++++++++++-------------------------- 1 file changed, 108 insertions(+), 107 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 674b42a699..562c3829c9 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -51,13 +51,13 @@ The effect is that you will get the function documented, but not with the parame ]] local function class(BaseClass, ClassInitialiser) - local newClass = {} -- a new class newClass + local newClass = {} -- a new class newClass if not ClassInitialiser and type(BaseClass) == 'function' then ClassInitialiser = BaseClass BaseClass = nil elseif type(BaseClass) == 'table' then -- our new class is a shallow copy of the base class! - for i,v in pairs(BaseClass) do + for i, v in pairs(BaseClass) do newClass[i] = v end newClass._base = BaseClass @@ -70,11 +70,11 @@ local function class(BaseClass, ClassInitialiser) local classMetatable = {} classMetatable.__call = function(class_tbl, ...) local newInstance = {} - setmetatable(newInstance,newClass) + setmetatable(newInstance, newClass) --if init then -- init(newInstance,...) if class_tbl.init then - class_tbl.init(newInstance,...) + class_tbl.init(newInstance, ...) else -- make sure that any stuff from the base class is initialized! if BaseClass and BaseClass.init then @@ -104,7 +104,7 @@ local TCore_Clock = class() --! \brief get the current time function TCore_Clock.GetTimeNow() - local gettimeofday = os.gettimeofday -- luacheck: ignore 143 Accessing an undefined field of a global variable. + local gettimeofday = os.gettimeofday -- luacheck: ignore 143 Accessing an undefined field of a global variable. if gettimeofday then return gettimeofday() else @@ -113,7 +113,7 @@ function TCore_Clock.GetTimeNow() end --! \brief constructor -function TCore_Clock.init(this,T0) +function TCore_Clock.init(this, T0) if T0 then this.t0 = T0 else @@ -122,36 +122,34 @@ function TCore_Clock.init(this,T0) end --! \brief get time string -function TCore_Clock.getTimeStamp(this,T0) +function TCore_Clock.getTimeStamp(this, T0) local t0 if T0 then t0 = T0 else t0 = this.t0 end - return os.date('%c %Z',t0) + return os.date('%c %Z', t0) end - --! \brief write to stdout local function TCore_IO_write(Str) - if (Str) then + if Str then io.write(Str) end end --! \brief write to stdout local function TCore_IO_writeln(Str) - if (Str) then + if Str then io.write(Str) end - io.write("\n") + io.write('\n') end - --! \brief trims a string local function string_trim(Str) - return Str:match("^%s*(.-)%s*$") + return Str:match('^%s*(.-)%s*$') end --! \brief split a string @@ -162,24 +160,23 @@ end ---@return string[] local function string_split(Str, Pattern) local splitStr = {} - local fpat = "(.-)" .. Pattern + local fpat = '(.-)' .. Pattern local last_end = 1 - local str, e, cap = string.find(Str,fpat, 1) + local str, e, cap = string.find(Str, fpat, 1) while str do - if str ~= 1 or cap ~= "" then - table.insert(splitStr,cap) + if str ~= 1 or cap ~= '' then + table.insert(splitStr, cap) end - last_end = e+1 - str, e, cap = string.find(Str,fpat, last_end) + last_end = e + 1 + str, e, cap = string.find(Str, fpat, last_end) end if last_end <= #Str then - cap = string.sub(Str,last_end) + cap = string.sub(Str, last_end) table.insert(splitStr, cap) end return splitStr end - --! \class TCore_Commandline --! \brief reads/parses commandline local TCore_Commandline = class() @@ -192,7 +189,7 @@ function TCore_Commandline.init(this) end --! \brief get value -function TCore_Commandline.getRaw(this,Key,Default) +function TCore_Commandline.getRaw(this, Key, Default) local val = this.argv[Key] if not val then val = Default @@ -209,14 +206,14 @@ local TStream_Read = class() --! \brief get contents of file --! --! \param Filename name of file to read (or nil == stdin) -function TStream_Read.getContents(this,Filename) +function TStream_Read.getContents(this, Filename) assert(Filename) -- get lines from file -- syphon lines to our table --TCore_Debug_show_var('Filename',Filename) - local filecontents={} + local filecontents = {} for line in io.lines(Filename) do - table.insert(filecontents,line) + table.insert(filecontents, line) end if filecontents then @@ -241,7 +238,7 @@ function TStream_Read.getLine(this) this.currentLine = nil else -- get line - if this.currentLineNo<=this.contentsLen then + if this.currentLineNo <= this.contentsLen then line = this.filecontents[this.currentLineNo] this.currentLineNo = this.currentLineNo + 1 else @@ -252,13 +249,13 @@ function TStream_Read.getLine(this) end --! \brief save line fragment -function TStream_Read.ungetLine(this,LineFrag) +function TStream_Read.ungetLine(this, LineFrag) this.currentLine = LineFrag end --! \brief is it eof? function TStream_Read.eof(this) - if this.currentLine or this.currentLineNo<=this.contentsLen then + if this.currentLine or this.currentLineNo <= this.contentsLen then return false end return true @@ -273,32 +270,32 @@ function TStream_Write.init(this) end --! \brief write immediately -function TStream_Write.write(_,Str) +function TStream_Write.write(_, Str) TCore_IO_write(Str) end --! \brief write immediately -function TStream_Write.writeln(_,Str) +function TStream_Write.writeln(_, Str) TCore_IO_writeln(Str) end --! \brief write immediately -function TStream_Write.writelnComment(_,Str) +function TStream_Write.writelnComment(_, Str) TCore_IO_write('// ZZ: ') TCore_IO_writeln(Str) end --! \brief write to tail -function TStream_Write.writelnTail(this,Line) +function TStream_Write.writelnTail(this, Line) if not Line then Line = '' end - table.insert(this.tailLine,Line) + table.insert(this.tailLine, Line) end --! \brief output tail lines function TStream_Write.write_tailLines(this) - for _,line in ipairs(this.tailLine) do + for _, line in ipairs(this.tailLine) do TCore_IO_writeln(line) end TCore_IO_write('// Lua2DoX new eof') @@ -308,9 +305,9 @@ end local TLua2DoX_filter = class() --! \brief allow us to do errormessages -function TLua2DoX_filter.warning(this,Line,LineNo,Legend) +function TLua2DoX_filter.warning(this, Line, LineNo, Legend) this.outStream:writelnTail( - '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"' + '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"' ) end @@ -319,47 +316,47 @@ end --! If the string has a comment on the end, this trims it off. --! local function TString_removeCommentFromLine(Line) - local pos_comment = string.find(Line,'%-%-') + local pos_comment = string.find(Line, '%-%-') local tailComment if pos_comment then - Line = string.sub(Line,1,pos_comment-1) - tailComment = string.sub(Line,pos_comment) + Line = string.sub(Line, 1, pos_comment - 1) + tailComment = string.sub(Line, pos_comment) end - return Line,tailComment + return Line, tailComment end --! \brief get directive from magic local function getMagicDirective(Line) - local macro,tail + local macro, tail local macroStr = '[\\@]' - local pos_macro = string.find(Line,macroStr) + local pos_macro = string.find(Line, macroStr) if pos_macro then --! ....\\ macro...stuff --! ....\@ macro...stuff - local line = string.sub(Line,pos_macro+1) - local space = string.find(line,'%s+') + local line = string.sub(Line, pos_macro + 1) + local space = string.find(line, '%s+') if space then - macro = string.sub(line,1,space-1) - tail = string_trim(string.sub(line,space+1)) + macro = string.sub(line, 1, space - 1) + tail = string_trim(string.sub(line, space + 1)) else macro = line - tail = '' + tail = '' end end - return macro,tail + return macro, tail end --! \brief check comment for fn -local function checkComment4fn(Fn_magic,MagicLines) +local function checkComment4fn(Fn_magic, MagicLines) local fn_magic = Fn_magic -- TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"') - local magicLines = string_split(MagicLines,'\n') + local magicLines = string_split(MagicLines, '\n') - local macro,tail + local macro, tail for _, line in ipairs(magicLines) do - macro,tail = getMagicDirective(line) + macro, tail = getMagicDirective(line) if macro == 'fn' then fn_magic = tail -- TCore_IO_writeln('// found fn "' .. fn_magic .. '"') @@ -371,15 +368,15 @@ local function checkComment4fn(Fn_magic,MagicLines) return fn_magic end -local types = {"number", "string", "table", "list", "boolean", "function"} +local types = { 'number', 'string', 'table', 'list', 'boolean', 'function' } --! \brief run the filter -function TLua2DoX_filter.readfile(this,AppStamp,Filename) +function TLua2DoX_filter.readfile(this, AppStamp, Filename) local inStream = TStream_Read() local outStream = TStream_Write() this.outStream = outStream -- save to this obj - if (inStream:getContents(Filename)) then + if inStream:getContents(Filename) then -- output the file local line local fn_magic -- function name/def from magic comment @@ -389,7 +386,7 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) outStream:writelnTail('// #######################') outStream:writelnTail() - local state = '' -- luacheck: ignore 231 variable is set but never accessed. + local state = '' -- luacheck: ignore 231 variable is set but never accessed. local offset = 0 local generic = {} local l = 0 @@ -398,7 +395,7 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) -- TCore_Debug_show_var('inStream',inStream) -- TCore_Debug_show_var('line',line ) l = l + 1 - if string.sub(line,1,2) == '--' then -- it's a comment + if string.sub(line, 1, 2) == '--' then -- it's a comment -- Allow people to write style similar to EmmyLua (since they are basically the same) -- instead of silently skipping things that start with --- if string.sub(line, 3, 3) == '@' then -- it's a magic comment @@ -414,22 +411,23 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local magic_split = string_split(magic, ' ') if magic_split[1] == 'param' then for _, type in ipairs(types) do - magic = magic:gsub("^param%s+([a-zA-Z_?]+)%s+.*%((" .. type .. ")%)", "param %1 %2" ) - magic = magic:gsub("^param%s+([a-zA-Z_?]+)%s+.*%((" .. type .. "|nil)%)", "param %1 %2" ) + magic = magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. ')%)', 'param %1 %2') + magic = + magic:gsub('^param%s+([a-zA-Z_?]+)%s+.*%((' .. type .. '|nil)%)', 'param %1 %2') end magic_split = string_split(magic, ' ') elseif magic_split[1] == 'return' then for _, type in ipairs(types) do - magic = magic:gsub("^return%s+.*%((" .. type .. ")%)", "return %1" ) - magic = magic:gsub("^return%s+.*%((" .. type .. "|nil)%)", "return %1" ) + magic = magic:gsub('^return%s+.*%((' .. type .. ')%)', 'return %1') + magic = magic:gsub('^return%s+.*%((' .. type .. '|nil)%)', 'return %1') end magic_split = string_split(magic, ' ') end - if magic_split[1] == "generic" then - local generic_name, generic_type = line:match("@generic%s*(%w+)%s*:?%s*(.*)") - if generic_type == "" then - generic_type = "any" + if magic_split[1] == 'generic' then + local generic_name, generic_type = line:match('@generic%s*(%w+)%s*:?%s*(.*)') + if generic_type == '' then + generic_type = 'any' end generic[generic_name] = generic_type else @@ -440,9 +438,9 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) if magic_split[type_index] then -- fix optional parameters - if magic_split[type_index] and magic_split[2]:find("%?$") then - if not magic_split[type_index]:find("nil") then - magic_split[type_index] = magic_split[type_index] .. "|nil" + if magic_split[type_index] and magic_split[2]:find('%?$') then + if not magic_split[type_index]:find('nil') then + magic_split[type_index] = magic_split[type_index] .. '|nil' end magic_split[2] = magic_split[2]:sub(1, -2) end @@ -454,42 +452,44 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) end -- surround some types by () for _, type in ipairs(types) do - magic_split[type_index] = magic_split[type_index]:gsub("^(" .. type .. "|nil):?$", "(%1)") - magic_split[type_index] = magic_split[type_index]:gsub("^(" .. type .. "):?$", "(%1)") + magic_split[type_index] = + magic_split[type_index]:gsub('^(' .. type .. '|nil):?$', '(%1)') + magic_split[type_index] = + magic_split[type_index]:gsub('^(' .. type .. '):?$', '(%1)') end end magic = table.concat(magic_split, ' ') outStream:writeln('/// @' .. magic) - fn_magic = checkComment4fn(fn_magic,magic) + fn_magic = checkComment4fn(fn_magic, magic) end - elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment - local comment = string.sub(line,4) - outStream:writeln('/// '.. comment) - elseif string.sub(line,3,4)=='[[' then -- it's a long comment - line = string.sub(line,5) -- nibble head + elseif string.sub(line, 3, 3) == '-' then -- it's a nonmagic doc comment + local comment = string.sub(line, 4) + outStream:writeln('/// ' .. comment) + elseif string.sub(line, 3, 4) == '[[' then -- it's a long comment + line = string.sub(line, 5) -- nibble head local comment = '' - local closeSquare,hitend,thisComment - while (not hitend) and (not inStream:eof()) do - closeSquare = string.find(line,']]') + local closeSquare, hitend, thisComment + while not hitend and (not inStream:eof()) do + closeSquare = string.find(line, ']]') if not closeSquare then -- need to look on another line thisComment = line .. '\n' line = inStream:getLine() else - thisComment = string.sub(line,1,closeSquare-1) + thisComment = string.sub(line, 1, closeSquare - 1) hitend = true -- unget the tail of the line -- in most cases it's empty. This may make us less efficient but -- easier to program - inStream:ungetLine(string_trim(string.sub(line,closeSquare+2))) + inStream:ungetLine(string_trim(string.sub(line, closeSquare + 2))) end comment = comment .. thisComment end - if string.sub(comment,1,1)=='@' then -- it's a long magic comment + if string.sub(comment, 1, 1) == '@' then -- it's a long magic comment outStream:write('/*' .. comment .. '*/ ') - fn_magic = checkComment4fn(fn_magic,comment) + fn_magic = checkComment4fn(fn_magic, comment) else -- discard outStream:write('/* zz:' .. comment .. '*/ ') fn_magic = nil @@ -504,64 +504,67 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) end elseif string.find(line, '^function') or string.find(line, '^local%s+function') then generic = {} - state = 'in_function' -- it's a function - local pos_fn = string.find(line,'function') + state = 'in_function' -- it's a function + local pos_fn = string.find(line, 'function') -- function -- ....v... if pos_fn then -- we've got a function local fn_type - if string.find(line,'^local%s+') then - fn_type = ''--'static ' -- static functions seem to be excluded + if string.find(line, '^local%s+') then + fn_type = '' --'static ' -- static functions seem to be excluded else fn_type = '' end - local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8))) + local fn = TString_removeCommentFromLine(string_trim(string.sub(line, pos_fn + 8))) if fn_magic then fn = fn_magic end - if string.sub(fn,1,1)=='(' then + if string.sub(fn, 1, 1) == '(' then -- it's an anonymous function outStream:writelnComment(line) else -- fn has a name, so is interesting -- want to fix for iffy declarations - local open_paren = string.find(fn,'[%({]') + local open_paren = string.find(fn, '[%({]') if open_paren then -- we might have a missing close paren - if not string.find(fn,'%)') then + if not string.find(fn, '%)') then fn = fn .. ' ___MissingCloseParenHere___)' end end -- Big hax - if string.find(fn, ":") then + if string.find(fn, ':') then -- TODO: We need to add a first parameter of "SELF" here -- local colon_place = string.find(fn, ":") -- local name = string.sub(fn, 1, colon_place) - fn = fn:gsub(":", ".", 1) - outStream:writeln("/// @param self") + fn = fn:gsub(':', '.', 1) + outStream:writeln('/// @param self') - local paren_start = string.find(fn, "(", 1, true) - local paren_finish = string.find(fn, ")", 1, true) + local paren_start = string.find(fn, '(', 1, true) + local paren_finish = string.find(fn, ')', 1, true) -- Nothing in between the parens local comma if paren_finish == paren_start + 1 then - comma = "" + comma = '' else - comma = ", " + comma = ', ' end - fn = string.sub(fn, 1, paren_start) .. "self" .. comma .. string.sub(fn, paren_start + 1) + fn = string.sub(fn, 1, paren_start) + .. 'self' + .. comma + .. string.sub(fn, paren_start + 1) end -- add vanilla function outStream:writeln(fn_type .. 'function ' .. fn .. '{}') end else - this:warning(inStream:getLineNo(),'something weird here') + this:warning(inStream:getLineNo(), 'something weird here') end fn_magic = nil -- mustn't indavertently use it again @@ -572,8 +575,8 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) -- fn_magic = nil -- mustn't indavertently use it again else - state = '' -- unknown - if #line>0 then -- we don't know what this line means, so just comment it out + state = '' -- unknown + if #line > 0 then -- we don't know what this line means, so just comment it out outStream:writeln('// zz: ' .. line) else outStream:writeln() -- keep this line blank @@ -601,8 +604,7 @@ function TApp.init(this) end function TApp.getRunStamp(this) - return this.name .. ' (' .. this.version .. ') ' - .. this.timestamp + return this.name .. ' (' .. this.version .. ') ' .. this.timestamp end function TApp.getVersion(this) @@ -639,8 +641,7 @@ else local filename = argv1 local filter = TLua2DoX_filter() - filter:readfile(appStamp,filename) + filter:readfile(appStamp, filename) end - --eof -- cgit From c401b06fe978167a7408375e769be38db4146935 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 10 Oct 2022 12:21:19 +0100 Subject: refactor(lua2dox): tidy --- scripts/lua2dox.lua | 96 ++++++++--------------------------------------------- 1 file changed, 13 insertions(+), 83 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 562c3829c9..86afe97a7e 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -50,88 +50,28 @@ However I have put in a hack that will insert the "missing" close paren. The effect is that you will get the function documented, but not with the parameter list you might expect. ]] -local function class(BaseClass, ClassInitialiser) +local function class() local newClass = {} -- a new class newClass - if not ClassInitialiser and type(BaseClass) == 'function' then - ClassInitialiser = BaseClass - BaseClass = nil - elseif type(BaseClass) == 'table' then - -- our new class is a shallow copy of the base class! - for i, v in pairs(BaseClass) do - newClass[i] = v - end - newClass._base = BaseClass - end -- the class will be the metatable for all its newInstanceects, -- and they will look up their methods in it. newClass.__index = newClass -- expose a constructor which can be called by () - local classMetatable = {} - classMetatable.__call = function(class_tbl, ...) - local newInstance = {} - setmetatable(newInstance, newClass) - --if init then - -- init(newInstance,...) - if class_tbl.init then - class_tbl.init(newInstance, ...) - else - -- make sure that any stuff from the base class is initialized! - if BaseClass and BaseClass.init then - BaseClass.init(newInstance, ...) - end - end - return newInstance - end - newClass.init = ClassInitialiser - newClass.is_a = function(this, klass) - local thisMetatable = getmetatable(this) - while thisMetatable do - if thisMetatable == klass then - return true + setmetatable(newClass, { + __call = function(class_tbl, ...) + local newInstance = {} + setmetatable(newInstance, newClass) + --if init then + -- init(newInstance,...) + if class_tbl.init then + class_tbl.init(newInstance, ...) end - thisMetatable = thisMetatable._base + return newInstance end - return false - end - setmetatable(newClass, classMetatable) + }) return newClass end ---! \class TCore_Clock ---! \brief a clock -local TCore_Clock = class() - ---! \brief get the current time -function TCore_Clock.GetTimeNow() - local gettimeofday = os.gettimeofday -- luacheck: ignore 143 Accessing an undefined field of a global variable. - if gettimeofday then - return gettimeofday() - else - return os.time() - end -end - ---! \brief constructor -function TCore_Clock.init(this, T0) - if T0 then - this.t0 = T0 - else - this.t0 = TCore_Clock.GetTimeNow() - end -end - ---! \brief get time string -function TCore_Clock.getTimeStamp(this, T0) - local t0 - if T0 then - t0 = T0 - else - t0 = this.t0 - end - return os.date('%c %Z', t0) -end - --! \brief write to stdout local function TCore_IO_write(Str) if Str then @@ -210,7 +150,6 @@ function TStream_Read.getContents(this, Filename) assert(Filename) -- get lines from file -- syphon lines to our table - --TCore_Debug_show_var('Filename',Filename) local filecontents = {} for line in io.lines(Filename) do table.insert(filecontents, line) @@ -392,8 +331,6 @@ function TLua2DoX_filter.readfile(this, AppStamp, Filename) local l = 0 while not (inStream:eof()) do line = string_trim(inStream:getLine()) - -- TCore_Debug_show_var('inStream',inStream) - -- TCore_Debug_show_var('line',line ) l = l + 1 if string.sub(line, 1, 2) == '--' then -- it's a comment -- Allow people to write style similar to EmmyLua (since they are basically the same) @@ -510,12 +447,6 @@ function TLua2DoX_filter.readfile(this, AppStamp, Filename) -- ....v... if pos_fn then -- we've got a function - local fn_type - if string.find(line, '^local%s+') then - fn_type = '' --'static ' -- static functions seem to be excluded - else - fn_type = '' - end local fn = TString_removeCommentFromLine(string_trim(string.sub(line, pos_fn + 8))) if fn_magic then fn = fn_magic @@ -561,7 +492,7 @@ function TLua2DoX_filter.readfile(this, AppStamp, Filename) end -- add vanilla function - outStream:writeln(fn_type .. 'function ' .. fn .. '{}') + outStream:writeln('function ' .. fn .. '{}') end else this:warning(inStream:getLineNo(), 'something weird here') @@ -596,8 +527,7 @@ local TApp = class() --! \brief constructor function TApp.init(this) - local t0 = TCore_Clock() - this.timestamp = t0:getTimeStamp() + this.timestamp = os.date('%c %Z', os.time()) this.name = 'Lua2DoX' this.version = '0.2 20130128' this.copyright = 'Copyright (c) Simon Dales 2012-13' -- cgit From 3b6c7f9c7f2df493e24990c47a338ef8b46dfe8a Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 10 Oct 2022 12:21:40 +0100 Subject: refactor(lua2dox_filter): format --- scripts/lua2dox_filter | 105 ++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 54 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter index 0b9f59b6ac..e3fa95d0cf 100755 --- a/scripts/lua2dox_filter +++ b/scripts/lua2dox_filter @@ -22,72 +22,69 @@ LANG="" ##! \brief test executable to see if it exists -test_executable(){ - P_EXE="$1" - ######### - WHICH=`which ${P_EXE}` - if test -z "${WHICH}" - then - echo "not found \"${P_EXE}\"" - else - EXE="${P_EXE}" - fi - } +test_executable() { + P_EXE="$1" + ######### + WHICH=$(which "$P_EXE") + if test -z "${WHICH}"; then + echo "not found \"${P_EXE}\"" + else + EXE="${P_EXE}" + fi +} ##! \brief sets the lua interpreter -set_lua(){ - if test -z "${EXE}"; then - test_executable '.deps/usr/bin/luajit' - fi +set_lua() { + if test -z "${EXE}"; then + test_executable '.deps/usr/bin/luajit' + fi - if test -z "${EXE}"; then - test_executable 'luajit' - fi + if test -z "${EXE}"; then + test_executable 'luajit' + fi - if test -z "${EXE}"; then - test_executable 'lua' - fi + if test -z "${EXE}"; then + test_executable 'lua' + fi } ##! \brief makes canonical name of file -##! +##! ##! Note that "readlink -f" doesn't work in MacOSX -##! -do_readlink(){ - pushd . > /dev/null - TARGET_FILE=$1 - - cd `dirname $TARGET_FILE` - TARGET_FILE=`basename $TARGET_FILE` - - # Iterate down a (possible) chain of symlinks - while [ -L "$TARGET_FILE" ] - do - TARGET_FILE=`readlink $TARGET_FILE` - cd `dirname $TARGET_FILE` - TARGET_FILE=`basename $TARGET_FILE` - done - - PHYS_DIR=`pwd -P` - RESULT=$PHYS_DIR - popd > /dev/null - } +##! +do_readlink() { + pushd . > /dev/null + TARGET_FILE=$1 + + cd "$(dirname $TARGET_FILE)" + TARGET_FILE=$(basename "$TARGET_FILE") + + # Iterate down a (possible) chain of symlinks + while [ -L "$TARGET_FILE" ]; do + TARGET_FILE=$(readlink "$TARGET_FILE") + cd $(dirname "$TARGET_FILE") + TARGET_FILE=$(basename "$TARGET_FILE") + done + + PHYS_DIR=$(pwd -P) + RESULT=$PHYS_DIR + popd > /dev/null +} ##main set_lua -if test -z "${EXE}" -then - echo "no lua interpreter found" - exit 1 +if test -z "${EXE}"; then + echo "no lua interpreter found" + exit 1 else - BASENAME=`basename "$0"` - do_readlink "$0" - DIRNAME="${RESULT}" - - LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}" - #echo "lua[${LUASCRIPT}]" + BASENAME=$(basename "$0") + do_readlink "$0" + DIRNAME="${RESULT}" + + LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}" + #echo "lua[${LUASCRIPT}]" - ${EXE} ${LUASCRIPT} $@ + ${EXE} ${LUASCRIPT} $@ fi -# + ##eof -- cgit From 288208257c8d6b3c8dcce7ee6c7b6c7bb7bafb27 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Sat, 8 Oct 2022 15:48:07 +0100 Subject: feat(cscope)!: remove --- scripts/vim-patch.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index ad1973603e..f824feb856 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -192,11 +192,11 @@ preprocess_patch() { 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/\<\%('"${na_files}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove *.proto, Make*, INSTALL*, gui_*, beval.*, some if_*, gvim, libvterm, tee, VisVim, xpm, xxd - local na_src='auto\|configure.*\|GvimExt\|libvterm\|proto\|tee\|VisVim\|xpm\|xxd\|Make.*\|INSTALL.*\|beval.*\|gui.*\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' + local na_src='auto\|configure.*\|GvimExt\|libvterm\|proto\|tee\|VisVim\|xpm\|xxd\|Make.*\|INSTALL.*\|beval.*\|gui.*\|if_cscop\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%(testdir/\)\@@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove unwanted Vim doc files. - local na_doc='channel\.txt\|netbeans\.txt\|os_\w\+\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags' + local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('"${na_doc}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove "Last change ..." changes in doc files. @@ -207,7 +207,7 @@ preprocess_patch() { 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<\%('"${na_src_testdir}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove testdir/test_*.vim files - local na_src_testdir='balloon.*\|channel.*\|crypt\.vim\|gui.*\|job_fails\.vim\|json\.vim\|mzscheme\.vim\|netbeans.*\|paste\.vim\|popupwin.*\|restricted\.vim\|shortpathname\.vim\|tcl\.vim\|terminal.*\|xxd\.vim' + local na_src_testdir='balloon.*\|channel.*\|crypt\.vim\|cscope\.vim\|gui.*\|job_fails\.vim\|json\.vim\|mzscheme\.vim\|netbeans.*\|paste\.vim\|popupwin.*\|restricted\.vim\|shortpathname\.vim\|tcl\.vim\|terminal.*\|xxd\.vim' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove version.c #7555 -- cgit From e5cb3104d07228de4f2614c425355e8f2f99507d Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Fri, 14 Oct 2022 11:01:13 -0400 Subject: docs: fix/remove invalid URLs #20647 --- scripts/gen_help_html.lua | 14 +++++++++++--- scripts/gen_vimdoc.py | 11 +++-------- 2 files changed, 14 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 06ea1831b0..39c516ee96 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -319,6 +319,16 @@ local function validate_link(node, bufnr, fname) return helppage, tagname, ignored end +local function validate_url(text, fname) + local ignored = false + if vim.fs.basename(fname) == 'pi_netrw.txt' then + ignored = true + elseif text:find('http%:') then + invalid_urls[text] = vim.fs.basename(fname) + end + return ignored +end + -- Traverses the tree at `root` and checks that |tag| links point to valid helptags. local function visit_validate(root, level, lang_tree, opt, stats) level = level or 0 @@ -353,9 +363,7 @@ local function visit_validate(root, level, lang_tree, opt, stats) end end elseif node_name == 'url' then - if text:find('http%:') then - invalid_urls[text] = vim.fs.basename(opt.fname) - end + validate_url(text, opt.fname) elseif node_name == 'taglink' or node_name == 'optionlink' then local _, _, _ = validate_link(root, opt.buf, opt.fname) end diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 3ee9d8b5dd..a044c8c39d 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -12,15 +12,10 @@ Flow: update_params_map / render_node -This would be easier using lxml and XSLT, but: +TODO: eliminate this script and use Lua+treesitter (requires parsers for C and +Lua markdown-style docstrings). - 1. This should avoid needing Python dependencies, especially ones that are - C modules that have library dependencies (lxml requires libxml and - libxslt). - 2. I wouldn't know how to deal with nested indentation in tags using - XSLT. - -Each function :help block is formatted as follows: +The generated :help text for each function is formatted as follows: - Max width of 78 columns (`text_width`). - Indent with spaces (not tabs). -- cgit From 042eb74ff1ed63d79f8a642649cd6be6ec4b0eb9 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Mon, 17 Oct 2022 08:52:40 +0200 Subject: feat(runtime)!: remove filetype.vim (#20428) Made obsolete by now graduated `filetype.lua` (enabled by default). Note that changes or additions to the filetype detection still need to be made through a PR to vim/vim as we port the _logic_ as well as tests. --- scripts/vim-patch.sh | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index f824feb856..fbade1b4f7 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -195,6 +195,10 @@ preprocess_patch() { local na_src='auto\|configure.*\|GvimExt\|libvterm\|proto\|tee\|VisVim\|xpm\|xxd\|Make.*\|INSTALL.*\|beval.*\|gui.*\|if_cscop\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%(testdir/\)\@@norm! d/\v(^diff)|%$ ' +w +q "$file" + # Remove runtime files ported to Lua. + local na_rt='filetype\.vim\|scripts\.vim\|autoload\/ft\/dist\.vim' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('"${na_rt}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" + # Remove unwanted Vim doc files. local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('"${na_doc}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" -- cgit From ef4c339fb9de87f7534303e006c281e40327f803 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 18 Oct 2022 10:18:44 -0400 Subject: feat(docs): update parser, HTML gen #20720 Note: although the tolerance in help_spec.lua increased, the actual error count with the new parser decreased by about 20%. The difference is that the old ignore_parse_error() ignored many more errors with the old parser. fix https://github.com/neovim/tree-sitter-vimdoc/issues/37 fix https://github.com/neovim/tree-sitter-vimdoc/issues/44 fix https://github.com/neovim/tree-sitter-vimdoc/issues/47 --- scripts/gen_help_html.lua | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 39c516ee96..b1d41cf89e 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -74,6 +74,23 @@ local exclude_invalid = { ["vim.treesitter.start()"] = "treesitter.txt", } +-- False-positive "invalid URLs". +local exclude_invalid_urls = { + ["http://"] = "usr_23.txt", + ["http://."] = "usr_23.txt", + ["http://aspell.net/man-html/Affix-Compression.html"] = "spell.txt", + ["http://aspell.net/man-html/Phonetic-Code.html"] = "spell.txt", + ["http://canna.sourceforge.jp/"] = "mbyte.txt", + ["http://gnuada.sourceforge.net"] = "ft_ada.txt", + ["http://lua-users.org/wiki/StringLibraryTutorial"] = "lua.txt", + ["http://michael.toren.net/code/"] = "pi_tar.txt", + ["http://papp.plan9.de"] = "syntax.txt", + ["http://wiki.services.openoffice.org/wiki/Dictionaries"] = "spell.txt", + ["http://www.adapower.com"] = "ft_ada.txt", + ["http://www.ghostscript.com/"] = "print.txt", + ["http://www.jclark.com/"] = "quickfix.txt", +} + local function tofile(fname, text) local f = io.open(fname, 'w') if not f then @@ -278,10 +295,13 @@ local function ignore_invalid(s) ) end -local function ignore_parse_error(s) - -- Ignore parse errors for unclosed codespan/optionlink/tag. - -- This is common in vimdocs and is treated as plaintext by :help. - return s:find("^[`'|*]") +local function ignore_parse_error(s, fname) + local helpfile = vim.fs.basename(fname) + return (helpfile == 'pi_netrw.txt' + -- Ignore parse errors for unclosed tag. + -- This is common in vimdocs and is treated as plaintext by :help. + or s:find("^[`'|*]") + ) end local function has_ancestor(node, ancestor_name) @@ -323,7 +343,7 @@ local function validate_url(text, fname) local ignored = false if vim.fs.basename(fname) == 'pi_netrw.txt' then ignored = true - elseif text:find('http%:') then + elseif text:find('http%:') and not exclude_invalid_urls[text] then invalid_urls[text] = vim.fs.basename(fname) end return ignored @@ -348,7 +368,7 @@ local function visit_validate(root, level, lang_tree, opt, stats) end if node_name == 'ERROR' then - if ignore_parse_error(text) then + if ignore_parse_error(text, opt.fname) then return end -- Store the raw text to give context to the error report. @@ -363,7 +383,8 @@ local function visit_validate(root, level, lang_tree, opt, stats) end end elseif node_name == 'url' then - validate_url(text, opt.fname) + local fixed_url, _ = fix_url(trim(text)) + validate_url(fixed_url, opt.fname) elseif node_name == 'taglink' or node_name == 'optionlink' then local _, _, _ = validate_link(root, opt.buf, opt.fname) end @@ -523,7 +544,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end return s elseif node_name == 'ERROR' then - if ignore_parse_error(trimmed) then + if ignore_parse_error(trimmed, opt.fname) then return text end -- cgit From 10ab7489ebb2bcbc7c1b5360921978c1ca2d0a4b Mon Sep 17 00:00:00 2001 From: Yee Cheng Chin Date: Thu, 20 Oct 2022 03:22:46 -0700 Subject: fix(docs-html): misaligned tabs after conceal #20690 Problem: `gen_help_html.lua` does not properly handle tab characters after "concealed" text (tags, taglinks, codespans). This causes misaligned layout in "old" (preformatted) docs. For text like `*tag*`, |tag_link|, and `code_span`, Vim hides the "*", "|", "`" characters, but Vim still counts those characters for "virtual column" when a tab character follows it. So if you have a tag of say 6 characters long, those two concealed character would lead to the tab character after it start at column 8. gen_help_html.lua doesn't account for that which leads to formatting flaws in the generated output. Solution: Add two spaces after concealed nodes that are followed by a tab char. --- scripts/gen_help_html.lua | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index b1d41cf89e..3a384bccf9 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -390,6 +390,18 @@ local function visit_validate(root, level, lang_tree, opt, stats) end end +-- Fix tab alignment issues caused by concealed characters like |, `, * in tags +-- and code blocks. +local function fix_tab_after_conceal(text, next_node_text) + -- Vim tabs take into account the two concealed characters even though they + -- are invisible, so we need to add back in the two spaces if this is + -- followed by a tab to make the tab alignment to match Vim's behavior. + if string.sub(next_node_text,1,1) == '\t' then + text = text .. ' ' + end + return text +end + -- Generates HTML from node `root` recursively. local function visit_node(root, level, lang_tree, headings, opt, stats) level = level or 0 @@ -506,12 +518,20 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) if ignored then return text end - return ('%s%s'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) + local s = ('%s%s'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) + if node_name == 'taglink' and opt.old then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + return s elseif vim.tbl_contains({'codespan', 'keycode'}, node_name) then if root:has_error() then return text end - return ('%s%s'):format(ws(), trimmed) + local s = ('%s%s'):format(ws(), trimmed) + if node_name == 'codespan' and opt.old then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + return s elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) elseif node_name == 'codeblock' then @@ -533,6 +553,10 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end local el = in_heading and 'span' or 'code' local s = ('%s<%s class="%s">%s'):format(ws(), url_encode(tagname), el, cssclass, trimmed, el) + if opt.old then + s = fix_tab_after_conceal(s, node_text(root:next_sibling())) + end + if in_heading and prev ~= 'tag' then -- Start the container for tags in a heading. -- This makes "justify-content:space-between" right-align the tags. -- cgit From e6917306f6d3ba99747d14bea3f0b078631c5c0e Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 20 Oct 2022 09:20:02 -0400 Subject: docs: update vimdoc parser #20747 Remove the user-manual ToC from help.txt, because: 1. it duplicates usr_toc.txt 2. it is not what most readers are looking for in the main help page. fix https://github.com/neovim/tree-sitter-vimdoc/issues/49 fix https://github.com/neovim/tree-sitter-vimdoc/issues/50 fix https://github.com/neovim/tree-sitter-vimdoc/issues/51 --- scripts/gen_help_html.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 3a384bccf9..afc045dc96 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -519,7 +519,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return text end local s = ('%s%s'):format(ws(), helppage, url_encode(tagname), html_esc(tagname)) - if node_name == 'taglink' and opt.old then + if opt.old and node_name == 'taglink' then s = fix_tab_after_conceal(s, node_text(root:next_sibling())) end return s @@ -528,7 +528,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return text end local s = ('%s%s'):format(ws(), trimmed) - if node_name == 'codespan' and opt.old then + if opt.old and node_name == 'codespan' then s = fix_tab_after_conceal(s, node_text(root:next_sibling())) end return s -- cgit From 24c9561a68aa9b9cf8d8a503c5e85f63d7ebb543 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Fri, 21 Oct 2022 06:56:09 -0400 Subject: vim-patch: bump VIM_VERSION from 8.0 => 8.1 #20762 There are 6 remaining 8.0.x patches, tracked in: https://github.com/neovim/neovim/issues/5431 --- scripts/gen_help_html.lua | 1 + scripts/vim-patch.sh | 10 +++++----- scripts/vimpatch.lua | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index afc045dc96..e5251b7f25 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -339,6 +339,7 @@ local function validate_link(node, bufnr, fname) return helppage, tagname, ignored end +-- TODO: port the logic from scripts/check_urls.vim local function validate_url(text, fname) local ignored = false if vim.fs.basename(fname) == 'pi_netrw.txt' then diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index fbade1b4f7..be7c6fd8a6 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -35,7 +35,7 @@ usage() { echo " -m {vim-revision} List previous (older) missing Vim patches." echo " -M List all merged patch-numbers (at current v:version)." echo " -p {vim-revision} Download and generate a Vim patch. vim-revision" - echo " can be a Vim version (8.0.xxx) or a Git hash." + echo " can be a Vim version (8.1.xxx) or a Git hash." echo " -P {vim-revision} Download, generate and apply a Vim patch." echo " -g {vim-revision} Download a Vim patch." echo " -s [pr args] Create a vim-patch pull request." @@ -470,7 +470,7 @@ submit_pr() { # Gets all Vim commits since the "start" commit. list_vim_commits() { ( - cd "${VIM_SOURCE_DIR}" && git log --reverse v8.0.0000..HEAD "$@" + cd "${VIM_SOURCE_DIR}" && git log --reverse v8.1.0000..HEAD "$@" ) } # Prints all (sorted) "vim-patch:xxx" tokens found in the Nvim git log. @@ -488,7 +488,7 @@ list_vimpatch_tokens() { list_vimpatch_numbers() { # Transform "vim-patch:X.Y.ZZZZ" to "ZZZZ". list_vimpatch_tokens | while read -r vimpatch_token; do - echo "$vimpatch_token" | grep '8\.0\.' | sed 's/.*vim-patch:8\.0\.\([0-9a-z]\+\).*/\1/' + echo "$vimpatch_token" | grep '8\.1\.' | sed -E 's/.*vim-patch:8\.1\.([0-9a-z]+).*/\1/' done } @@ -585,7 +585,7 @@ _set_missing_vimpatches() { else info=${line#* } if [[ -n $info ]]; then - # Remove any "patch 8.0.0902: " prefixes, and prefix with ": ". + # Remove any "patch 8.1.0902: " prefixes, and prefix with ": ". info=": ${info#patch*: }" fi fi @@ -628,7 +628,7 @@ show_vimpatches() { Instructions: To port one of the above patches to Neovim, execute this script with the patch revision as argument and follow the instructions, e.g. - '${BASENAME} -p v8.0.1234', or '${BASENAME} -P v8.0.1234' + '${BASENAME} -p v8.1.1234', or '${BASENAME} -P v8.1.1234' NOTE: Please port the _oldest_ patch if you possibly can. You can use '${BASENAME} -l path/to/file' to see what patches are missing for a file. diff --git a/scripts/vimpatch.lua b/scripts/vimpatch.lua index 11eb285462..836f672f6e 100755 --- a/scripts/vimpatch.lua +++ b/scripts/vimpatch.lua @@ -1,7 +1,7 @@ -- Updates version.c list of applied Vim patches. -- -- Usage: --- VIM_SOURCE_DIR=~/neovim/.vim-src/ nvim -i NONE -u NONE --headless +'luafile ./scripts/vimpatch.lua' +q +-- VIM_SOURCE_DIR=~/neovim/.vim-src/ nvim -V1 -es -i NONE +'luafile ./scripts/vimpatch.lua' +q local nvim = vim.api @@ -22,12 +22,13 @@ end -- Generates the lines to be inserted into the src/version.c -- `included_patches[]` definition. local function gen_version_c_lines() - -- Set of merged Vim 8.0.zzzz patch numbers. + -- Set of merged Vim 8.1.zzzz patch numbers. local merged_patch_numbers = {} local highest = 0 for _, n in ipairs(vimpatch_sh_list_numbers()) do + n = tonumber(n) if n then - merged_patch_numbers[tonumber(n)] = true + merged_patch_numbers[n] = true highest = math.max(highest, n) end end -- cgit From 144d7b37accbb9f846f955c3b02c785894863bba Mon Sep 17 00:00:00 2001 From: Maxime Brunet Date: Sun, 23 Oct 2022 04:03:25 -0700 Subject: feat(vim-patch): mention original author #20772 --- scripts/vim-patch.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index be7c6fd8a6..07c9848807 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -120,9 +120,9 @@ get_vim_sources() { commit_message() { if [[ -n "$vim_tag" ]]; then - printf '%s\n%s' "${vim_message}" "${vim_commit_url}" + printf '%s\n\n%s\n\n%s' "${vim_message}" "${vim_commit_url}" "${vim_coauthor}" else - printf 'vim-patch:%s\n\n%s\n%s' "$vim_version" "$vim_message" "$vim_commit_url" + printf 'vim-patch:%s\n\n%s\n\n%s\n\n%s' "$vim_version" "$vim_message" "$vim_commit_url" "$vim_coauthor" fi } @@ -175,6 +175,7 @@ assign_commit_details() { vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}" vim_message="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:%B' "${vim_commit}" \ | sed -e 's/\(#[0-9]\{1,\}\)/vim\/vim\1/g')" + vim_coauthor="$(git -C "${VIM_SOURCE_DIR}" log -1 --pretty='format:Co-authored-by: %an <%ae>' "${vim_commit}")" if [[ ${munge_commit_line} == "true" ]]; then # Remove first line of commit message. vim_message="$(echo "${vim_message}" | sed -e '1s/^patch /vim-patch:/')" -- cgit From 1248c12666dc4d8fb801c2a8dbc355027f8f9bbe Mon Sep 17 00:00:00 2001 From: dundargoc <33953936+dundargoc@users.noreply.github.com> Date: Mon, 24 Oct 2022 13:24:08 +0200 Subject: ci(lintcommit): enforce common scope names #20393 This is useful to ensure related commits aren't spread out when generating the changelog. --- scripts/lintcommit.lua | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/lintcommit.lua b/scripts/lintcommit.lua index 16326cfe66..87a8e62503 100644 --- a/scripts/lintcommit.lua +++ b/scripts/lintcommit.lua @@ -86,11 +86,28 @@ local function validate_commit(commit_message) vim.inspect(allowed_types)) end - -- Check if scope is empty + -- Check if scope is appropriate if before_colon:match("%(") then local scope = vim.trim(before_colon:match("%((.*)%)")) + if scope == '' then - return [[Scope can't be empty.]] + return [[Scope can't be empty]] + end + + if vim.startswith(scope, "nvim_") then + return [[Scope should be "api" instead of "nvim_..."]] + end + + local alternative_scope = { + ['filetype.vim'] = 'filetype', + ['filetype.lua'] = 'filetype', + ['tree-sitter'] = 'treesitter', + ['ts'] = 'treesitter', + ['hl'] = 'highlight', + } + + if alternative_scope[scope] then + return ('Scope should be "%s" instead of "%s"'):format(alternative_scope[scope], scope) end end -- cgit From 69ffbda84d33d09107a18c56614e471ff4729593 Mon Sep 17 00:00:00 2001 From: dundargoc <33953936+dundargoc@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:27:52 +0200 Subject: build: preprocess vim patches with uncrustify #20786 This will enable a larger amount of chunks being automatically included due to fewer formatting differences between the vim and neovim files. The strategy is straightforward, if a bit tedious: - Get a list of all changed files. - Checkout parent commit. Copy all relevant files to a temporary location. - Checkout patch commit. Copy all relevant files to a temporary location. - Format .c and .h files with uncrustify. - Generate a diff from from these files. Closes https://github.com/neovim/neovim/issues/6226 --- scripts/vim-patch.sh | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 07c9848807..0a2b1df197 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -279,6 +279,53 @@ preprocess_patch() { "$file" > "$file".tmp && mv "$file".tmp "$file" } +uncrustify_patch() { + local commit="$1" + local changed_files=() + while IFS='' read -r file; do changed_files+=("$file"); done < <(git diff-tree --name-only --no-commit-id -r "${commit}") + + local patch_path=$NVIM_SOURCE_DIR/build/vim_patch + rm -rf "$patch_path" + mkdir -p "$patch_path"/{before,after,patch} + + git checkout --quiet "$commit"~ + for file in "${changed_files[@]}"; do + if [[ -e $file ]]; then + cp "$file" "$patch_path"/before + fi + done + + git checkout --quiet "$commit" + for file in "${changed_files[@]}"; do + if [[ -e $file ]]; then + cp "$file" "$patch_path"/after + fi + done + + # If the difference are drastic enough uncrustify may need to be used more + # than once. This is obviously a bug that needs to be fixed on uncrustify's + # end, but in the meantime this workaround is sufficient. + for _ in {1..2}; do + uncrustify -c "$NVIM_SOURCE_DIR"/src/uncrustify.cfg -q --replace --no-backup "$patch_path"/{before,after}/*.[ch] + done + + for file in "${changed_files[@]}"; do + local basename + basename=$(basename "$file") + local before=$patch_path/before/$basename + local after=$patch_path/after/$basename + local patchfile="$patch_path"/patch/"$basename".patch + if [[ ! -e $before ]] || [[ ! -e $after ]]; then + continue + fi + git --no-pager diff --no-index --patch --unified=5 --color=never "$before" "$after" > "$patchfile" + sed -E "s|$before|/$file|g" -i "$patchfile" + sed -E "s|$after|/$file|g" -i "$patchfile" + done + + cat "$patch_path"/patch/*.patch +} + get_vimpatch() { get_vim_sources @@ -287,7 +334,12 @@ get_vimpatch() { msg_ok "Found Vim revision '${vim_commit}'." local patch_content - patch_content="$(git --no-pager show --unified=5 --color=never -1 --pretty=medium "${vim_commit}")" + if check_executable uncrustify; then + patch_content="$(uncrustify_patch "${vim_commit}")" + git switch --quiet master + else + patch_content="$(git --no-pager show --unified=5 --color=never -1 --pretty=medium "${vim_commit}")" + fi cd "${NVIM_SOURCE_DIR}" -- cgit From 9f6502535b8227a514a9225376d0db03484c803b Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Thu, 27 Oct 2022 13:38:47 +0800 Subject: build(vim-patch.sh): handle added/removed files properly --- scripts/vim-patch.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 0a2b1df197..c2fac658a9 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -315,12 +315,11 @@ uncrustify_patch() { local before=$patch_path/before/$basename local after=$patch_path/after/$basename local patchfile="$patch_path"/patch/"$basename".patch - if [[ ! -e $before ]] || [[ ! -e $after ]]; then - continue - fi + [[ ! -e $before ]] && before=/dev/null + [[ ! -e $after ]] && after=/dev/null git --no-pager diff --no-index --patch --unified=5 --color=never "$before" "$after" > "$patchfile" - sed -E "s|$before|/$file|g" -i "$patchfile" - sed -E "s|$after|/$file|g" -i "$patchfile" + [[ "$before" != /dev/null ]] && sed -E "s|$before|/$file|g" -i "$patchfile" + [[ "$after" != /dev/null ]] && sed -E "s|$after|/$file|g" -i "$patchfile" done cat "$patch_path"/patch/*.patch -- cgit From 8d38e1ad34552293de9e562b73c727ec119fd847 Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Sun, 30 Oct 2022 08:05:50 +0800 Subject: build(vim-patch.sh): checkout files with path for uncrustify (#20863) --- scripts/vim-patch.sh | 44 +++++++++++++------------------------------- 1 file changed, 13 insertions(+), 31 deletions(-) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index c2fac658a9..0854c2f627 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -280,49 +280,32 @@ preprocess_patch() { } uncrustify_patch() { - local commit="$1" - local changed_files=() - while IFS='' read -r file; do changed_files+=("$file"); done < <(git diff-tree --name-only --no-commit-id -r "${commit}") + git diff --quiet || { + >&2 echo 'Vim source working tree dirty, aborting.' + exit 1 + } local patch_path=$NVIM_SOURCE_DIR/build/vim_patch rm -rf "$patch_path" - mkdir -p "$patch_path"/{before,after,patch} + mkdir -p "$patch_path"/{a,b} - git checkout --quiet "$commit"~ - for file in "${changed_files[@]}"; do - if [[ -e $file ]]; then - cp "$file" "$patch_path"/before - fi + local commit="$1" + for file in $(git diff-tree --name-only --no-commit-id -r --diff-filter=a "$commit"); do + git --work-tree="$patch_path"/a checkout --quiet "$commit"~ -- "$file" done - - git checkout --quiet "$commit" - for file in "${changed_files[@]}"; do - if [[ -e $file ]]; then - cp "$file" "$patch_path"/after - fi + for file in $(git diff-tree --name-only --no-commit-id -r --diff-filter=d "$commit"); do + git --work-tree="$patch_path"/b checkout --quiet "$commit" -- "$file" done + git reset --quiet --hard HEAD # If the difference are drastic enough uncrustify may need to be used more # than once. This is obviously a bug that needs to be fixed on uncrustify's # end, but in the meantime this workaround is sufficient. for _ in {1..2}; do - uncrustify -c "$NVIM_SOURCE_DIR"/src/uncrustify.cfg -q --replace --no-backup "$patch_path"/{before,after}/*.[ch] - done - - for file in "${changed_files[@]}"; do - local basename - basename=$(basename "$file") - local before=$patch_path/before/$basename - local after=$patch_path/after/$basename - local patchfile="$patch_path"/patch/"$basename".patch - [[ ! -e $before ]] && before=/dev/null - [[ ! -e $after ]] && after=/dev/null - git --no-pager diff --no-index --patch --unified=5 --color=never "$before" "$after" > "$patchfile" - [[ "$before" != /dev/null ]] && sed -E "s|$before|/$file|g" -i "$patchfile" - [[ "$after" != /dev/null ]] && sed -E "s|$after|/$file|g" -i "$patchfile" + uncrustify -c "$NVIM_SOURCE_DIR"/src/uncrustify.cfg -q --replace --no-backup "$patch_path"/{a,b}/src/*.[ch] done - cat "$patch_path"/patch/*.patch + (cd "$patch_path" && (git --no-pager diff --no-index --no-prefix --patch --unified=5 --color=never a/ b/ || true)) } get_vimpatch() { @@ -335,7 +318,6 @@ get_vimpatch() { local patch_content if check_executable uncrustify; then patch_content="$(uncrustify_patch "${vim_commit}")" - git switch --quiet master else patch_content="$(git --no-pager show --unified=5 --color=never -1 --pretty=medium "${vim_commit}")" fi -- cgit From 502b5ee10fedc97e1455e7365a10c9bad7268c67 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Sun, 30 Oct 2022 13:50:41 +0100 Subject: build(lint): add more shell scripts to lintsh --- scripts/download-unicode-files.sh | 19 ++++++------------- scripts/genappimage.sh | 12 +++++++----- scripts/git-log-pretty-since.sh | 14 +++++++------- scripts/release.sh | 4 ++-- scripts/update_terminfo.sh | 2 +- 5 files changed, 23 insertions(+), 28 deletions(-) (limited to 'scripts') diff --git a/scripts/download-unicode-files.sh b/scripts/download-unicode-files.sh index 4482cefa34..687b946e6f 100755 --- a/scripts/download-unicode-files.sh +++ b/scripts/download-unicode-files.sh @@ -3,11 +3,12 @@ set -e data_files="UnicodeData.txt CaseFolding.txt EastAsianWidth.txt" emoji_files="emoji-data.txt" +files="'$data_files $emoji_files'" UNIDIR_DEFAULT=src/unicode DOWNLOAD_URL_BASE_DEFAULT='http://unicode.org/Public' -if test x$1 = 'x--help' ; then +if test "$1" = '--help' ; then echo 'Usage:' echo " $0[ TARGET_DIRECTORY[ URL_BASE]]" echo @@ -16,6 +17,7 @@ if test x$1 = 'x--help' ; then echo echo "Default target directory is $PWD/${UNIDIR_DEFAULT}." echo "Default URL base is ${DOWNLOAD_URL_BASE_DEFAULT}." + exit 0 fi UNIDIR=${1:-$UNIDIR_DEFAULT} @@ -23,21 +25,12 @@ DOWNLOAD_URL_BASE=${2:-$DOWNLOAD_URL_BASE_DEFAULT} for filename in $data_files ; do curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/$filename" - ( - cd "$UNIDIR" - git add $filename - ) + git -C "$UNIDIR" add "$filename" done for filename in $emoji_files ; do curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/emoji/$filename" - ( - cd "$UNIDIR" - git add $filename - ) + git -C "$UNIDIR" add $filename done -( - cd "$UNIDIR" - git commit -m "feat: update unicode tables" -- $files -) +git -C "$UNIDIR" commit -m "feat: update unicode tables" . diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh index cc88ab5559..9944b5eb31 100755 --- a/scripts/genappimage.sh +++ b/scripts/genappimage.sh @@ -8,7 +8,8 @@ # App arch, used by generate_appimage. if [ -z "$ARCH" ]; then - export ARCH="$(arch)" + ARCH="$(arch)" + export ARCH fi TAG=$1 @@ -34,8 +35,9 @@ make install # App version, used by generate_appimage. VERSION=$("$ROOT_DIR"/build/bin/nvim --version | head -n 1 | grep -o 'v.*') +export VERSION -cd "$APP_BUILD_DIR" +cd "$APP_BUILD_DIR" || exit # Only downloads linuxdeploy if the remote file is different from local if [ -e "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage ]; then @@ -53,7 +55,7 @@ chmod +x "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage mkdir "$APP_DIR/usr/share/metainfo/" cp "$ROOT_DIR/runtime/nvim.appdata.xml" "$APP_DIR/usr/share/metainfo/" -cd "$APP_DIR" +cd "$APP_DIR" || exit ######################################################################## # AppDir complete. Now package it as an AppImage. @@ -71,7 +73,7 @@ exec "$(dirname "$(readlink -f "${0}")")/usr/bin/nvim" ${@+"$@"} EOF chmod 755 AppRun -cd "$APP_BUILD_DIR" # Get out of AppImage directory. +cd "$APP_BUILD_DIR" || exit # Get out of AppImage directory. # Set the name of the file generated by appimage export OUTPUT=nvim.appimage @@ -85,7 +87,7 @@ fi # - Expects: $ARCH, $APP, $VERSION env vars # - Expects: ./$APP.AppDir/ directory # - Produces: ./nvim.appimage -./linuxdeploy-x86_64.AppImage --appdir $APP.AppDir -d $ROOT_DIR/runtime/nvim.desktop -i \ +./linuxdeploy-x86_64.AppImage --appdir $APP.AppDir -d "$ROOT_DIR"/runtime/nvim.desktop -i \ "$ROOT_DIR/runtime/nvim.png" --output appimage # Moving the final executable to a different folder so it isn't in the diff --git a/scripts/git-log-pretty-since.sh b/scripts/git-log-pretty-since.sh index a0aa4354b6..95dcee23f5 100755 --- a/scripts/git-log-pretty-since.sh +++ b/scripts/git-log-pretty-since.sh @@ -16,9 +16,9 @@ __SINCE=$1 __INVMATCH=$2 is_merge_commit() { - git rev-parse $1 >/dev/null 2>&1 \ + git rev-parse "$1" >/dev/null 2>&1 \ || { echo "ERROR: invalid commit: $1"; exit 1; } - git log $1^2 >/dev/null 2>&1 && return 0 || return 1 + git log "$1"^2 >/dev/null 2>&1 && return 0 || return 1 } # Removes parens from issue/ticket/PR numbers. @@ -40,13 +40,13 @@ _format_ticketnums() { } for commit in $(git log --format='%H' --first-parent "$__SINCE"..HEAD); do - if is_merge_commit ${commit} ; then - if [ -z "$__INVMATCH" ] || ! git log --oneline ${commit}^1..${commit}^2 \ + if is_merge_commit "${commit}" ; then + if [ -z "$__INVMATCH" ] || ! git log --oneline "${commit}^1..${commit}^2" \ | >/dev/null 2>&1 grep -E "$__INVMATCH" ; then - git log -1 --oneline ${commit} - git log --format=' %h %s' ${commit}^1..${commit}^2 + git log -1 --oneline "${commit}" + git log --format=' %h %s' "${commit}^1..${commit}^2" fi else - git log -1 --oneline ${commit} + git log -1 --oneline "${commit}" fi done | _format_ticketnums diff --git a/scripts/release.sh b/scripts/release.sh index 380503662d..4321d96f62 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -59,8 +59,8 @@ _do_release_commit() { $__sed -i.bk 's/(NVIM_VERSION_PRERELEASE) "-dev"/\1 ""/' CMakeLists.txt if grep '(NVIM_API_PRERELEASE true)' CMakeLists.txt > /dev/null; then $__sed -i.bk 's/(NVIM_API_PRERELEASE) true/\1 false/' CMakeLists.txt - build/bin/nvim --api-info > test/functional/fixtures/api_level_$__API_LEVEL.mpack - git add test/functional/fixtures/api_level_$__API_LEVEL.mpack + build/bin/nvim --api-info > "test/functional/fixtures/api_level_$__API_LEVEL.mpack" + git add "test/functional/fixtures/api_level_${__API_LEVEL}.mpack" fi $__sed -i.bk 's,(),\1\ diff --git a/scripts/update_terminfo.sh b/scripts/update_terminfo.sh index 8a0937cc8c..775048f246 100755 --- a/scripts/update_terminfo.sh +++ b/scripts/update_terminfo.sh @@ -35,7 +35,7 @@ readonly -A entries=( db="$(mktemp -du)" print_bold() { - printf "\\e[1m$*\\e[0m" + printf "\\e[1m%b\\e[0m" "$*" } cd "$(git rev-parse --show-toplevel)" -- cgit From 4716a578ae0c3516d685495bb55e40c939a4ac87 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Sun, 23 Oct 2022 10:17:45 +0200 Subject: docs: fix typos --- scripts/lua2dox.lua | 4 ++-- scripts/pvscheck.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 86afe97a7e..83bf213f40 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -497,14 +497,14 @@ function TLua2DoX_filter.readfile(this, AppStamp, Filename) else this:warning(inStream:getLineNo(), 'something weird here') end - fn_magic = nil -- mustn't indavertently use it again + fn_magic = nil -- mustn't inadvertently use it again -- TODO: If we can make this learn how to generate these, that would be helpful. -- elseif string.find(line, "^M%['.*'%] = function") then -- state = 'in_function' -- it's a function -- outStream:writeln("function textDocument/publishDiagnostics(...){}") - -- fn_magic = nil -- mustn't indavertently use it again + -- fn_magic = nil -- mustn't inadvertently use it again else state = '' -- unknown if #line > 0 then -- we don't know what this line means, so just comment it out diff --git a/scripts/pvscheck.sh b/scripts/pvscheck.sh index 610c20eb48..97757c0848 100755 --- a/scripts/pvscheck.sh +++ b/scripts/pvscheck.sh @@ -328,7 +328,7 @@ realdir() {( patch_sources() {( local tgt="$1" ; shift - local only_bulid="${1}" ; shift + local only_build="${1}" ; shift get_pvs_comment "$tgt" -- cgit From f8c671827710c6e9cca3bfd60c32098b2be8239a Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 14 Nov 2022 18:04:36 +0000 Subject: feat(lua-api): avoid unnecessary allocations (#19877) Lua makes (or reuses) an internal copy of strings, so we can safely push buf pointers onto the stack. --- scripts/gen_vimdoc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index a044c8c39d..e77d3ea286 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -796,7 +796,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error', 'Arena')) + if a[0] not in ('void', 'Error', 'Arena', + 'lua_State')) if not fmt_vimhelp: c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) -- cgit From f1922e78a1df1b1d32779769432fb5586edf5fbb Mon Sep 17 00:00:00 2001 From: Gregory Anders Date: Sat, 5 Nov 2022 13:37:05 -0600 Subject: feat: add vim.secure.read() This function accepts a path to a file and prompts the user if the file is trusted. If the user confirms that the file is trusted, the contents of the file are returned. The user's decision is stored in a trust database at $XDG_STATE_HOME/nvim/trust. When this function is invoked with a path that is already marked as trusted in the trust database, the user is not prompted for a response. --- scripts/gen_vimdoc.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index e77d3ea286..a720f055ed 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -131,6 +131,7 @@ CONFIG = { 'filetype.lua', 'keymap.lua', 'fs.lua', + 'secure.lua', ], 'files': [ 'runtime/lua/vim/_editor.lua', @@ -140,6 +141,7 @@ CONFIG = { 'runtime/lua/vim/filetype.lua', 'runtime/lua/vim/keymap.lua', 'runtime/lua/vim/fs.lua', + 'runtime/lua/vim/secure.lua', ], 'file_patterns': '*.lua', 'fn_name_prefix': '', @@ -166,6 +168,7 @@ CONFIG = { 'filetype': 'vim.filetype', 'keymap': 'vim.keymap', 'fs': 'vim.fs', + 'secure': 'vim.secure', }, 'append_only': [ 'shared.lua', -- cgit From 0b79137c59fbe44bded76f123602e552dc6f7b03 Mon Sep 17 00:00:00 2001 From: zeertzjq Date: Tue, 29 Nov 2022 16:47:29 +0800 Subject: vim-patch:8.1.2001: some source files are too big (#21231) Problem: Some source files are too big. Solution: Move buffer and window related functions to evalbuffer.c and evalwindow.c. (Yegappan Lakshmanan, closes vim/vim#4898) https://github.com/vim/vim/commit/261f346f8154c0ec7094a4a211c653c74e9f7c2e --- scripts/vim-patch.sh | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'scripts') diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 0854c2f627..e676705560 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -242,6 +242,14 @@ preprocess_patch() { LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/userfunc\.c/\1\/eval\/userfunc\.c/g' \ "$file" > "$file".tmp && mv "$file".tmp "$file" + # Rename evalbuffer.c to eval/buffer.c + LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/evalbuffer\.c/\1\/eval\/buffer\.c/g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + + # Rename evalwindow.c to eval/window.c + LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/evalwindow\.c/\1\/eval\/window\.c/g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + # Rename map.c to mapping.c LC_ALL=C sed -e 's/\( [ab]\/src\/nvim\)\/map\(\.[ch]\)/\1\/mapping\2/g' \ "$file" > "$file".tmp && mv "$file".tmp "$file" -- cgit From 5093f38c9fed9fae04234035ea253862ba8375ef Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Tue, 22 Nov 2022 13:50:50 +0100 Subject: feat(help): highlighted codeblocks --- scripts/gen_help_html.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index e5251b7f25..e5e99b308a 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -489,7 +489,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end return string.format('
\n%s\n
\n', text) elseif node_name == 'line' then - if parent ~= 'codeblock' and (is_blank(text) or is_noise(text, stats.noise_lines)) then + if parent ~= 'code' and (is_blank(text) or is_noise(text, stats.noise_lines)) then return '' -- Discard common "noise" lines. end -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. @@ -535,7 +535,12 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return s elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) + -- TODO: use language for proper syntax highlighted code blocks elseif node_name == 'codeblock' then + return text + elseif node_name == 'language' then + return '' + elseif node_name == 'code' then if is_blank(text) then return '' end -- cgit From 9e1187e4896bebb481a3f9595155f2a40adbc45e Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Tue, 22 Nov 2022 21:23:33 +0100 Subject: feat(web): syntax highlighting via highlight.js download from https://highlightjs.org/download/ place `highlight/` directory next to `css/` style needs adapting for Neovim colors --- scripts/gen_help_html.lua | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index e5e99b308a..532e28ebb8 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -35,6 +35,7 @@ local spell_dict = { lua = 'Lua', VimL = 'Vimscript', } +local language = nil local M = {} @@ -489,7 +490,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end return string.format('
\n%s\n
\n', text) elseif node_name == 'line' then - if parent ~= 'code' and (is_blank(text) or is_noise(text, stats.noise_lines)) then + if (parent ~= 'codeblock' or parent ~= 'code') and (is_blank(text) or is_noise(text, stats.noise_lines)) then return '' -- Discard common "noise" lines. end -- XXX: Avoid newlines (too much whitespace) after block elements in old (preformatted) layout. @@ -535,16 +536,23 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) return s elseif node_name == 'argument' then return ('%s{%s}'):format(ws(), text) - -- TODO: use language for proper syntax highlighted code blocks elseif node_name == 'codeblock' then return text elseif node_name == 'language' then + language = node_text(root) return '' elseif node_name == 'code' then if is_blank(text) then return '' end - return ('
%s
'):format(trim(trim_indent(text), 2)) + local code + if language then + code = ('
%s
'):format(language,trim(trim_indent(text), 2)) + language = nil + else + code = ('
%s
'):format(trim(trim_indent(text), 2)) + end + return code elseif node_name == 'tag' then -- anchor if root:has_error() then return text @@ -690,6 +698,9 @@ local function gen_one(fname, to_fname, old, commit) + + + %s - Neovim docs -- cgit From 0b05bd87c04f9cde5c84a062453619349e370795 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Wed, 23 Nov 2022 12:31:49 +0100 Subject: docs(gen): support language annotation in docstrings --- scripts/gen_vimdoc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index a720f055ed..801c8ea790 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -496,7 +496,12 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, if n.nodeName == 'preformatted': o = get_text(n, preformatted=True) ensure_nl = '' if o[-1] == '\n' else '\n' - text += '>{}{}\n<'.format(ensure_nl, o) + if o[0:4] == 'lua\n': + text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1]) + elif o[0:4] == 'vim\n': + text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1]) + else: + text += '>{}{}\n<'.format(ensure_nl, o) elif is_inline(n): text = doc_wrap(get_text(n), indent=indent, width=width) -- cgit From f8aa2a0deaf473af0e6b4640356eaf5477c6ee90 Mon Sep 17 00:00:00 2001 From: Gregory Anders <8965202+gpanders@users.noreply.github.com> Date: Wed, 7 Dec 2022 01:27:41 -0700 Subject: docs(treesitter): use full function names in tags (#21321) --- scripts/gen_vimdoc.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 801c8ea790..78f3c458eb 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -258,17 +258,11 @@ CONFIG = { if name.lower() == 'treesitter' else f'*lua-treesitter-{name.lower()}*'), 'fn_helptag_fmt': lambda fstem, name: ( - f'*{name}()*' - if name != 'new' - else f'*{fstem}.{name}()*'), - # 'fn_helptag_fmt': lambda fstem, name: ( - # f'*vim.treesitter.{name}()*' - # if fstem == 'treesitter' - # else ( - # '*vim.lsp.client*' - # # HACK. TODO(justinmk): class/structure support in lua2dox - # if 'lsp.client' == f'{fstem}.{name}' - # else f'*vim.lsp.{fstem}.{name}()*')), + f'*vim.{fstem}.{name}()*' + if fstem == 'treesitter' + else f'*{name}()*' + if name[0].isupper() + else f'*vim.treesitter.{fstem}.{name}()*'), 'module_override': {}, 'append_only': [], } -- cgit From 9f035559defd9d575f37fd825954610065d9cf96 Mon Sep 17 00:00:00 2001 From: John Drouhard Date: Wed, 23 Nov 2022 10:06:36 -0600 Subject: feat(lsp): initial support for semantic token highlighting * credit to @smolck and @theHamsta for their contributions in laying the groundwork for this feature and for their work on some of the helper utility functions and tests --- scripts/gen_vimdoc.py | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 78f3c458eb..b18179b498 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -183,6 +183,7 @@ CONFIG = { 'diagnostic.lua', 'codelens.lua', 'tagfunc.lua', + 'semantic_tokens.lua', 'handlers.lua', 'util.lua', 'log.lua', -- cgit From ea39fc2cadc1d87109216da354a876427eeea31a Mon Sep 17 00:00:00 2001 From: Dave Lage Date: Thu, 8 Dec 2022 17:00:18 -0500 Subject: docs: dark/light color/accessibilty pass for generated html docs #21345 --- scripts/gen_help_html.lua | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 532e28ebb8..3a0ed5ffc5 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -843,8 +843,14 @@ end local function gen_css(fname) local css = [[ :root { - --code-color: #008B8B; - --tag-color: gray; + --code-color: #004b4b; + --tag-color: #095943; + } + @media (prefers-color-scheme: dark) { + :root { + --code-color: #00c243; + --tag-color: #00b7b7; + } } @media (min-width: 40em) { .toc { @@ -863,11 +869,6 @@ local function gen_css(fname) display: block; } } - @media (prefers-color-scheme: dark) { - :root { - --code-color: cyan; - } - } .toc { /* max-width: 12rem; */ height: 85%; /* Scroll if there are too many items. https://github.com/neovim/neovim.github.io/issues/297 */ @@ -887,7 +888,7 @@ local function gen_css(fname) } h1, h2, h3, h4, h5 { font-family: sans-serif; - border-bottom: 1px solid #41464bd6; /*rgba(0, 0, 0, .9);*/ + border-bottom: 1px solid var(--tag-color); /*rgba(0, 0, 0, .9);*/ } h3, h4, h5 { border-bottom-style: dashed; -- cgit From 1c324cb1927e03b5a3584a8982e3d5029498f14e Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 11 Dec 2022 21:41:26 -0500 Subject: docs #20986 - https://github.com/neovim/tree-sitter-vimdoc v1.2.4 eliminates most errors in pi_netrw.txt, so we can remove that workaround from ignore_parse_error(). - improved codeblock --- scripts/gen_help_html.lua | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 3a0ed5ffc5..78fb917764 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -296,12 +296,11 @@ local function ignore_invalid(s) ) end -local function ignore_parse_error(s, fname) - local helpfile = vim.fs.basename(fname) - return (helpfile == 'pi_netrw.txt' +local function ignore_parse_error(s) + return ( -- Ignore parse errors for unclosed tag. -- This is common in vimdocs and is treated as plaintext by :help. - or s:find("^[`'|*]") + s:find("^[`'|*]") ) end @@ -370,7 +369,7 @@ local function visit_validate(root, level, lang_tree, opt, stats) end if node_name == 'ERROR' then - if ignore_parse_error(text, opt.fname) then + if ignore_parse_error(text) then return end -- Store the raw text to give context to the error report. @@ -582,7 +581,7 @@ local function visit_node(root, level, lang_tree, headings, opt, stats) end return s elseif node_name == 'ERROR' then - if ignore_parse_error(trimmed, opt.fname) then + if ignore_parse_error(trimmed) then return text end -- cgit From ef91146efcece1b6d97152251e7137d301146189 Mon Sep 17 00:00:00 2001 From: Folke Lemaitre Date: Wed, 14 Dec 2022 10:46:54 +0100 Subject: feat: `vim.inspect_pos`, `vim.show_pos`, `:Inspect` --- scripts/gen_vimdoc.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index b18179b498..502c9161b6 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -125,6 +125,7 @@ CONFIG = { 'filename': 'lua.txt', 'section_order': [ '_editor.lua', + '_inspector.lua', 'shared.lua', 'uri.lua', 'ui.lua', @@ -142,11 +143,13 @@ CONFIG = { 'runtime/lua/vim/keymap.lua', 'runtime/lua/vim/fs.lua', 'runtime/lua/vim/secure.lua', + 'runtime/lua/vim/_inspector.lua', ], 'file_patterns': '*.lua', 'fn_name_prefix': '', 'section_name': { 'lsp.lua': 'core', + '_inspector.lua': 'inspector', }, 'section_fmt': lambda name: ( 'Lua module: vim' @@ -163,6 +166,7 @@ CONFIG = { 'module_override': { # `shared` functions are exposed on the `vim` module. 'shared': 'vim', + '_inspector': 'vim', 'uri': 'vim', 'ui': 'vim.ui', 'filetype': 'vim.filetype', @@ -346,6 +350,17 @@ def self_or_child(n): return n.childNodes[0] +def align_tags(line): + tag_regex = r"\s(\*.+?\*)(?:\s|$)" + tags = re.findall(tag_regex, line) + + if len(tags) > 0: + line = re.sub(tag_regex, "", line) + tags = " " + " ".join(tags) + line = line + (" " * (78 - len(line) - len(tags))) + tags + return line + + def clean_lines(text): """Removes superfluous lines. @@ -950,7 +965,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): start = end - func_doc = "\n".join(split_lines) + func_doc = "\n".join(map(align_tags, split_lines)) if (name.startswith(CONFIG[target]['fn_name_prefix']) and name != "nvim_error_event"): -- cgit From 7067cde6576b19f6ea910a505f049b3c746c115c Mon Sep 17 00:00:00 2001 From: dundargoc <33953936+dundargoc@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:50:24 +0100 Subject: build(lintsh): double quote to prevent word splitting (#21571) --- scripts/download-unicode-files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/download-unicode-files.sh b/scripts/download-unicode-files.sh index 687b946e6f..f0fd4c66ea 100755 --- a/scripts/download-unicode-files.sh +++ b/scripts/download-unicode-files.sh @@ -30,7 +30,7 @@ done for filename in $emoji_files ; do curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/emoji/$filename" - git -C "$UNIDIR" add $filename + git -C "$UNIDIR" add "$filename" done git -C "$UNIDIR" commit -m "feat: update unicode tables" . -- cgit From 4703e561d5bc0eef13da171c4f8f8b6e02ae4883 Mon Sep 17 00:00:00 2001 From: bfredl Date: Thu, 29 Dec 2022 17:36:28 +0100 Subject: chore: forward-port changes from v0.8.2 release --- scripts/cliff.toml | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 scripts/cliff.toml (limited to 'scripts') diff --git a/scripts/cliff.toml b/scripts/cliff.toml new file mode 100644 index 0000000000..3fc10e5d16 --- /dev/null +++ b/scripts/cliff.toml @@ -0,0 +1,73 @@ +# configuration file for git-cliff (0.1.0) + +[changelog] +# changelog header +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +""" +# template for the changelog body +# https://tera.netlify.app/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | upper_first }} + {% for commit in commits%}\ + {% if not commit.scope %}\ + - {{ commit.message | upper_first }} + {% endif %}\ + {% endfor %}\ + {% for group, commits in commits | group_by(attribute="scope") %}\ + {% for commit in commits %}\ + - **{{commit.scope}}**: {{ commit.message | upper_first }} + {% endfor %}\ + {% endfor %} +{% endfor %}\n +""" +# remove the leading and trailing whitespace from the template +trim = true +# changelog footer +footer = """ + +""" + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = true +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ +# { pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](https://github.com/neovim/neovim/issues/${2}))"}, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "!:", group = "Breaking"}, + { message = "^feat", group = "Features"}, + { message = "^fix", group = "Bug Fixes"}, + { message = "^doc", group = "Documentation"}, + { message = "^perf", group = "Performance"}, + { message = "^refactor", group = "Refactor"}, + { message = "^test", group = "Testing"}, + { message = "^chore", group = "Miscellaneous Tasks"}, + { message = "^build", group = "Build System"}, + { message = "^Revert", group = "Reverted Changes"}, +] +# filter out the commits that are not matched by commit parsers +filter_commits = true +# glob pattern for matching git tags +tag_pattern = "v[0-9]*" +# regex for skipping tags +skip_tags = "v0.1.0-beta.1" +# regex for ignoring tags +ignore_tags = "" +# sort the tags chronologically +date_order = false +# sort the commits inside sections by oldest/newest order +sort_commits = "oldest" -- cgit From 5841a97500bffa5a2b9eed2eb41025f5587790ba Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 3 Jan 2023 10:07:43 +0000 Subject: feat!: remove hardcopy Co-authored-by: Justin M. Keyes --- scripts/gen_help_html.lua | 1 - scripts/vim-patch.sh | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 78fb917764..4f42633c57 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -88,7 +88,6 @@ local exclude_invalid_urls = { ["http://papp.plan9.de"] = "syntax.txt", ["http://wiki.services.openoffice.org/wiki/Dictionaries"] = "spell.txt", ["http://www.adapower.com"] = "ft_ada.txt", - ["http://www.ghostscript.com/"] = "print.txt", ["http://www.jclark.com/"] = "quickfix.txt", } diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index e676705560..f9f7330097 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -193,15 +193,15 @@ preprocess_patch() { 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/\<\%('"${na_files}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove *.proto, Make*, INSTALL*, gui_*, beval.*, some if_*, gvim, libvterm, tee, VisVim, xpm, xxd - local na_src='auto\|configure.*\|GvimExt\|libvterm\|proto\|tee\|VisVim\|xpm\|xxd\|Make.*\|INSTALL.*\|beval.*\|gui.*\|if_cscop\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' + local na_src='auto\|configure.*\|GvimExt\|hardcopy.*\|libvterm\|proto\|tee\|VisVim\|xpm\|xxd\|Make.*\|INSTALL.*\|beval.*\|gui.*\|if_cscop\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%(testdir/\)\@@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove runtime files ported to Lua. - local na_rt='filetype\.vim\|scripts\.vim\|autoload\/ft\/dist\.vim' + local na_rt='filetype\.vim\|scripts\.vim\|autoload\/ft\/dist\.vim\|print\/.*' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('"${na_rt}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove unwanted Vim doc files. - local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags' + local na_doc='channel\.txt\|if_cscop\.txt\|netbeans\.txt\|os_\w\+\.txt\|print\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|vim9\.txt\|sponsor\.txt\|intro\.txt\|tags' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('"${na_doc}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove "Last change ..." changes in doc files. @@ -212,7 +212,7 @@ preprocess_patch() { 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<\%('"${na_src_testdir}"'\)\>@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove testdir/test_*.vim files - local na_src_testdir='balloon.*\|channel.*\|crypt\.vim\|cscope\.vim\|gui.*\|job_fails\.vim\|json\.vim\|mzscheme\.vim\|netbeans.*\|paste\.vim\|popupwin.*\|restricted\.vim\|shortpathname\.vim\|tcl\.vim\|terminal.*\|xxd\.vim' + local na_src_testdir='balloon.*\|channel.*\|crypt\.vim\|cscope\.vim\|gui.*\|hardcopy\.vim\|job_fails\.vim\|json\.vim\|mzscheme\.vim\|netbeans.*\|paste\.vim\|popupwin.*\|restricted\.vim\|shortpathname\.vim\|tcl\.vim\|terminal.*\|xxd\.vim' 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\@norm! d/\v(^diff)|%$ ' +w +q "$file" # Remove version.c #7555 -- cgit From 1bd6e4469bb84bb49b342c10d9aa14ffd5f01187 Mon Sep 17 00:00:00 2001 From: Chris DeLuca <637174+bronzehedwick@users.noreply.github.com> Date: Wed, 4 Jan 2023 10:15:08 -0500 Subject: docs(website): soft wrap code blocks #21644 Use `white-space: pre-wrap` to preserve white space as per `pre`, but to allow line wrapping if the display runs out of horizontal space. This prevents lines overflowing their box, and causing horizontal scrolling across the entire page on small screens. This `pre-wrap` technique is used by GitHub to format code for mobile. See https://developer.mozilla.org/en-US/docs/Web/CSS/white-space#pre-wrap --- scripts/gen_help_html.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index 4f42633c57..fa7c14eaa3 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -961,7 +961,7 @@ local function gen_css(fname) pre { /* Tabs are used in codeblocks only for indentation, not alignment, so we can aggressively shrink them. */ tab-size: 2; - white-space: pre; + white-space: pre-wrap; line-height: 1.3; /* Important for ascii art. */ overflow: visible; /* font-family: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace; */ -- cgit From 7c94bcd2d77e2e54b8836ab8325460a367b79eae Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Mon, 20 Sep 2021 19:00:50 -0700 Subject: feat(lua)!: execute Lua with "nvim -l" Problem: Nvim has Lua but the "nvim" CLI can't easily be used to execute Lua scripts, especially scripts that take arguments or produce output. Solution: - support "nvim -l [args...]" for running scripts. closes #15749 - exit without +q - remove lua2dox_filter - remove Doxyfile. This wasn't used anyway, because the doxygen config is inlined in gen_vimdoc.py (`Doxyfile` variable). - use "nvim -l" in docs-gen CI job Examples: $ nvim -l scripts/lua2dox.lua --help Lua2DoX (0.2 20130128) ... $ echo "print(vim.inspect(_G.arg))" | nvim -l - --arg1 --arg2 $ echo 'print(vim.inspect(vim.api.nvim_buf_get_text(1,0,0,-1,-1,{})))' | nvim +"put ='text'" -l - TODO? -e executes Lua code -l loads a module -i enters REPL _after running the other arguments_. --- scripts/gen_vimdoc.py | 22 +++++++----- scripts/lua2dox.lua | 32 +++--------------- scripts/lua2dox_filter | 90 -------------------------------------------------- 3 files changed, 18 insertions(+), 126 deletions(-) delete mode 100755 scripts/lua2dox_filter (limited to 'scripts') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 502c9161b6..8e1d6ef80a 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -55,11 +55,19 @@ if sys.version_info < MIN_PYTHON_VERSION: doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"], universal_newlines=True).split()[0].split('.'))) +# Until 0.9 is released, need this hacky way to check that "nvim -l foo.lua" works. +nvim_version = list(line for line in subprocess.check_output(['nvim', '-h'], universal_newlines=True).split('\n') + if '-l ' in line) + if doxygen_version < MIN_DOXYGEN_VERSION: print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION)) print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version)) sys.exit(1) +if len(nvim_version) == 0: + print("\nRequires 'nvim -l' feature, see https://github.com/neovim/neovim/pull/18706") + sys.exit(1) + # DEBUG = ('DEBUG' in os.environ) INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) @@ -79,7 +87,7 @@ base_dir = os.path.dirname(os.path.dirname(script_path)) out_dir = os.path.join(base_dir, 'tmp-{target}-doc') filter_cmd = '%s %s' % (sys.executable, script_path) msgs = [] # Messages to show on exit. -lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter') +lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') CONFIG = { 'api': { @@ -993,7 +1001,7 @@ def delete_lines_below(filename, tokenstr): fp.writelines(lines[0:i]) -def main(config, args): +def main(doxygen_config, args): """Generates: 1. Vim :help docs @@ -1021,7 +1029,7 @@ def main(config, args): # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL)) p.communicate( - config.format( + doxygen_config.format( input=' '.join( [f'"{file}"' for file in CONFIG[target]['files']]), output=output_dir, @@ -1108,11 +1116,7 @@ def main(config, args): fn_map_full.update(fn_map) if len(sections) == 0: - if target == 'lua': - fail(f'no sections for target: {target} (this usually means' - + ' "luajit" was not found by scripts/lua2dox_filter)') - else: - fail(f'no sections for target: {target}') + fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') if len(sections) > len(CONFIG[target]['section_order']): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( @@ -1159,7 +1163,7 @@ def main(config, args): def filter_source(filename): name, extension = os.path.splitext(filename) if extension == '.lua': - p = subprocess.run([lua2dox_filter, filename], stdout=subprocess.PIPE) + p = subprocess.run(['nvim', '-l', lua2dox, filename], stdout=subprocess.PIPE) op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8')) print(op) else: diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 83bf213f40..19f8f8141d 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -27,14 +27,13 @@ http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm Running ------- -This file "lua2dox.lua" gets called by "lua2dox_filter" (bash). +This script "lua2dox.lua" gets called by "gen_vimdoc.py". Doxygen must be on your system. You can experiment like so: - Run "doxygen -g" to create a default Doxyfile. -- Then alter it to let it recognise lua. Add the two following lines: +- Then alter it to let it recognise lua. Add the following line: FILE_PATTERNS = *.lua - FILTER_PATTERNS = *.lua=lua2dox_filter - Then run "doxygen". The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. @@ -117,26 +116,6 @@ local function string_split(Str, Pattern) return splitStr end ---! \class TCore_Commandline ---! \brief reads/parses commandline -local TCore_Commandline = class() - ---! \brief constructor -function TCore_Commandline.init(this) - this.argv = arg - this.parsed = {} - this.params = {} -end - ---! \brief get value -function TCore_Commandline.getRaw(this, Key, Default) - local val = this.argv[Key] - if not val then - val = Default - end - return val -end - ------------------------------- --! \brief file buffer --! @@ -147,7 +126,7 @@ local TStream_Read = class() --! --! \param Filename name of file to read (or nil == stdin) function TStream_Read.getContents(this, Filename) - assert(Filename) + assert(Filename, ('invalid file: %s'):format(Filename)) -- get lines from file -- syphon lines to our table local filecontents = {} @@ -548,15 +527,14 @@ end local This_app = TApp() --main -local cl = TCore_Commandline() -local argv1 = cl:getRaw(2) +local argv1 = arg[1] if argv1 == '--help' then TCore_IO_writeln(This_app:getVersion()) TCore_IO_writeln(This_app:getCopyright()) TCore_IO_writeln([[ run as: - lua2dox_filter + nvim -l scripts/lua2dox.lua -------------- Param: : interprets filename diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter deleted file mode 100755 index e3fa95d0cf..0000000000 --- a/scripts/lua2dox_filter +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -########################################################################### -# Copyright (C) 2012 by Simon Dales # -# simon@purrsoft.co.uk # -# # -# This program is free software; you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation; either version 2 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program; if not, write to the # -# Free Software Foundation, Inc., # -# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -########################################################################### -LANG="" - -##! \brief test executable to see if it exists -test_executable() { - P_EXE="$1" - ######### - WHICH=$(which "$P_EXE") - if test -z "${WHICH}"; then - echo "not found \"${P_EXE}\"" - else - EXE="${P_EXE}" - fi -} - -##! \brief sets the lua interpreter -set_lua() { - if test -z "${EXE}"; then - test_executable '.deps/usr/bin/luajit' - fi - - if test -z "${EXE}"; then - test_executable 'luajit' - fi - - if test -z "${EXE}"; then - test_executable 'lua' - fi -} - -##! \brief makes canonical name of file -##! -##! Note that "readlink -f" doesn't work in MacOSX -##! -do_readlink() { - pushd . > /dev/null - TARGET_FILE=$1 - - cd "$(dirname $TARGET_FILE)" - TARGET_FILE=$(basename "$TARGET_FILE") - - # Iterate down a (possible) chain of symlinks - while [ -L "$TARGET_FILE" ]; do - TARGET_FILE=$(readlink "$TARGET_FILE") - cd $(dirname "$TARGET_FILE") - TARGET_FILE=$(basename "$TARGET_FILE") - done - - PHYS_DIR=$(pwd -P) - RESULT=$PHYS_DIR - popd > /dev/null -} - -##main -set_lua -if test -z "${EXE}"; then - echo "no lua interpreter found" - exit 1 -else - BASENAME=$(basename "$0") - do_readlink "$0" - DIRNAME="${RESULT}" - - LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}" - #echo "lua[${LUASCRIPT}]" - - ${EXE} ${LUASCRIPT} $@ -fi - -##eof -- cgit From b741788a3ab3f470b99e2f3f38a9258c7464067a Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 5 Jan 2023 18:15:49 +0100 Subject: lint: bump_deps.lua --- scripts/bump_deps.lua | 493 +++++++++++++++++++++++++++----------------------- 1 file changed, 262 insertions(+), 231 deletions(-) (limited to 'scripts') diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua index 17e3fd35d6..f16fb4f2a3 100644 --- a/scripts/bump_deps.lua +++ b/scripts/bump_deps.lua @@ -17,19 +17,19 @@ local M = {} local _trace = false -local required_branch_prefix = "bump-" -local commit_prefix = "build(deps): " +local required_branch_prefix = 'bump-' +local commit_prefix = 'build(deps): ' -- Print message local function p(s) - vim.cmd("set verbose=1") - vim.api.nvim_echo({ { s, "" } }, false, {}) - vim.cmd("set verbose=0") + vim.cmd('set verbose=1') + vim.api.nvim_echo({ { s, '' } }, false, {}) + vim.cmd('set verbose=0') end local function die() - p("") - vim.cmd("cquit 1") + p('') + vim.cmd('cquit 1') end -- Executes and returns the output of `cmd`, or nil on failure. @@ -37,307 +37,338 @@ end -- -- Prints `cmd` if `trace` is enabled. local function _run(cmd, die_on_fail, die_msg) - if _trace then - p("run: " .. vim.inspect(cmd)) - end - local rv = vim.trim(vim.fn.system(cmd)) or "" - if vim.v.shell_error ~= 0 then - if die_on_fail then - if _trace then - p(rv) - end - p(die_msg) - die() - end - return nil - end - return rv + if _trace then + p('run: ' .. vim.inspect(cmd)) + end + local rv = vim.trim(vim.fn.system(cmd)) or '' + if vim.v.shell_error ~= 0 then + if die_on_fail then + if _trace then + p(rv) + end + p(die_msg) + die() + end + return nil + end + return rv end -- Run a command, return nil on failure local function run(cmd) - return _run(cmd, false, "") + return _run(cmd, false, '') end -- Run a command, die on failure with err_msg local function run_die(cmd, err_msg) - return _run(cmd, true, err_msg) + return _run(cmd, true, err_msg) end local function require_executable(cmd) - local cmd_path = run_die({ "command", "-v", cmd }, cmd .. " not found!") - run_die({ "test", "-x", cmd_path }, cmd .. " is not executable") + local cmd_path = run_die({ 'command', '-v', cmd }, cmd .. ' not found!') + run_die({ 'test', '-x', cmd_path }, cmd .. ' is not executable') end local function rm_file_if_present(path_to_file) - run({ "rm", "-f", path_to_file }) + run({ 'rm', '-f', path_to_file }) end local nvim_src_dir = vim.fn.getcwd() -local temp_dir = nvim_src_dir .. "/tmp" -run({ "mkdir", "-p", temp_dir }) +local temp_dir = nvim_src_dir .. '/tmp' +run({ 'mkdir', '-p', temp_dir }) local function get_dependency(dependency_name) - local dependency_table = { - ["LuaJIT"] = { - repo = "LuaJIT/LuaJIT", - symbol = "LUAJIT", - }, - ["libuv"] = { - repo = "libuv/libuv", - symbol = "LIBUV", - }, - ["Luv"] = { - repo = "luvit/luv", - symbol = "LUV", - }, - ["tree-sitter"] = { - repo = "tree-sitter/tree-sitter", - symbol = "TREESITTER", - }, - } - local dependency = dependency_table[dependency_name] - if dependency == nil then - p("Not a dependency: " .. dependency_name) - die() - end - dependency.name = dependency_name - return dependency + local dependency_table = { + ['LuaJIT'] = { + repo = 'LuaJIT/LuaJIT', + symbol = 'LUAJIT', + }, + ['libuv'] = { + repo = 'libuv/libuv', + symbol = 'LIBUV', + }, + ['Luv'] = { + repo = 'luvit/luv', + symbol = 'LUV', + }, + ['tree-sitter'] = { + repo = 'tree-sitter/tree-sitter', + symbol = 'TREESITTER', + }, + } + local dependency = dependency_table[dependency_name] + if dependency == nil then + p('Not a dependency: ' .. dependency_name) + die() + end + dependency.name = dependency_name + return dependency end local function get_gh_commit_sha(repo, ref) - require_executable("gh") + require_executable('gh') - local sha = run_die( - { "gh", "api", "repos/" .. repo .. "/commits/" .. ref, "--jq", ".sha" }, - "Failed to get commit hash from GitHub. Not a valid ref?" - ) - return sha + local sha = run_die( + { 'gh', 'api', 'repos/' .. repo .. '/commits/' .. ref, '--jq', '.sha' }, + 'Failed to get commit hash from GitHub. Not a valid ref?' + ) + return sha end local function get_archive_info(repo, ref) - require_executable("curl") + require_executable('curl') - local archive_name = ref .. ".tar.gz" - local archive_path = temp_dir .. "/" .. archive_name - local archive_url = "https://github.com/" .. repo .. "/archive/" .. archive_name + local archive_name = ref .. '.tar.gz' + local archive_path = temp_dir .. '/' .. archive_name + local archive_url = 'https://github.com/' .. repo .. '/archive/' .. archive_name - rm_file_if_present(archive_path) - run_die({ "curl", "-sL", archive_url, "-o", archive_path }, "Failed to download archive from GitHub") + rm_file_if_present(archive_path) + run_die( + { 'curl', '-sL', archive_url, '-o', archive_path }, + 'Failed to download archive from GitHub' + ) - local archive_sha = run({ "sha256sum", archive_path }):gmatch("%w+")() - return { url = archive_url, sha = archive_sha } + local archive_sha = run({ 'sha256sum', archive_path }):gmatch('%w+')() + return { url = archive_url, sha = archive_sha } end local function write_cmakelists_line(symbol, kind, value) - require_executable("sed") - - local cmakelists_path = nvim_src_dir .. "/" .. "cmake.deps/CMakeLists.txt" - run_die({ - "sed", - "-i", - "-e", - "s/set(" .. symbol .. "_" .. kind .. ".*$" .. "/set(" .. symbol .. "_" .. kind .. " " .. value .. ")" .. "/", - cmakelists_path, - }, "Failed to write " .. cmakelists_path) + require_executable('sed') + + local cmakelists_path = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt' + run_die({ + 'sed', + '-i', + '-e', + 's/set(' + .. symbol + .. '_' + .. kind + .. '.*$' + .. '/set(' + .. symbol + .. '_' + .. kind + .. ' ' + .. value + .. ')' + .. '/', + cmakelists_path, + }, 'Failed to write ' .. cmakelists_path) end local function explicit_create_branch(dep) - require_executable("git") - - local checked_out_branch = run({ "git", "rev-parse", "--abbrev-ref", "HEAD" }) - if checked_out_branch ~= "master" then - p("Not on master!") - die() - end - run_die({ "git", "checkout", "-b", "bump-" .. dep }, "git failed to create branch") + require_executable('git') + + local checked_out_branch = run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' }) + if checked_out_branch ~= 'master' then + p('Not on master!') + die() + end + run_die({ 'git', 'checkout', '-b', 'bump-' .. dep }, 'git failed to create branch') end local function verify_branch(new_branch_suffix) - require_executable("git") - - local checked_out_branch = run({ "git", "rev-parse", "--abbrev-ref", "HEAD" }) - if not checked_out_branch:match("^" .. required_branch_prefix) then - p("Current branch '" .. checked_out_branch .. "' doesn't seem to start with " .. required_branch_prefix) - p("Checking out to bump-" .. new_branch_suffix) - explicit_create_branch(new_branch_suffix) - end + require_executable('git') + + local checked_out_branch = run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' }) + if not checked_out_branch:match('^' .. required_branch_prefix) then + p( + "Current branch '" + .. checked_out_branch + .. "' doesn't seem to start with " + .. required_branch_prefix + ) + p('Checking out to bump-' .. new_branch_suffix) + explicit_create_branch(new_branch_suffix) + end end local function update_cmakelists(dependency, archive, comment) - require_executable("git") - - verify_branch(dependency.name) - - local changed_file = nvim_src_dir .. "/" .. "cmake.deps/CMakeLists.txt" - - p("Updating " .. dependency.name .. " to " .. archive.url .. "\n") - write_cmakelists_line(dependency.symbol, "URL", archive.url:gsub("/", "\\/")) - write_cmakelists_line(dependency.symbol, "SHA256", archive.sha) - run_die( - { "git", "commit", changed_file, "-m", commit_prefix .. "bump " .. dependency.name .. " to " .. comment }, - "git failed to commit" - ) + require_executable('git') + + verify_branch(dependency.name) + + local changed_file = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt' + + p('Updating ' .. dependency.name .. ' to ' .. archive.url .. '\n') + write_cmakelists_line(dependency.symbol, 'URL', archive.url:gsub('/', '\\/')) + write_cmakelists_line(dependency.symbol, 'SHA256', archive.sha) + run_die( + { + 'git', + 'commit', + changed_file, + '-m', + commit_prefix .. 'bump ' .. dependency.name .. ' to ' .. comment, + }, + 'git failed to commit' + ) end local function verify_cmakelists_committed() - require_executable("git") + require_executable('git') - local cmakelists_path = nvim_src_dir .. "/" .. "cmake.deps/CMakeLists.txt" - run_die({ "git", "diff", "--quiet", "HEAD", "--", cmakelists_path }, cmakelists_path .. " has uncommitted changes") + local cmakelists_path = nvim_src_dir .. '/' .. 'cmake.deps/CMakeLists.txt' + run_die( + { 'git', 'diff', '--quiet', 'HEAD', '--', cmakelists_path }, + cmakelists_path .. ' has uncommitted changes' + ) end local function warn_luv_symbol() - p("warning: " .. get_dependency("Luv").symbol .. "_VERSION will not be updated") + p('warning: ' .. get_dependency('Luv').symbol .. '_VERSION will not be updated') end -- return first 9 chars of commit local function short_commit(commit) - return string.sub(commit, 1, 9) + return string.sub(commit, 1, 9) end -- TODO: remove hardcoded fork local function gh_pr(pr_title, pr_body) - require_executable("gh") - - local pr_url = run_die({ - "gh", - "pr", - "create", - "--title", - pr_title, - "--body", - pr_body, - }, "Failed to create PR") - return pr_url + require_executable('gh') + + local pr_url = run_die({ + 'gh', + 'pr', + 'create', + '--title', + pr_title, + '--body', + pr_body, + }, 'Failed to create PR') + return pr_url end local function find_git_remote(fork) - require_executable("git") - - local remotes = run({ "git", "remote", "-v" }) - local git_remote = "" - for remote in remotes:gmatch("[^\r\n]+") do - local words = {} - for word in remote:gmatch("%w+") do - table.insert(words, word) - end - local match = words[1]:match("/github.com[:/]neovim/neovim/") - if fork == "fork" then - match = not match - end - if match and words[3] == "(fetch)" then - git_remote = words[0] - break - end - end - if git_remote == "" then - git_remote = "origin" - end - return git_remote + require_executable('git') + + local remotes = run({ 'git', 'remote', '-v' }) + local git_remote = '' + for remote in remotes:gmatch('[^\r\n]+') do + local words = {} + for word in remote:gmatch('%w+') do + table.insert(words, word) + end + local match = words[1]:match('/github.com[:/]neovim/neovim/') + if fork == 'fork' then + match = not match + end + if match and words[3] == '(fetch)' then + git_remote = words[0] + break + end + end + if git_remote == '' then + git_remote = 'origin' + end + return git_remote end local function create_pr(pr_title, pr_body) - require_executable("git") - - local push_first = true - - local checked_out_branch = run({ "git", "rev-parse", "--abbrev-ref", "HEAD" }) - if push_first then - local push_remote = run({ "git", "config", "--get", "branch." .. checked_out_branch .. ".pushRemote" }) - if push_remote == nil then - push_remote = run({ "git", "config", "--get", "remote.pushDefault" }) - if push_remote == nil then - push_remote = run({ "git", "config", "--get", "branch." .. checked_out_branch .. ".remote" }) - if push_remote == nil or push_remote == find_git_remote(nil) then - push_remote = find_git_remote("fork") - end - end - end - - p("Pushing to " .. push_remote .. "/" .. checked_out_branch) - run_die({ "git", "push", push_remote, checked_out_branch }, "Git failed to push") - end - - local pr_url = gh_pr(pr_title, pr_body) - p("\nCreated PR: " .. pr_url .. "\n") + require_executable('git') + + local push_first = true + + local checked_out_branch = run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' }) + if push_first then + local push_remote = + run({ 'git', 'config', '--get', 'branch.' .. checked_out_branch .. '.pushRemote' }) + if push_remote == nil then + push_remote = run({ 'git', 'config', '--get', 'remote.pushDefault' }) + if push_remote == nil then + push_remote = + run({ 'git', 'config', '--get', 'branch.' .. checked_out_branch .. '.remote' }) + if push_remote == nil or push_remote == find_git_remote(nil) then + push_remote = find_git_remote('fork') + end + end + end + + p('Pushing to ' .. push_remote .. '/' .. checked_out_branch) + run_die({ 'git', 'push', push_remote, checked_out_branch }, 'Git failed to push') + end + + local pr_url = gh_pr(pr_title, pr_body) + p('\nCreated PR: ' .. pr_url .. '\n') end function M.commit(dependency_name, commit) - local dependency = get_dependency(dependency_name) - verify_cmakelists_committed() - local commit_sha = get_gh_commit_sha(dependency.repo, commit) - if commit_sha ~= commit then - p("Not a commit: " .. commit .. ". Did you mean version?") - die() - end - local archive = get_archive_info(dependency.repo, commit) - if dependency_name == "Luv" then - warn_luv_symbol() - end - update_cmakelists(dependency, archive, short_commit(commit)) + local dependency = get_dependency(dependency_name) + verify_cmakelists_committed() + local commit_sha = get_gh_commit_sha(dependency.repo, commit) + if commit_sha ~= commit then + p('Not a commit: ' .. commit .. '. Did you mean version?') + die() + end + local archive = get_archive_info(dependency.repo, commit) + if dependency_name == 'Luv' then + warn_luv_symbol() + end + update_cmakelists(dependency, archive, short_commit(commit)) end function M.version(dependency_name, version) - local dependency = get_dependency(dependency_name) - verify_cmakelists_committed() - local commit_sha = get_gh_commit_sha(dependency.repo, version) - if commit_sha == version then - p("Not a version: " .. version .. ". Did you mean commit?") - die() - end - local archive = get_archive_info(dependency.repo, version) - if dependency_name == "Luv" then - write_cmakelists_line(dependency.symbol, "VERSION", version) - end - update_cmakelists(dependency, archive, version) + local dependency = get_dependency(dependency_name) + verify_cmakelists_committed() + local commit_sha = get_gh_commit_sha(dependency.repo, version) + if commit_sha == version then + p('Not a version: ' .. version .. '. Did you mean commit?') + die() + end + local archive = get_archive_info(dependency.repo, version) + if dependency_name == 'Luv' then + write_cmakelists_line(dependency.symbol, 'VERSION', version) + end + update_cmakelists(dependency, archive, version) end function M.head(dependency_name) - local dependency = get_dependency(dependency_name) - verify_cmakelists_committed() - local commit_sha = get_gh_commit_sha(dependency.repo, "HEAD") - local archive = get_archive_info(dependency.repo, commit_sha) - if dependency_name == "Luv" then - warn_luv_symbol() - end - update_cmakelists(dependency, archive, "HEAD - " .. short_commit(commit_sha)) + local dependency = get_dependency(dependency_name) + verify_cmakelists_committed() + local commit_sha = get_gh_commit_sha(dependency.repo, 'HEAD') + local archive = get_archive_info(dependency.repo, commit_sha) + if dependency_name == 'Luv' then + warn_luv_symbol() + end + update_cmakelists(dependency, archive, 'HEAD - ' .. short_commit(commit_sha)) end function M.create_branch(dep) - explicit_create_branch(dep) + explicit_create_branch(dep) end function M.submit_pr() - require_executable("git") - - verify_branch("deps") - - local nvim_remote = find_git_remote(nil) - local relevant_commit = run_die({ - "git", - "log", - "--grep=" .. commit_prefix, - "--reverse", - "--format='%s'", - nvim_remote .. "/master..HEAD", - "-1", - }, "Failed to fetch commits") - - local pr_title - local pr_body - - if relevant_commit == "" then - pr_title = commit_prefix .. "bump some dependencies" - pr_body = "bump some dependencies" - else - relevant_commit = relevant_commit:gsub("'", "") - pr_title = relevant_commit - pr_body = relevant_commit:gsub(commit_prefix:gsub("%(", "%%("):gsub("%)", "%%)"), "") - end - pr_body = pr_body .. "\n\n(add explanations if needed)" - p(pr_title .. "\n" .. pr_body .. "\n") - create_pr(pr_title, pr_body) + require_executable('git') + + verify_branch('deps') + + local nvim_remote = find_git_remote(nil) + local relevant_commit = run_die({ + 'git', + 'log', + '--grep=' .. commit_prefix, + '--reverse', + "--format='%s'", + nvim_remote .. '/master..HEAD', + '-1', + }, 'Failed to fetch commits') + + local pr_title + local pr_body + + if relevant_commit == '' then + pr_title = commit_prefix .. 'bump some dependencies' + pr_body = 'bump some dependencies' + else + relevant_commit = relevant_commit:gsub("'", '') + pr_title = relevant_commit + pr_body = relevant_commit:gsub(commit_prefix:gsub('%(', '%%('):gsub('%)', '%%)'), '') + end + pr_body = pr_body .. '\n\n(add explanations if needed)' + p(pr_title .. '\n' .. pr_body .. '\n') + create_pr(pr_title, pr_body) end return M -- cgit From 7fc5d6ea50f5d04ad1d4a71fd0429bfd72f2c66e Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Thu, 5 Jan 2023 20:51:45 +0100 Subject: refactor: eliminate bump-deps.sh using "nvim -l" --- scripts/bump-deps.sh | 108 --------------------------------------------- scripts/bump_deps.lua | 118 ++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 95 insertions(+), 131 deletions(-) delete mode 100755 scripts/bump-deps.sh mode change 100644 => 100755 scripts/bump_deps.lua (limited to 'scripts') diff --git a/scripts/bump-deps.sh b/scripts/bump-deps.sh deleted file mode 100755 index e725608b39..0000000000 --- a/scripts/bump-deps.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash -set -e -set -u -# Use privileged mode, which e.g. skips using CDPATH. -set -p - -# Ensure that the user has a bash that supports -A -if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then - echo >&2 "error: script requires bash 4+ (you have ${BASH_VERSION})." - exit 1 -fi - -readonly NVIM_SOURCE_DIR="${NVIM_SOURCE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" -readonly VIM_SOURCE_DIR_DEFAULT="${NVIM_SOURCE_DIR}/.vim-src" -readonly VIM_SOURCE_DIR="${VIM_SOURCE_DIR:-${VIM_SOURCE_DIR_DEFAULT}}" -BASENAME="$(basename "${0}")" -readonly BASENAME - -usage() { - echo "Bump Nvim dependencies" - echo - echo "Usage: ${BASENAME} [ -h | --pr | --branch= | --dep= ]" - echo " Update a dependency:" - echo " ./scripts/bump-deps.sh --dep Luv --version 1.43.0-0" - echo " Create a PR:" - echo " ./scripts/bump-deps.sh --pr" - echo - echo "Options:" - echo " -h show this message and exit." - echo " --pr submit pr for bumping deps." - echo " --branch= create a branch bump- from current branch." - echo " --dep= bump to a specific release or tag." - echo - echo "Dependency Options:" - echo " --version= bump to a specific release or tag." - echo " --commit= bump to a specific commit." - echo " --HEAD bump to a current head." - echo - echo " is one of:" - echo " \"LuaJIT\", \"libuv\", \"Luv\", \"tree-sitter\"" -} - -# Checks if a program is in the user's PATH, and is executable. -check_executable() { - test -x "$(command -v "${1}")" -} - -require_executable() { - if ! check_executable "${1}"; then - echo >&2 "${BASENAME}: '${1}' not found in PATH or not executable." - exit 1 - fi -} - -require_executable "nvim" - -if [ $# -eq 0 ]; then - usage - exit 1 -fi - -PARSED_ARGS=$(getopt -a -n "$BASENAME" -o h --long pr,branch:,dep:,version:,commit:,HEAD -- "$@") - -DEPENDENCY="" -eval set -- "$PARSED_ARGS" -while :; do - case "$1" in - -h) - usage - exit 0 - ;; - --pr) - nvim -es +"lua require('scripts.bump_deps').submit_pr()" - exit 0 - ;; - --branch) - DEP=$2 - nvim -es +"lua require('scripts.bump_deps').create_branch('$DEP')" - exit 0 - ;; - --dep) - DEPENDENCY=$2 - shift 2 - ;; - --version) - VERSION=$2 - nvim -es +"lua require('scripts.bump_deps').version('$DEPENDENCY', '$VERSION')" - exit 0 - ;; - --commit) - COMMIT=$2 - nvim -es +"lua require('scripts.bump_deps').commit('$DEPENDENCY', '$COMMIT')" - exit 0 - ;; - --HEAD) - nvim -es +"lua require('scripts.bump_deps').head('$DEPENDENCY')" - exit 0 - ;; - *) - break - ;; - esac -done - -usage -exit 1 - -# vim: et sw=2 diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua old mode 100644 new mode 100755 index f16fb4f2a3..1873c3cd0d --- a/scripts/bump_deps.lua +++ b/scripts/bump_deps.lua @@ -1,18 +1,7 @@ +#!/usr/bin/env -S nvim -l + -- Usage: --- # bump to version --- nvim -es +"lua require('scripts.bump_deps').version(dependency, version_tag)" --- --- # bump to commit --- nvim -es +"lua require('scripts.bump_deps').commit(dependency, commit_hash)" --- --- # bump to HEAD --- nvim -es +"lua require('scripts.bump_deps').head(dependency)" --- --- # submit PR --- nvim -es +"lua require('scripts.bump_deps').submit_pr()" --- --- # create branch --- nvim -es +"lua require('scripts.bump_deps').create_branch()" +-- ./scripts/bump_deps.lua -h local M = {} @@ -128,7 +117,10 @@ local function get_archive_info(repo, ref) 'Failed to download archive from GitHub' ) - local archive_sha = run({ 'sha256sum', archive_path }):gmatch('%w+')() + local shacmd = (vim.fn.executable('sha256sum') == 1 + and{ 'sha256sum', archive_path } + or { 'shasum', '-a', '256', archive_path }) + local archive_sha = run(shacmd):gmatch('%w+')() return { url = archive_url, sha = archive_sha } end @@ -171,7 +163,7 @@ end local function verify_branch(new_branch_suffix) require_executable('git') - local checked_out_branch = run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' }) + local checked_out_branch = assert(run({ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' })) if not checked_out_branch:match('^' .. required_branch_prefix) then p( "Current branch '" @@ -244,7 +236,7 @@ end local function find_git_remote(fork) require_executable('git') - local remotes = run({ 'git', 'remote', '-v' }) + local remotes = assert(run({ 'git', 'remote', '-v' })) local git_remote = '' for remote in remotes:gmatch('[^\r\n]+') do local words = {} @@ -295,7 +287,7 @@ local function create_pr(pr_title, pr_body) end function M.commit(dependency_name, commit) - local dependency = get_dependency(dependency_name) + local dependency = assert(get_dependency(dependency_name)) verify_cmakelists_committed() local commit_sha = get_gh_commit_sha(dependency.repo, commit) if commit_sha ~= commit then @@ -310,7 +302,11 @@ function M.commit(dependency_name, commit) end function M.version(dependency_name, version) - local dependency = get_dependency(dependency_name) + vim.validate{ + dependency_name={dependency_name,'s'}, + version={version,'s'}, + } + local dependency = assert(get_dependency(dependency_name)) verify_cmakelists_committed() local commit_sha = get_gh_commit_sha(dependency.repo, version) if commit_sha == version then @@ -325,7 +321,7 @@ function M.version(dependency_name, version) end function M.head(dependency_name) - local dependency = get_dependency(dependency_name) + local dependency = assert(get_dependency(dependency_name)) verify_cmakelists_committed() local commit_sha = get_gh_commit_sha(dependency.repo, 'HEAD') local archive = get_archive_info(dependency.repo, commit_sha) @@ -345,7 +341,7 @@ function M.submit_pr() verify_branch('deps') local nvim_remote = find_git_remote(nil) - local relevant_commit = run_die({ + local relevant_commit = assert(run_die({ 'git', 'log', '--grep=' .. commit_prefix, @@ -353,7 +349,7 @@ function M.submit_pr() "--format='%s'", nvim_remote .. '/master..HEAD', '-1', - }, 'Failed to fetch commits') + }, 'Failed to fetch commits')) local pr_title local pr_body @@ -371,4 +367,80 @@ function M.submit_pr() create_pr(pr_title, pr_body) end -return M +local function usage() + local this_script = _G.arg[0]:match("[^/]*.lua$") + print(([=[ + Bump Nvim dependencies + + Usage: nvim -l %s [options] + Bump to HEAD, tagged version, commit, or branch: + nvim -l %s --dep Luv --head + nvim -l %s --dep Luv --version 1.43.0-0 + nvim -l %s --dep Luv --commit abc123 + nvim -l %s --dep Luv --branch + Create a PR: + nvim -l %s --pr + + Options: + -h show this message and exit. + --pr submit pr for bumping deps. + --branch create a branch bump- from current branch. + --dep bump to a specific release or tag. + + Dependency Options: + --version bump to a specific release or tag. + --commit bump to a specific commit. + --HEAD bump to a current head. + + is one of: + "LuaJIT", "libuv", "Luv", "tree-sitter" + ]=]):format(this_script, this_script, this_script, this_script, this_script, this_script)) +end + +local function parseargs() + local args = {} + for i = 1, #_G.arg do + if _G.arg[i] == '-h' then + args.h = true + elseif _G.arg[i] == '--pr' then + args.pr = true + elseif _G.arg[i] == '--branch' then + args.branch = _G.arg[i+1] + elseif _G.arg[i] == '--dep' then + args.dep = _G.arg[i+1] + elseif _G.arg[i] == '--version' then + args.version = _G.arg[i+1] + elseif _G.arg[i] == '--commit' then + args.commit = _G.arg[i+1] + elseif _G.arg[i] == '--head' then + args.head = true + end + end + return args +end + +local is_main = _G.arg[0]:match('bump_deps.lua') + +if is_main then + local args = parseargs() + if args.h then + usage() + elseif args.pr then + M.submit_pr() + elseif args.head then + M.head(args.dep) + elseif args.branch then + M.create_branch(args.dep) + elseif args.version then + M.version(args.dep, args.version) + elseif args.commit then + M.commit(args.dep, args.commit) + elseif args.pr then + M.submit_pr() + else + print('missing required arg\n') + os.exit(1) + end +else + return M +end -- cgit From ef89f9fd46ab591183b7f59f31f5a2e55f7a526b Mon Sep 17 00:00:00 2001 From: Ching Pei Yang <59727193+horriblename@users.noreply.github.com> Date: Mon, 16 Jan 2023 13:39:19 +0100 Subject: docs: treesitter.add_directive, add_predicate #21206 --- scripts/gen_help_html.lua | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_help_html.lua b/scripts/gen_help_html.lua index fa7c14eaa3..2563f2f410 100644 --- a/scripts/gen_help_html.lua +++ b/scripts/gen_help_html.lua @@ -60,19 +60,18 @@ local exclude_invalid = { ["'previewpopup'"] = "quickref.txt", ["'pvp'"] = "quickref.txt", ["'string'"] = "eval.txt", - Query = "treesitter.txt", - ["eq?"] = "treesitter.txt", - ["lsp-request"] = "lsp.txt", - matchit = "vim_diff.txt", - ["matchit.txt"] = "help.txt", + Query = 'treesitter.txt', + ['eq?'] = 'treesitter.txt', + ['lsp-request'] = 'lsp.txt', + matchit = 'vim_diff.txt', + ['matchit.txt'] = 'help.txt', ["set!"] = "treesitter.txt", - ["v:_null_blob"] = "builtin.txt", - ["v:_null_dict"] = "builtin.txt", - ["v:_null_list"] = "builtin.txt", - ["v:_null_string"] = "builtin.txt", - ["vim.lsp.buf_request()"] = "lsp.txt", - ["vim.lsp.util.get_progress_messages()"] = "lsp.txt", - ["vim.treesitter.start()"] = "treesitter.txt", + ['v:_null_blob'] = 'builtin.txt', + ['v:_null_dict'] = 'builtin.txt', + ['v:_null_list'] = 'builtin.txt', + ['v:_null_string'] = 'builtin.txt', + ['vim.lsp.buf_request()'] = 'lsp.txt', + ['vim.lsp.util.get_progress_messages()'] = 'lsp.txt', } -- False-positive "invalid URLs". -- cgit From cb757f2663e6950e655c6306d713338dfa66b18d Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Mon, 23 Jan 2023 10:26:46 +0100 Subject: build: make generated source files reproducible #21586 Problem: Build is not reproducible, because generated source files (.c/.h/) are not deterministic, mostly because Lua pairs() is unordered by design (for security). https://github.com/LuaJIT/LuaJIT/issues/626#issuecomment-707005671 https://www.lua.org/manual/5.1/manual.html#pdf-next > The order in which the indices are enumerated is not specified [...] > >> The hardening of the VM deliberately randomizes string hashes. This in >> turn randomizes the iteration order of tables with string keys. Solution: - Update the code generation scripts to be deterministic. - That is only a partial solution: the exported function (funcs_metadata.generated.h) and ui event (ui_events_metadata.generated.h) metadata have some mpack'ed tables, which are not serialized deterministically. - As a workaround, introduce `PRG_GEN_LUA` cmake setting, so you can inject a modified build of luajit (with LUAJIT_SECURITY_PRN=0) that preserves table order. - Longer-term we should change the mpack'ed data structure so it no longer uses tables keyed by strings. Closes #20124 Co-Authored-By: dundargoc Co-Authored-By: Arnout Engelen --- scripts/genvimvim.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/genvimvim.lua b/scripts/genvimvim.lua index 868084a583..3e9e7077be 100644 --- a/scripts/genvimvim.lua +++ b/scripts/genvimvim.lua @@ -11,6 +11,8 @@ local funcs_file = arg[3] package.path = nvimsrcdir .. '/?.lua;' .. package.path +_G.vim = loadfile(nvimsrcdir..'/../../runtime/lua/vim/shared.lua')() + local lld = {} local syn_fd = io.open(syntax_file, 'w') lld.line_length = 0 @@ -115,7 +117,7 @@ end local nvimau_start = 'syn keyword nvimAutoEvent contained ' w('\n\n' .. nvimau_start) -for au, _ in pairs(auevents.nvim_specific) do +for au, _ in vim.spairs(auevents.nvim_specific) do if lld.line_length > 850 then w('\n' .. nvimau_start) end @@ -126,7 +128,7 @@ w('\n\nsyn case match') local vimfun_start = 'syn keyword vimFuncName contained ' w('\n\n' .. vimfun_start) local funcs = mpack.unpack(io.open(funcs_file, 'rb'):read("*all")) -for name, _ in pairs(funcs) do +for _, name in ipairs(funcs) do if name then if lld.line_length > 850 then w('\n' .. vimfun_start) -- cgit