From c0d60526541a3cf977ae623471ae4a347b492af1 Mon Sep 17 00:00:00 2001 From: bfredl Date: Tue, 23 Aug 2022 09:33:08 +0200 Subject: perf(api): allow to use an arena for return values --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index bca3dd816f..23ed0e3f08 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -801,7 +801,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error')) + if a[0] not in ('void', 'Error', 'Arena')) if not fmt_vimhelp: c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) -- cgit From ddb762f4013ac2532ad45704466058d867e3a6ed Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Wed, 14 Sep 2022 11:08:31 +0200 Subject: docs(treesitter): clean up and update treesitter.txt (#20142) * add type annotations to code * clean up and expand static documentation * consistent use of tags for static and generated docs --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 23ed0e3f08..766c420c7d 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -260,7 +260,7 @@ CONFIG = { 'helptag_fmt': lambda name: ( '*lua-treesitter-core*' if name.lower() == 'treesitter' - else f'*treesitter-{name.lower()}*'), + else f'*lua-treesitter-{name.lower()}*'), 'fn_helptag_fmt': lambda fstem, name: ( f'*{name}()*' if name != 'new' -- cgit From 1b60b5ec94001f18b70dbebf6c232c33209f11b5 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 25 Sep 2022 13:45:41 +0200 Subject: fix(gen_vimdoc.py): handle missing luajit --- scripts/gen_vimdoc.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 766c420c7d..306857ca6c 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -295,14 +295,16 @@ annotation_map = { # or if `cond()` is callable and returns True. def debug_this(o, cond=True): name = '' + if cond is False: + return if not isinstance(o, str): try: name = o.nodeName o = o.toprettyxml(indent=' ', newl='\n') except Exception: pass - if ((callable(cond) and cond()) - or (not callable(cond) and cond) + if (cond is True + or (callable(cond) and cond()) or (not callable(cond) and cond in o)): raise RuntimeError('xxx: {}\n{}'.format(name, o)) @@ -887,7 +889,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): def fmt_doxygen_xml_as_vimhelp(filename, target): """Entrypoint for generating Vim :help from from Doxygen XML. - Returns 3 items: + Returns 2 items: 1. Vim help text for functions found in `filename`. 2. Vim help text for deprecated functions. """ @@ -1094,7 +1096,11 @@ def main(config, args): fn_map_full.update(fn_map) if len(sections) == 0: - fail(f'no sections for target: {target}') + if target == 'lua': + fail(f'no sections for target: {target} (this usually means' + + ' "luajit" was not found by scripts/lua2dox_filter)') + else: + fail(f'no sections for target: {target}') if len(sections) > len(CONFIG[target]['section_order']): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( -- cgit From 18afacee1d98b9987391b8bdef08282fb156fa88 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Wed, 5 Oct 2022 08:15:55 -0400 Subject: feat(docs): format parameters as a list #20485 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: The {foo} parameters listed in `:help api` and similar generated docs, are intended to be a "list" but they aren't prefixed with a list symbol. This prevents parsers from understanding the list, which forces generators like `gen_help_html.lua` to use hard-wrapped/preformatted layout instead of a soft-wrapped "flow" layout. Solution: Modify gen_vimdoc.py to prefix {foo} parameters with a "•" symbol. --- scripts/gen_vimdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 306857ca6c..2612260226 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -673,7 +673,7 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='', max_name_len = max_name(m.keys()) + 4 out = '' for name, desc in m.items(): - name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) out += '{}{}\n'.format(name, desc) return out.rstrip() -- cgit From 09dffb9db7d16496e55e86f78ab60241533d86f6 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Sun, 9 Oct 2022 08:21:52 -0400 Subject: docs: various #12823 - increase python line-length limit from 88 => 100. - gen_help_html: fix bug in "tag" case (tbl_count => tbl_contains) ref #15632 fix #18215 fix #18479 fix #20527 fix #20532 Co-authored-by: Ben Weedon --- scripts/gen_vimdoc.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 2612260226..3ee9d8b5dd 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -2,9 +2,7 @@ """Generates Nvim :help docs from C/Lua docstrings, using Doxygen. Also generates *.mpack files. To inspect the *.mpack structure: - - :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack'))) - + :new | put=v:lua.vim.inspect(v:lua.vim.mpack.unpack(readfile('runtime/doc/api.mpack','B'))) Flow: main @@ -287,7 +285,7 @@ annotation_map = { 'FUNC_API_FAST': '|api-fast|', 'FUNC_API_CHECK_TEXTLOCK': 'not allowed when |textlock| is active', 'FUNC_API_REMOTE_ONLY': '|RPC| only', - 'FUNC_API_LUA_ONLY': '|vim.api| only', + 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only', } -- cgit From e5cb3104d07228de4f2614c425355e8f2f99507d Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Fri, 14 Oct 2022 11:01:13 -0400 Subject: docs: fix/remove invalid URLs #20647 --- scripts/gen_vimdoc.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 3ee9d8b5dd..a044c8c39d 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -12,15 +12,10 @@ Flow: update_params_map / render_node -This would be easier using lxml and XSLT, but: +TODO: eliminate this script and use Lua+treesitter (requires parsers for C and +Lua markdown-style docstrings). - 1. This should avoid needing Python dependencies, especially ones that are - C modules that have library dependencies (lxml requires libxml and - libxslt). - 2. I wouldn't know how to deal with nested indentation in tags using - XSLT. - -Each function :help block is formatted as follows: +The generated :help text for each function is formatted as follows: - Max width of 78 columns (`text_width`). - Indent with spaces (not tabs). -- cgit From f8c671827710c6e9cca3bfd60c32098b2be8239a Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 14 Nov 2022 18:04:36 +0000 Subject: feat(lua-api): avoid unnecessary allocations (#19877) Lua makes (or reuses) an internal copy of strings, so we can safely push buf pointers onto the stack. --- scripts/gen_vimdoc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index a044c8c39d..e77d3ea286 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -796,7 +796,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error', 'Arena')) + if a[0] not in ('void', 'Error', 'Arena', + 'lua_State')) if not fmt_vimhelp: c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) -- cgit From f1922e78a1df1b1d32779769432fb5586edf5fbb Mon Sep 17 00:00:00 2001 From: Gregory Anders Date: Sat, 5 Nov 2022 13:37:05 -0600 Subject: feat: add vim.secure.read() This function accepts a path to a file and prompts the user if the file is trusted. If the user confirms that the file is trusted, the contents of the file are returned. The user's decision is stored in a trust database at $XDG_STATE_HOME/nvim/trust. When this function is invoked with a path that is already marked as trusted in the trust database, the user is not prompted for a response. --- scripts/gen_vimdoc.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index e77d3ea286..a720f055ed 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -131,6 +131,7 @@ CONFIG = { 'filetype.lua', 'keymap.lua', 'fs.lua', + 'secure.lua', ], 'files': [ 'runtime/lua/vim/_editor.lua', @@ -140,6 +141,7 @@ CONFIG = { 'runtime/lua/vim/filetype.lua', 'runtime/lua/vim/keymap.lua', 'runtime/lua/vim/fs.lua', + 'runtime/lua/vim/secure.lua', ], 'file_patterns': '*.lua', 'fn_name_prefix': '', @@ -166,6 +168,7 @@ CONFIG = { 'filetype': 'vim.filetype', 'keymap': 'vim.keymap', 'fs': 'vim.fs', + 'secure': 'vim.secure', }, 'append_only': [ 'shared.lua', -- cgit From 0b05bd87c04f9cde5c84a062453619349e370795 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Wed, 23 Nov 2022 12:31:49 +0100 Subject: docs(gen): support language annotation in docstrings --- scripts/gen_vimdoc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index a720f055ed..801c8ea790 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -496,7 +496,12 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, if n.nodeName == 'preformatted': o = get_text(n, preformatted=True) ensure_nl = '' if o[-1] == '\n' else '\n' - text += '>{}{}\n<'.format(ensure_nl, o) + if o[0:4] == 'lua\n': + text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1]) + elif o[0:4] == 'vim\n': + text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1]) + else: + text += '>{}{}\n<'.format(ensure_nl, o) elif is_inline(n): text = doc_wrap(get_text(n), indent=indent, width=width) -- cgit From f8aa2a0deaf473af0e6b4640356eaf5477c6ee90 Mon Sep 17 00:00:00 2001 From: Gregory Anders <8965202+gpanders@users.noreply.github.com> Date: Wed, 7 Dec 2022 01:27:41 -0700 Subject: docs(treesitter): use full function names in tags (#21321) --- scripts/gen_vimdoc.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 801c8ea790..78f3c458eb 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -258,17 +258,11 @@ CONFIG = { if name.lower() == 'treesitter' else f'*lua-treesitter-{name.lower()}*'), 'fn_helptag_fmt': lambda fstem, name: ( - f'*{name}()*' - if name != 'new' - else f'*{fstem}.{name}()*'), - # 'fn_helptag_fmt': lambda fstem, name: ( - # f'*vim.treesitter.{name}()*' - # if fstem == 'treesitter' - # else ( - # '*vim.lsp.client*' - # # HACK. TODO(justinmk): class/structure support in lua2dox - # if 'lsp.client' == f'{fstem}.{name}' - # else f'*vim.lsp.{fstem}.{name}()*')), + f'*vim.{fstem}.{name}()*' + if fstem == 'treesitter' + else f'*{name}()*' + if name[0].isupper() + else f'*vim.treesitter.{fstem}.{name}()*'), 'module_override': {}, 'append_only': [], } -- cgit From 9f035559defd9d575f37fd825954610065d9cf96 Mon Sep 17 00:00:00 2001 From: John Drouhard Date: Wed, 23 Nov 2022 10:06:36 -0600 Subject: feat(lsp): initial support for semantic token highlighting * credit to @smolck and @theHamsta for their contributions in laying the groundwork for this feature and for their work on some of the helper utility functions and tests --- scripts/gen_vimdoc.py | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 78f3c458eb..b18179b498 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -183,6 +183,7 @@ CONFIG = { 'diagnostic.lua', 'codelens.lua', 'tagfunc.lua', + 'semantic_tokens.lua', 'handlers.lua', 'util.lua', 'log.lua', -- cgit From ef91146efcece1b6d97152251e7137d301146189 Mon Sep 17 00:00:00 2001 From: Folke Lemaitre Date: Wed, 14 Dec 2022 10:46:54 +0100 Subject: feat: `vim.inspect_pos`, `vim.show_pos`, `:Inspect` --- scripts/gen_vimdoc.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index b18179b498..502c9161b6 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -125,6 +125,7 @@ CONFIG = { 'filename': 'lua.txt', 'section_order': [ '_editor.lua', + '_inspector.lua', 'shared.lua', 'uri.lua', 'ui.lua', @@ -142,11 +143,13 @@ CONFIG = { 'runtime/lua/vim/keymap.lua', 'runtime/lua/vim/fs.lua', 'runtime/lua/vim/secure.lua', + 'runtime/lua/vim/_inspector.lua', ], 'file_patterns': '*.lua', 'fn_name_prefix': '', 'section_name': { 'lsp.lua': 'core', + '_inspector.lua': 'inspector', }, 'section_fmt': lambda name: ( 'Lua module: vim' @@ -163,6 +166,7 @@ CONFIG = { 'module_override': { # `shared` functions are exposed on the `vim` module. 'shared': 'vim', + '_inspector': 'vim', 'uri': 'vim', 'ui': 'vim.ui', 'filetype': 'vim.filetype', @@ -346,6 +350,17 @@ def self_or_child(n): return n.childNodes[0] +def align_tags(line): + tag_regex = r"\s(\*.+?\*)(?:\s|$)" + tags = re.findall(tag_regex, line) + + if len(tags) > 0: + line = re.sub(tag_regex, "", line) + tags = " " + " ".join(tags) + line = line + (" " * (78 - len(line) - len(tags))) + tags + return line + + def clean_lines(text): """Removes superfluous lines. @@ -950,7 +965,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): start = end - func_doc = "\n".join(split_lines) + func_doc = "\n".join(map(align_tags, split_lines)) if (name.startswith(CONFIG[target]['fn_name_prefix']) and name != "nvim_error_event"): -- cgit From 7c94bcd2d77e2e54b8836ab8325460a367b79eae Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Mon, 20 Sep 2021 19:00:50 -0700 Subject: feat(lua)!: execute Lua with "nvim -l" Problem: Nvim has Lua but the "nvim" CLI can't easily be used to execute Lua scripts, especially scripts that take arguments or produce output. Solution: - support "nvim -l [args...]" for running scripts. closes #15749 - exit without +q - remove lua2dox_filter - remove Doxyfile. This wasn't used anyway, because the doxygen config is inlined in gen_vimdoc.py (`Doxyfile` variable). - use "nvim -l" in docs-gen CI job Examples: $ nvim -l scripts/lua2dox.lua --help Lua2DoX (0.2 20130128) ... $ echo "print(vim.inspect(_G.arg))" | nvim -l - --arg1 --arg2 $ echo 'print(vim.inspect(vim.api.nvim_buf_get_text(1,0,0,-1,-1,{})))' | nvim +"put ='text'" -l - TODO? -e executes Lua code -l loads a module -i enters REPL _after running the other arguments_. --- scripts/gen_vimdoc.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 502c9161b6..8e1d6ef80a 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -55,11 +55,19 @@ if sys.version_info < MIN_PYTHON_VERSION: doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"], universal_newlines=True).split()[0].split('.'))) +# Until 0.9 is released, need this hacky way to check that "nvim -l foo.lua" works. +nvim_version = list(line for line in subprocess.check_output(['nvim', '-h'], universal_newlines=True).split('\n') + if '-l ' in line) + if doxygen_version < MIN_DOXYGEN_VERSION: print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION)) print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version)) sys.exit(1) +if len(nvim_version) == 0: + print("\nRequires 'nvim -l' feature, see https://github.com/neovim/neovim/pull/18706") + sys.exit(1) + # DEBUG = ('DEBUG' in os.environ) INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) @@ -79,7 +87,7 @@ base_dir = os.path.dirname(os.path.dirname(script_path)) out_dir = os.path.join(base_dir, 'tmp-{target}-doc') filter_cmd = '%s %s' % (sys.executable, script_path) msgs = [] # Messages to show on exit. -lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter') +lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') CONFIG = { 'api': { @@ -993,7 +1001,7 @@ def delete_lines_below(filename, tokenstr): fp.writelines(lines[0:i]) -def main(config, args): +def main(doxygen_config, args): """Generates: 1. Vim :help docs @@ -1021,7 +1029,7 @@ def main(config, args): # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL)) p.communicate( - config.format( + doxygen_config.format( input=' '.join( [f'"{file}"' for file in CONFIG[target]['files']]), output=output_dir, @@ -1108,11 +1116,7 @@ def main(config, args): fn_map_full.update(fn_map) if len(sections) == 0: - if target == 'lua': - fail(f'no sections for target: {target} (this usually means' - + ' "luajit" was not found by scripts/lua2dox_filter)') - else: - fail(f'no sections for target: {target}') + fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') if len(sections) > len(CONFIG[target]['section_order']): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( @@ -1159,7 +1163,7 @@ def main(config, args): def filter_source(filename): name, extension = os.path.splitext(filename) if extension == '.lua': - p = subprocess.run([lua2dox_filter, filename], stdout=subprocess.PIPE) + p = subprocess.run(['nvim', '-l', lua2dox, filename], stdout=subprocess.PIPE) op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8')) print(op) else: -- cgit