diff options
Diffstat (limited to 'scripts/gen_vimdoc.py')
| -rwxr-xr-x | scripts/gen_vimdoc.py | 409 |
1 files changed, 307 insertions, 102 deletions
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 8e1d6ef80a..8ed88cb8f5 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -2,7 +2,7 @@ """Generates Nvim :help docs from C/Lua docstrings, using Doxygen. Also generates *.mpack files. To inspect the *.mpack structure: - :new | put=v:lua.vim.inspect(v:lua.vim.mpack.unpack(readfile('runtime/doc/api.mpack','B'))) + :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B'))) Flow: main @@ -42,8 +42,12 @@ import subprocess import collections import msgpack import logging +from typing import Tuple +from pathlib import Path from xml.dom import minidom +Element = minidom.Element +Document = minidom.Document MIN_PYTHON_VERSION = (3, 6) MIN_DOXYGEN_VERSION = (1, 9, 0) @@ -55,18 +59,28 @@ if sys.version_info < MIN_PYTHON_VERSION: doxygen_version = tuple((int(i) for i in subprocess.check_output(["doxygen", "-v"], universal_newlines=True).split()[0].split('.'))) -# Until 0.9 is released, need this hacky way to check that "nvim -l foo.lua" works. -nvim_version = list(line for line in subprocess.check_output(['nvim', '-h'], universal_newlines=True).split('\n') - if '-l ' in line) - if doxygen_version < MIN_DOXYGEN_VERSION: print("\nRequires doxygen {}.{}.{}+".format(*MIN_DOXYGEN_VERSION)) print("Your doxygen version is {}.{}.{}\n".format(*doxygen_version)) sys.exit(1) -if len(nvim_version) == 0: - print("\nRequires 'nvim -l' feature, see https://github.com/neovim/neovim/pull/18706") - sys.exit(1) + +# Need a `nvim` that supports `-l`, try the local build +nvim_path = Path(__file__).parent / "../build/bin/nvim" +if nvim_path.exists(): + nvim = str(nvim_path) +else: + # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works. + nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True) + nvim_version = [line for line in nvim_out.split('\n') + if '-l ' in line] + if len(nvim_version) == 0: + print(( + "\nYou need to have a local Neovim build or a `nvim` version 0.9 for `-l` " + "support to build the documentation.")) + sys.exit(1) + nvim = 'nvim' + # DEBUG = ('DEBUG' in os.environ) INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) @@ -122,7 +136,7 @@ CONFIG = { # Section helptag. 'helptag_fmt': lambda name: f'*api-{name.lower()}*', # Per-function helptag. - 'fn_helptag_fmt': lambda fstem, name: f'*{name}()*', + 'fn_helptag_fmt': lambda fstem, name, istbl: f'*{name}()*', # Module name overrides (for Lua). 'module_override': {}, # Append the docs for these modules, do not start a new section. @@ -132,55 +146,111 @@ CONFIG = { 'mode': 'lua', 'filename': 'lua.txt', 'section_order': [ + 'highlight.lua', + 'regex.lua', + 'diff.lua', + 'mpack.lua', + 'json.lua', + 'base64.lua', + 'spell.lua', + 'builtin.lua', + '_options.lua', '_editor.lua', '_inspector.lua', 'shared.lua', + 'loader.lua', 'uri.lua', 'ui.lua', 'filetype.lua', 'keymap.lua', 'fs.lua', 'secure.lua', + 'version.lua', + 'iter.lua', + 'snippet.lua', + 'text.lua', ], 'files': [ + 'runtime/lua/vim/iter.lua', 'runtime/lua/vim/_editor.lua', + 'runtime/lua/vim/_options.lua', 'runtime/lua/vim/shared.lua', + 'runtime/lua/vim/loader.lua', 'runtime/lua/vim/uri.lua', 'runtime/lua/vim/ui.lua', 'runtime/lua/vim/filetype.lua', 'runtime/lua/vim/keymap.lua', 'runtime/lua/vim/fs.lua', + 'runtime/lua/vim/highlight.lua', 'runtime/lua/vim/secure.lua', + 'runtime/lua/vim/version.lua', 'runtime/lua/vim/_inspector.lua', + 'runtime/lua/vim/snippet.lua', + 'runtime/lua/vim/text.lua', + 'runtime/lua/vim/_meta/builtin.lua', + 'runtime/lua/vim/_meta/diff.lua', + 'runtime/lua/vim/_meta/mpack.lua', + 'runtime/lua/vim/_meta/json.lua', + 'runtime/lua/vim/_meta/base64.lua', + 'runtime/lua/vim/_meta/regex.lua', + 'runtime/lua/vim/_meta/spell.lua', ], 'file_patterns': '*.lua', 'fn_name_prefix': '', + 'fn_name_fmt': lambda fstem, name: ( + name if fstem in [ 'vim.iter' ] else + f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else + f'vim.{name}' if fstem == '_options' and not name[0].isupper() else + f'{fstem}.{name}' if fstem.startswith('vim') else + name + ), 'section_name': { 'lsp.lua': 'core', '_inspector.lua': 'inspector', }, 'section_fmt': lambda name: ( - 'Lua module: vim' - if name.lower() == '_editor' - else f'Lua module: {name.lower()}'), + 'Lua module: vim' if name.lower() == '_editor' else + 'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else + f'VIM.{name.upper()}' if name.lower() in [ 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', 'regex' ] else + 'VIM' if name.lower() == 'builtin' else + f'Lua module: vim.{name.lower()}'), 'helptag_fmt': lambda name: ( - '*lua-vim*' - if name.lower() == '_editor' - else f'*lua-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name: ( - f'*vim.{name}()*' - if fstem.lower() == '_editor' - else f'*{fstem}.{name}()*'), + '*lua-vim*' if name.lower() == '_editor' else + '*lua-vimscript*' if name.lower() == '_options' else + f'*vim.{name.lower()}*'), + 'fn_helptag_fmt': lambda fstem, name, istbl: ( + f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else + # Exclude fstem for methods + f'*{name}()*' if ':' in name else + f'*vim.{name}()*' if fstem.lower() == '_editor' else + f'*vim.{name}*' if fstem.lower() == '_options' and istbl else + # Prevents vim.regex.regex + f'*{fstem}()*' if fstem.endswith('.' + name) else + f'*{fstem}.{name}{"" if istbl else "()"}*' + ), 'module_override': { # `shared` functions are exposed on the `vim` module. 'shared': 'vim', '_inspector': 'vim', 'uri': 'vim', 'ui': 'vim.ui', + 'loader': 'vim.loader', 'filetype': 'vim.filetype', 'keymap': 'vim.keymap', 'fs': 'vim.fs', + 'highlight': 'vim.highlight', 'secure': 'vim.secure', + 'version': 'vim.version', + 'iter': 'vim.iter', + 'diff': 'vim', + 'builtin': 'vim', + 'mpack': 'vim.mpack', + 'json': 'vim.json', + 'base64': 'vim.base64', + 'regex': 'vim.regex', + 'spell': 'vim.spell', + 'snippet': 'vim.snippet', + 'text': 'vim.text', }, 'append_only': [ 'shared.lua', @@ -194,13 +264,13 @@ CONFIG = { 'buf.lua', 'diagnostic.lua', 'codelens.lua', + 'inlay_hint.lua', 'tagfunc.lua', 'semantic_tokens.lua', 'handlers.lua', 'util.lua', 'log.lua', 'rpc.lua', - 'sync.lua', 'protocol.lua', ], 'files': [ @@ -218,14 +288,11 @@ CONFIG = { '*lsp-core*' if name.lower() == 'lsp' else f'*lsp-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name: ( - f'*vim.lsp.{name}()*' - if fstem == 'lsp' and name != 'client' - else ( - '*vim.lsp.client*' - # HACK. TODO(justinmk): class/structure support in lua2dox - if 'lsp.client' == f'{fstem}.{name}' - else f'*vim.lsp.{fstem}.{name}()*')), + 'fn_helptag_fmt': lambda fstem, name, istbl: ( + f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else + # HACK. TODO(justinmk): class/structure support in lua2dox + '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else + f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'), 'module_override': {}, 'append_only': [], }, @@ -238,10 +305,11 @@ CONFIG = { 'files': ['runtime/lua/vim/diagnostic.lua'], 'file_patterns': '*.lua', 'fn_name_prefix': '', + 'include_tables': False, 'section_name': {'diagnostic.lua': 'diagnostic'}, 'section_fmt': lambda _: 'Lua module: vim.diagnostic', 'helptag_fmt': lambda _: '*diagnostic-api*', - 'fn_helptag_fmt': lambda fstem, name: f'*vim.{fstem}.{name}()*', + 'fn_helptag_fmt': lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*', 'module_override': {}, 'append_only': [], }, @@ -254,6 +322,7 @@ CONFIG = { 'query.lua', 'highlighter.lua', 'languagetree.lua', + 'dev.lua', ], 'files': [ 'runtime/lua/vim/treesitter.lua', @@ -270,7 +339,7 @@ CONFIG = { '*lua-treesitter-core*' if name.lower() == 'treesitter' else f'*lua-treesitter-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name: ( + 'fn_helptag_fmt': lambda fstem, name, istbl: ( f'*vim.{fstem}.{name}()*' if fstem == 'treesitter' else f'*{name}()*' @@ -288,12 +357,37 @@ param_exclude = ( # Annotations are displayed as line items after API function descriptions. annotation_map = { 'FUNC_API_FAST': '|api-fast|', - 'FUNC_API_CHECK_TEXTLOCK': 'not allowed when |textlock| is active', + 'FUNC_API_TEXTLOCK': 'not allowed when |textlock| is active or in the |cmdwin|', + 'FUNC_API_TEXTLOCK_ALLOW_CMDWIN': 'not allowed when |textlock| is active', 'FUNC_API_REMOTE_ONLY': '|RPC| only', 'FUNC_API_LUA_ONLY': 'Lua |vim.api| only', } +def nvim_api_info() -> Tuple[int, bool]: + """Returns NVIM_API_LEVEL, NVIM_API_PRERELEASE from CMakeLists.txt""" + if not hasattr(nvim_api_info, 'LEVEL'): + script_dir = os.path.dirname(os.path.abspath(__file__)) + cmake_file_path = os.path.join(script_dir, '..', 'CMakeLists.txt') + with open(cmake_file_path, 'r') as cmake_file: + cmake_content = cmake_file.read() + + api_level_match = re.search(r'set\(NVIM_API_LEVEL (\d+)\)', cmake_content) + api_prerelease_match = re.search( + r'set\(NVIM_API_PRERELEASE (\w+)\)', cmake_content + ) + + if not api_level_match or not api_prerelease_match: + raise RuntimeError( + 'Could not find NVIM_API_LEVEL or NVIM_API_PRERELEASE in CMakeLists.txt' + ) + + nvim_api_info.LEVEL = int(api_level_match.group(1)) + nvim_api_info.PRERELEASE = api_prerelease_match.group(1).lower() == 'true' + + return nvim_api_info.LEVEL, nvim_api_info.PRERELEASE + + # Raises an error with details about `o`, if `cond` is in object `o`, # or if `cond()` is callable and returns True. def debug_this(o, cond=True): @@ -381,7 +475,7 @@ def is_blank(text): return '' == clean_lines(text) -def get_text(n, preformatted=False): +def get_text(n): """Recursively concatenates all text in a node tree.""" text = '' if n.nodeType == n.TEXT_NODE: @@ -390,11 +484,13 @@ def get_text(n, preformatted=False): for node in n.childNodes: text += get_text(node) return '`{}`'.format(text) + if n.nodeName == 'sp': # space, used in "programlisting" nodes + return ' ' for node in n.childNodes: if node.nodeType == node.TEXT_NODE: text += node.data elif node.nodeType == node.ELEMENT_NODE: - text += get_text(node, preformatted) + text += get_text(node) return text @@ -512,17 +608,26 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, # text += (int(not space_preceding) * ' ') if n.nodeName == 'preformatted': - o = get_text(n, preformatted=True) + o = get_text(n) ensure_nl = '' if o[-1] == '\n' else '\n' if o[0:4] == 'lua\n': text += '>lua{}{}\n<'.format(ensure_nl, o[3:-1]) elif o[0:4] == 'vim\n': text += '>vim{}{}\n<'.format(ensure_nl, o[3:-1]) + elif o[0:5] == 'help\n': + text += o[4:-1] else: text += '>{}{}\n<'.format(ensure_nl, o) - + elif n.nodeName == 'programlisting': # codeblock (```) + o = get_text(n) + text += '>' + if 'filename' in n.attributes: + filename = n.attributes['filename'].value + text += filename.lstrip('.') + + text += '\n{}\n<'.format(textwrap.indent(o, ' ' * 4)) elif is_inline(n): - text = doc_wrap(get_text(n), indent=indent, width=width) + text = doc_wrap(get_text(n), prefix=prefix, indent=indent, width=width) elif n.nodeName == 'verbatim': # TODO: currently we don't use this. The "[verbatim]" hint is there as # a reminder that we must decide how to format this if we do use it. @@ -535,19 +640,19 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, indent=indent + (' ' * len(prefix)), width=width ) - if is_blank(result): continue - text += indent + prefix + result elif n.nodeName in ('para', 'heading'): + did_prefix = False for c in n.childNodes: if (is_inline(c) and '' != get_text(c).strip() and text and ' ' != text[-1]): text += ' ' - text += render_node(c, text, indent=indent, width=width) + text += render_node(c, text, prefix=(prefix if not did_prefix else ''), indent=indent, width=width) + did_prefix = True elif n.nodeName == 'itemizedlist': for c in n.childNodes: text += '{}\n'.format(render_node(c, text, prefix='• ', @@ -562,17 +667,26 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, indent=indent, width=width)) i = i + 1 elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'): - text += '\nNote:\n ' + text += ind(' ') for c in n.childNodes: - text += render_node(c, text, indent=' ', width=width) - text += '\n' + if is_blank(render_node(c, text, prefix='• ', indent=' ', width=width)): + continue + text += render_node(c, text, prefix='• ', indent=' ', width=width) + # text += '\n' elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'): text += 'Warning:\n ' for c in n.childNodes: text += render_node(c, text, indent=' ', width=width) text += '\n' - elif (n.nodeName == 'simplesect' - and n.getAttribute('kind') in ('return', 'see')): + elif n.nodeName == 'simplesect' and 'see' == n.getAttribute('kind'): + text += ind(' ') + # Example: + # <simplesect kind="see"> + # <para>|autocommand|</para> + # </simplesect> + for c in n.childNodes: + text += render_node(c, text, prefix='• ', indent=' ', width=width) + elif n.nodeName == 'simplesect' and 'return' == n.getAttribute('kind'): text += ind(' ') for c in n.childNodes: text += render_node(c, text, indent=' ', width=width) @@ -590,6 +704,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F Keys: 'text': Text from this <para> element + 'note': List of @note strings 'params': <parameterlist> map 'return': List of @return strings 'seealso': List of @see strings @@ -597,14 +712,17 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F """ chunks = { 'text': '', + 'note': [], 'params': collections.OrderedDict(), 'return': [], 'seealso': [], + 'prerelease': False, 'xrefs': [] } # Ordered dict of ordered lists. groups = collections.OrderedDict([ + ('note', []), ('params', []), ('return', []), ('seealso', []), @@ -615,7 +733,6 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F # nodes to appear together. text = '' kind = '' - last = '' if is_inline(parent): # Flatten inline text from a tree of non-block nodes. text = doc_wrap(render_node(parent, "", fmt_vimhelp=fmt_vimhelp), @@ -628,15 +745,24 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F elif child.nodeName == 'xrefsect': groups['xrefs'].append(child) elif child.nodeName == 'simplesect': - last = kind kind = child.getAttribute('kind') - if kind == 'return' or (kind == 'note' and last == 'return'): + if kind == 'note': + groups['note'].append(child) + elif kind == 'return': groups['return'].append(child) elif kind == 'see': groups['seealso'].append(child) - elif kind in ('note', 'warning'): + elif kind == 'warning': text += render_node(child, text, indent=indent, width=width, fmt_vimhelp=fmt_vimhelp) + elif kind == 'since': + since_match = re.match(r'^(\d+)', get_text(child)) + since = int(since_match.group(1)) if since_match else 0 + NVIM_API_LEVEL, NVIM_API_PRERELEASE = nvim_api_info() + if since > NVIM_API_LEVEL or ( + since == NVIM_API_LEVEL and NVIM_API_PRERELEASE + ): + chunks['prerelease'] = True else: raise RuntimeError('unhandled simplesect: {}\n{}'.format( child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) @@ -659,10 +785,17 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F if len(groups['params']) > 0: for child in groups['params']: update_params_map(child, ret_map=chunks['params'], width=width) + for child in groups['note']: + chunks['note'].append(render_node( + child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp).rstrip()) for child in groups['return']: chunks['return'].append(render_node( child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp)) for child in groups['seealso']: + # Example: + # <simplesect kind="see"> + # <para>|autocommand|</para> + # </simplesect> chunks['seealso'].append(render_node( child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp)) @@ -678,8 +811,28 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F return chunks, xrefs +def is_program_listing(para): + """ + Return True if `para` contains a "programlisting" (i.e. a Markdown code + block ```). + + Sometimes a <para> element will have only a single "programlisting" child + node, but othertimes it will have extra whitespace around the + "programlisting" node. -def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='', + @param para XML <para> node + @return True if <para> is a programlisting + """ + + # Remove any child text nodes that are only whitespace + children = [ + n for n in para.childNodes + if n.nodeType != n.TEXT_NODE or n.data.strip() != '' + ] + + return len(children) == 1 and children[0].nodeName == 'programlisting' + +def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='', fmt_vimhelp=False): """Renders (nested) Doxygen <para> nodes as Vim :help text. @@ -692,6 +845,8 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='', max_name_len = max_name(m.keys()) + 4 out = '' for name, desc in m.items(): + if name == 'self': + continue name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) out += '{}{}\n'.format(name, desc) return out.rstrip() @@ -705,13 +860,28 @@ def fmt_node_as_vimhelp(parent, width=text_width - indentation, indent='', for child in parent.childNodes: para, _ = para_as_map(child, indent, width, fmt_vimhelp) + # 'programlisting' blocks are Markdown code blocks. Do not include + # these as a separate paragraph, but append to the last non-empty line + # in the text + if is_program_listing(child): + while rendered_blocks and rendered_blocks[-1] == '': + rendered_blocks.pop() + rendered_blocks[-1] += ' ' + para['text'] + continue + # Generate text from the gathered items. chunks = [para['text']] + notes = [" This API is pre-release (unstable)."] if para['prerelease'] else [] + notes += para['note'] + if len(notes) > 0: + chunks.append('\nNote: ~') + for s in notes: + chunks.append(s) if len(para['params']) > 0 and has_nonexcluded_params(para['params']): chunks.append('\nParameters: ~') chunks.append(fmt_param_doc(para['params'])) if len(para['return']) > 0: - chunks.append('\nReturn: ~') + chunks.append('\nReturn (multiple): ~' if len(para['return']) > 1 else '\nReturn: ~') for s in para['return']: chunks.append(s) if len(para['seealso']) > 0: @@ -757,6 +927,13 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): if return_type == '': continue + if 'local_function' in return_type: # Special from lua2dox.lua. + continue + + istbl = return_type.startswith('table') # Special from lua2dox.lua. + if istbl and not CONFIG[target].get('include_tables', True): + continue + if return_type.startswith(('ArrayOf', 'DictionaryOf')): parts = return_type.strip('_').split('_') return_type = '{}({})'.format(parts[0], ', '.join(parts[1:])) @@ -805,6 +982,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): and any(x[1] == 'self' for x in params): split_return = return_type.split(' ') name = f'{split_return[1]}:{name}' + params = [x for x in params if x[1] != 'self'] c_args = [] for param_type, param_name in params: @@ -818,12 +996,20 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): if '.' in compoundname: fstem = compoundname.split('.')[0] fstem = CONFIG[target]['module_override'].get(fstem, fstem) - vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name) + vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name, istbl) + + if 'fn_name_fmt' in CONFIG[target]: + name = CONFIG[target]['fn_name_fmt'](fstem, name) + + if istbl: + aopen, aclose = '', '' + else: + aopen, aclose = '(', ')' - prefix = '%s(' % name - suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params - if a[0] not in ('void', 'Error', 'Arena', - 'lua_State')) + prefix = name + aopen + suffix = ', '.join('{%s}' % a[1] for a in params + if a[0] not in ('void', 'Error', 'Arena', + 'lua_State')) + aclose if not fmt_vimhelp: c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) @@ -1001,6 +1187,42 @@ def delete_lines_below(filename, tokenstr): fp.writelines(lines[0:i]) +def extract_defgroups(base: str, dom: Document): + '''Generate module-level (section) docs (@defgroup).''' + section_docs = {} + + for compound in dom.getElementsByTagName('compound'): + if compound.getAttribute('kind') != 'group': + continue + + # Doxygen "@defgroup" directive. + groupname = get_text(find_first(compound, 'name')) + groupxml = os.path.join(base, '%s.xml' % + compound.getAttribute('refid')) + + group_parsed = minidom.parse(groupxml) + doc_list = [] + brief_desc = find_first(group_parsed, 'briefdescription') + if brief_desc: + for child in brief_desc.childNodes: + doc_list.append(fmt_node_as_vimhelp(child)) + + desc = find_first(group_parsed, 'detaileddescription') + if desc: + doc = fmt_node_as_vimhelp(desc) + + if doc: + doc_list.append(doc) + + # Can't use '.' in @defgroup, so convert to '--' + # "vim.json" => "vim-dot-json" + groupname = groupname.replace('-dot-', '.') + + section_docs[groupname] = "\n".join(doc_list) + + return section_docs + + def main(doxygen_config, args): """Generates: @@ -1042,62 +1264,44 @@ def main(doxygen_config, args): fn_map_full = {} # Collects all functions as each module is processed. sections = {} - intros = {} sep = '=' * text_width base = os.path.join(output_dir, 'xml') dom = minidom.parse(os.path.join(base, 'index.xml')) - # generate docs for section intros - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'group': - continue - - groupname = get_text(find_first(compound, 'name')) - groupxml = os.path.join(base, '%s.xml' % - compound.getAttribute('refid')) - - group_parsed = minidom.parse(groupxml) - doc_list = [] - brief_desc = find_first(group_parsed, 'briefdescription') - if brief_desc: - for child in brief_desc.childNodes: - doc_list.append(fmt_node_as_vimhelp(child)) - - desc = find_first(group_parsed, 'detaileddescription') - if desc: - doc = fmt_node_as_vimhelp(desc) - - if doc: - doc_list.append(doc) - - intros[groupname] = "\n".join(doc_list) + section_docs = extract_defgroups(base, dom) + # Generate docs for all functions in the current module. for compound in dom.getElementsByTagName('compound'): if compound.getAttribute('kind') != 'file': continue filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): - xmlfile = os.path.join(base, - '{}.xml'.format(compound.getAttribute('refid'))) + xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) # Extract unformatted (*.mpack). fn_map, _ = extract_from_xml(xmlfile, target, 9999, False) # Extract formatted (:help). functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( - os.path.join(base, '{}.xml'.format( - compound.getAttribute('refid'))), target) + os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))), target) if not functions_text and not deprecated_text: continue else: - name = os.path.splitext( - os.path.basename(filename))[0].lower() + filename = os.path.basename(filename) + name = os.path.splitext(filename)[0].lower() sectname = name.upper() if name == 'ui' else name.title() + sectname = CONFIG[target]['section_name'].get(filename, sectname) + title = CONFIG[target]['section_fmt'](sectname) + section_tag = CONFIG[target]['helptag_fmt'](sectname) + # Module/Section id matched against @defgroup. + # "*api-buffer*" => "api-buffer" + section_id = section_tag.strip('*') + doc = '' - intro = intros.get(f'api-{name}') - if intro: - doc += '\n\n' + intro + section_doc = section_docs.get(section_id) + if section_doc: + doc += '\n\n' + section_doc if functions_text: doc += '\n\n' + functions_text @@ -1107,12 +1311,7 @@ def main(doxygen_config, args): doc += deprecated_text if doc: - filename = os.path.basename(filename) - sectname = CONFIG[target]['section_name'].get( - filename, sectname) - title = CONFIG[target]['section_fmt'](sectname) - helptag = CONFIG[target]['helptag_fmt'](sectname) - sections[filename] = (title, helptag, doc) + sections[filename] = (title, section_tag, doc) fn_map_full.update(fn_map) if len(sections) == 0: @@ -1127,15 +1326,14 @@ def main(doxygen_config, args): for filename in CONFIG[target]['section_order']: try: - title, helptag, section_doc = sections.pop(filename) + title, section_tag, section_doc = sections.pop(filename) except KeyError: msg(f'warning: empty docs, skipping (target={target}): {filename}') msg(f' existing docs: {sections.keys()}') continue if filename not in CONFIG[target]['append_only']: docs += sep - docs += '\n%s%s' % (title, - helptag.rjust(text_width - len(title))) + docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title))) docs += section_doc docs += '\n\n\n' @@ -1160,10 +1358,12 @@ def main(doxygen_config, args): msg_report() -def filter_source(filename): +def filter_source(filename, keep_tmpfiles): + output_dir = out_dir.format(target='lua2dox') name, extension = os.path.splitext(filename) if extension == '.lua': - p = subprocess.run(['nvim', '-l', lua2dox, filename], stdout=subprocess.PIPE) + args = [str(nvim), '-l', lua2dox, filename] + (['--outdir', output_dir] if keep_tmpfiles else []) + p = subprocess.run(args, stdout=subprocess.PIPE) op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8')) print(op) else: @@ -1186,7 +1386,7 @@ def parse_args(): ap.add_argument('source_filter', nargs='*', help="Filter source file(s)") ap.add_argument('-k', '--keep-tmpfiles', action='store_true', - help="Keep temporary files") + help="Keep temporary files (tmp-xx-doc/ directories, including tmp-lua2dox-doc/ for lua2dox.lua quasi-C output)") ap.add_argument('-t', '--target', help=f'One of ({targets}), defaults to "all"') return ap.parse_args() @@ -1234,8 +1434,13 @@ if __name__ == "__main__": log.setLevel(args.log_level) log.addHandler(logging.StreamHandler()) + # When invoked as a filter, args won't be passed, so use an env var. + if args.keep_tmpfiles: + os.environ['NVIM_KEEP_TMPFILES'] = '1' + keep_tmpfiles = ('NVIM_KEEP_TMPFILES' in os.environ) + if len(args.source_filter) > 0: - filter_source(args.source_filter[0]) + filter_source(args.source_filter[0], keep_tmpfiles) else: main(Doxyfile, args) |