From 5e2d4b3c4dbd56342cabb6993d354f690e0a1575 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 28 Dec 2023 13:04:34 -0500 Subject: refactor(gen_vimdoc): use stronger typing for CONFIG, avoid dict --- scripts/gen_vimdoc.py | 279 +++++++++++++++++++++++++++++--------------------- 1 file changed, 165 insertions(+), 114 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index b51bd5fbf5..f22d11ed6b 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. + +r"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. Also generates *.mpack files. To inspect the *.mpack structure: :new | put=v:lua.vim.inspect(v:lua.vim.mpack.decode(readfile('runtime/doc/api.mpack','B'))) @@ -32,24 +33,29 @@ The generated :help text for each function is formatted as follows: parameter is marked as [out]. - Each function documentation is separated by a single line. """ + +from __future__ import annotations + import argparse +import collections +import dataclasses +import logging import os import re -import sys import shutil -import textwrap import subprocess -import collections -import msgpack -import logging -from typing import Tuple +import sys +import textwrap from pathlib import Path - +from typing import Any, Callable, Dict, List, Literal, Tuple from xml.dom import minidom + +import msgpack + Element = minidom.Element Document = minidom.Document -MIN_PYTHON_VERSION = (3, 6) +MIN_PYTHON_VERSION = (3, 7) MIN_DOXYGEN_VERSION = (1, 9, 0) if sys.version_info < MIN_PYTHON_VERSION: @@ -68,7 +74,7 @@ if doxygen_version < MIN_DOXYGEN_VERSION: # Need a `nvim` that supports `-l`, try the local build nvim_path = Path(__file__).parent / "../build/bin/nvim" if nvim_path.exists(): - nvim = str(nvim_path) + nvim = nvim_path.resolve() else: # Until 0.9 is released, use this hacky way to check that "nvim -l foo.lua" works. nvim_out = subprocess.check_output(['nvim', '-h'], universal_newlines=True) @@ -103,12 +109,59 @@ filter_cmd = '%s %s' % (sys.executable, script_path) msgs = [] # Messages to show on exit. lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') -CONFIG = { - 'api': { - 'mode': 'c', - 'filename': 'api.txt', + +SectionName = str + +@dataclasses.dataclass +class Config: + """Config for documentation.""" + + mode: Literal['c', 'lua'] + + filename: str + """Generated documentation target, e.g. api.txt""" + + section_order: List[str] + """Section ordering.""" + + files: List[str] + """List of files/directories for doxygen to read, relative to `base_dir`.""" + + file_patterns: str + """file patterns used by doxygen.""" + + section_name: Dict[str, SectionName] + """Section name overrides. Key: filename (e.g., vim.c)""" + + section_fmt: Callable[[SectionName], str] + """For generated section names.""" + + helptag_fmt: Callable[[SectionName], str] + """Section helptag.""" + + fn_helptag_fmt: Callable[[str, str, bool], str] + """Per-function helptag.""" + + module_override: Dict[str, str] + """Module name overrides (for Lua).""" + + append_only: List[str] + """Append the docs for these modules, do not start a new section.""" + + fn_name_prefix: str + """Only function with this prefix are considered""" + + fn_name_fmt: Callable[[str, str], str] | None = None + + include_tables: bool = True + + +CONFIG: Dict[str, Config] = { + 'api': Config( + mode = 'c', + filename = 'api.txt', # Section ordering. - 'section_order': [ + section_order=[ 'vim.c', 'vimscript.c', 'command.c', @@ -121,31 +174,22 @@ CONFIG = { 'autocmd.c', 'ui.c', ], - # List of files/directories for doxygen to read, relative to `base_dir` - 'files': ['src/nvim/api'], - # file patterns used by doxygen - 'file_patterns': '*.h *.c', - # Only function with this prefix are considered - 'fn_name_prefix': 'nvim_', - # Section name overrides. - 'section_name': { + files=['src/nvim/api'], + file_patterns = '*.h *.c', + fn_name_prefix = 'nvim_', + section_name={ 'vim.c': 'Global', }, - # For generated section names. - 'section_fmt': lambda name: f'{name} Functions', - # Section helptag. - 'helptag_fmt': lambda name: f'*api-{name.lower()}*', - # Per-function helptag. - 'fn_helptag_fmt': lambda fstem, name, istbl: f'*{name}()*', - # Module name overrides (for Lua). - 'module_override': {}, - # Append the docs for these modules, do not start a new section. - 'append_only': [], - }, - 'lua': { - 'mode': 'lua', - 'filename': 'lua.txt', - 'section_order': [ + section_fmt=lambda name: f'{name} Functions', + helptag_fmt=lambda name: f'*api-{name.lower()}*', + fn_helptag_fmt=lambda fstem, name, istbl: f'*{name}()*', + module_override={}, + append_only=[], + ), + 'lua': Config( + mode='lua', + filename='lua.txt', + section_order=[ 'highlight.lua', 'regex.lua', 'diff.lua', @@ -171,7 +215,7 @@ CONFIG = { 'snippet.lua', 'text.lua', ], - 'files': [ + files=[ 'runtime/lua/vim/iter.lua', 'runtime/lua/vim/_editor.lua', 'runtime/lua/vim/_options.lua', @@ -197,30 +241,30 @@ CONFIG = { 'runtime/lua/vim/_meta/regex.lua', 'runtime/lua/vim/_meta/spell.lua', ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'fn_name_fmt': lambda fstem, name: ( + file_patterns='*.lua', + fn_name_prefix='', + fn_name_fmt=lambda fstem, name: ( name if fstem in [ 'vim.iter' ] else f'vim.{name}' if fstem in [ '_editor', 'vim.regex'] else f'vim.{name}' if fstem == '_options' and not name[0].isupper() else f'{fstem}.{name}' if fstem.startswith('vim') else name ), - 'section_name': { + section_name={ 'lsp.lua': 'core', '_inspector.lua': 'inspector', }, - 'section_fmt': lambda name: ( + section_fmt=lambda name: ( 'Lua module: vim' if name.lower() == '_editor' else 'LUA-VIMSCRIPT BRIDGE' if name.lower() == '_options' else f'VIM.{name.upper()}' if name.lower() in [ 'highlight', 'mpack', 'json', 'base64', 'diff', 'spell', 'regex' ] else 'VIM' if name.lower() == 'builtin' else f'Lua module: vim.{name.lower()}'), - 'helptag_fmt': lambda name: ( + helptag_fmt=lambda name: ( '*lua-vim*' if name.lower() == '_editor' else '*lua-vimscript*' if name.lower() == '_options' else f'*vim.{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( + fn_helptag_fmt=lambda fstem, name, istbl: ( f'*vim.opt:{name.split(":")[-1]}()*' if ':' in name and name.startswith('Option') else # Exclude fstem for methods f'*{name}()*' if ':' in name else @@ -230,7 +274,7 @@ CONFIG = { f'*{fstem}()*' if fstem.endswith('.' + name) else f'*{fstem}.{name}{"" if istbl else "()"}*' ), - 'module_override': { + module_override={ # `shared` functions are exposed on the `vim` module. 'shared': 'vim', '_inspector': 'vim', @@ -255,14 +299,14 @@ CONFIG = { 'text': 'vim.text', 'glob': 'vim.glob', }, - 'append_only': [ + append_only=[ 'shared.lua', ], - }, - 'lsp': { - 'mode': 'lua', - 'filename': 'lsp.txt', - 'section_order': [ + ), + 'lsp': Config( + mode='lua', + filename='lsp.txt', + section_order=[ 'lsp.lua', 'buf.lua', 'diagnostic.lua', @@ -276,50 +320,50 @@ CONFIG = { 'rpc.lua', 'protocol.lua', ], - 'files': [ + files=[ 'runtime/lua/vim/lsp', 'runtime/lua/vim/lsp.lua', ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'section_name': {'lsp.lua': 'lsp'}, - 'section_fmt': lambda name: ( + file_patterns='*.lua', + fn_name_prefix='', + section_name={'lsp.lua': 'lsp'}, + section_fmt=lambda name: ( 'Lua module: vim.lsp' if name.lower() == 'lsp' else f'Lua module: vim.lsp.{name.lower()}'), - 'helptag_fmt': lambda name: ( + helptag_fmt=lambda name: ( '*lsp-core*' if name.lower() == 'lsp' else f'*lsp-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( + fn_helptag_fmt=lambda fstem, name, istbl: ( f'*vim.lsp.{name}{"" if istbl else "()"}*' if fstem == 'lsp' and name != 'client' else # HACK. TODO(justinmk): class/structure support in lua2dox '*vim.lsp.client*' if 'lsp.client' == f'{fstem}.{name}' else f'*vim.lsp.{fstem}.{name}{"" if istbl else "()"}*'), - 'module_override': {}, - 'append_only': [], - }, - 'diagnostic': { - 'mode': 'lua', - 'filename': 'diagnostic.txt', - 'section_order': [ + module_override={}, + append_only=[], + ), + 'diagnostic': Config( + mode='lua', + filename='diagnostic.txt', + section_order=[ 'diagnostic.lua', ], - 'files': ['runtime/lua/vim/diagnostic.lua'], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'include_tables': False, - 'section_name': {'diagnostic.lua': 'diagnostic'}, - 'section_fmt': lambda _: 'Lua module: vim.diagnostic', - 'helptag_fmt': lambda _: '*diagnostic-api*', - 'fn_helptag_fmt': lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*', - 'module_override': {}, - 'append_only': [], - }, - 'treesitter': { - 'mode': 'lua', - 'filename': 'treesitter.txt', - 'section_order': [ + files=['runtime/lua/vim/diagnostic.lua'], + file_patterns='*.lua', + fn_name_prefix='', + include_tables=False, + section_name={'diagnostic.lua': 'diagnostic'}, + section_fmt=lambda _: 'Lua module: vim.diagnostic', + helptag_fmt=lambda _: '*diagnostic-api*', + fn_helptag_fmt=lambda fstem, name, istbl: f'*vim.{fstem}.{name}{"" if istbl else "()"}*', + module_override={}, + append_only=[], + ), + 'treesitter': Config( + mode='lua', + filename='treesitter.txt', + section_order=[ 'treesitter.lua', 'language.lua', 'query.lua', @@ -327,30 +371,30 @@ CONFIG = { 'languagetree.lua', 'dev.lua', ], - 'files': [ + files=[ 'runtime/lua/vim/treesitter.lua', 'runtime/lua/vim/treesitter/', ], - 'file_patterns': '*.lua', - 'fn_name_prefix': '', - 'section_name': {}, - 'section_fmt': lambda name: ( + file_patterns='*.lua', + fn_name_prefix='', + section_name={}, + section_fmt=lambda name: ( 'Lua module: vim.treesitter' if name.lower() == 'treesitter' else f'Lua module: vim.treesitter.{name.lower()}'), - 'helptag_fmt': lambda name: ( + helptag_fmt=lambda name: ( '*lua-treesitter-core*' if name.lower() == 'treesitter' else f'*lua-treesitter-{name.lower()}*'), - 'fn_helptag_fmt': lambda fstem, name, istbl: ( + fn_helptag_fmt=lambda fstem, name, istbl: ( f'*vim.{fstem}.{name}()*' if fstem == 'treesitter' else f'*{name}()*' if name[0].isupper() else f'*vim.treesitter.{fstem}.{name}()*'), - 'module_override': {}, - 'append_only': [], - } + module_override={}, + append_only=[], + ), } param_exclude = ( @@ -814,6 +858,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F return chunks, xrefs + def is_program_listing(para): """ Return True if `para` contains a "programlisting" (i.e. a Markdown code @@ -835,6 +880,7 @@ def is_program_listing(para): return len(children) == 1 and children[0].nodeName == 'programlisting' + def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='', fmt_vimhelp=False): """Renders (nested) Doxygen nodes as Vim :help text. @@ -910,6 +956,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): The `fmt_vimhelp` variable controls some special cases for use by fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) """ + config: Config = CONFIG[target] + fns = {} # Map of func_name:docstring. deprecated_fns = {} # Map of func_name:docstring. @@ -934,7 +982,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): continue istbl = return_type.startswith('table') # Special from lua2dox.lua. - if istbl and not CONFIG[target].get('include_tables', True): + if istbl and not config.include_tables: continue if return_type.startswith(('ArrayOf', 'DictionaryOf')): @@ -962,7 +1010,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): declname = get_child(param, 'declname') if declname: param_name = get_text(declname).strip() - elif CONFIG[target]['mode'] == 'lua': + elif config.mode == 'lua': # XXX: this is what lua2dox gives us... param_name = param_type param_type = '' @@ -998,11 +1046,11 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): fstem = '?' if '.' in compoundname: fstem = compoundname.split('.')[0] - fstem = CONFIG[target]['module_override'].get(fstem, fstem) - vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name, istbl) + fstem = config.module_override.get(fstem, fstem) + vimtag = config.fn_helptag_fmt(fstem, name, istbl) - if 'fn_name_fmt' in CONFIG[target]: - name = CONFIG[target]['fn_name_fmt'](fstem, name) + if config.fn_name_fmt: + name = config.fn_name_fmt(fstem, name) if istbl: aopen, aclose = '', '' @@ -1085,7 +1133,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): if 'Deprecated' in str(xrefs_all): deprecated_fns[name] = fn - elif name.startswith(CONFIG[target]['fn_name_prefix']): + elif name.startswith(config.fn_name_prefix): fns[name] = fn fns = collections.OrderedDict(sorted( @@ -1102,6 +1150,8 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): 1. Vim help text for functions found in `filename`. 2. Vim help text for deprecated functions. """ + config: Config = CONFIG[target] + fns_txt = {} # Map of func_name:vim-help-text. deprecated_fns_txt = {} # Map of func_name:vim-help-text. fns, _ = extract_from_xml(filename, target, text_width, True) @@ -1164,7 +1214,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): func_doc = "\n".join(map(align_tags, split_lines)) - if (name.startswith(CONFIG[target]['fn_name_prefix']) + if (name.startswith(config.fn_name_prefix) and name != "nvim_error_event"): fns_txt[name] = func_doc @@ -1237,9 +1287,12 @@ def main(doxygen_config, args): for target in CONFIG: if args.target is not None and target != args.target: continue + + config: Config = CONFIG[target] + mpack_file = os.path.join( base_dir, 'runtime', 'doc', - CONFIG[target]['filename'].replace('.txt', '.mpack')) + config.filename.replace('.txt', '.mpack')) if os.path.exists(mpack_file): os.remove(mpack_file) @@ -1255,11 +1308,10 @@ def main(doxygen_config, args): stderr=(subprocess.STDOUT if debug else subprocess.DEVNULL)) p.communicate( doxygen_config.format( - input=' '.join( - [f'"{file}"' for file in CONFIG[target]['files']]), + input=' '.join([f'"{file}"' for file in config.files]), output=output_dir, filter=filter_cmd, - file_patterns=CONFIG[target]['file_patterns']) + file_patterns=config.file_patterns) .encode('utf8') ) if p.returncode: @@ -1294,9 +1346,9 @@ def main(doxygen_config, args): filename = os.path.basename(filename) name = os.path.splitext(filename)[0].lower() sectname = name.upper() if name == 'ui' else name.title() - sectname = CONFIG[target]['section_name'].get(filename, sectname) - title = CONFIG[target]['section_fmt'](sectname) - section_tag = CONFIG[target]['helptag_fmt'](sectname) + sectname = config.section_name.get(filename, sectname) + title = config.section_fmt(sectname) + section_tag = config.helptag_fmt(sectname) # Module/Section id matched against @defgroup. # "*api-buffer*" => "api-buffer" section_id = section_tag.strip('*') @@ -1319,22 +1371,22 @@ def main(doxygen_config, args): if len(sections) == 0: fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') - if len(sections) > len(CONFIG[target]['section_order']): + if len(sections) > len(config.section_order): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( - set(sections).difference(CONFIG[target]['section_order']))) - first_section_tag = sections[CONFIG[target]['section_order'][0]][1] + set(sections).difference(config.section_order))) + first_section_tag = sections[config.section_order[0]][1] docs = '' - for filename in CONFIG[target]['section_order']: + for filename in config.section_order: try: title, section_tag, section_doc = sections.pop(filename) except KeyError: msg(f'warning: empty docs, skipping (target={target}): {filename}') msg(f' existing docs: {sections.keys()}') continue - if filename not in CONFIG[target]['append_only']: + if filename not in config.append_only: docs += sep docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title))) docs += section_doc @@ -1343,8 +1395,7 @@ def main(doxygen_config, args): docs = docs.rstrip() + '\n\n' docs += f' vim:tw=78:ts=8:sw={indentation}:sts={indentation}:et:ft=help:norl:\n' - doc_file = os.path.join(base_dir, 'runtime', 'doc', - CONFIG[target]['filename']) + doc_file = os.path.join(base_dir, 'runtime', 'doc', config.filename) if os.path.exists(doc_file): delete_lines_below(doc_file, first_section_tag) -- cgit From 1a31d4cf2b88e9f677d7a2f23b10e4dacf803a9d Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 28 Dec 2023 14:11:13 -0500 Subject: refactor(gen_vimdoc): use typing for function API vimdoc generation --- scripts/gen_vimdoc.py | 143 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 41 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index f22d11ed6b..40ece01ba3 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -112,6 +112,11 @@ lua2dox = os.path.join(base_dir, 'scripts', 'lua2dox.lua') SectionName = str +Docstring = str # Represents (formatted) vimdoc string + +FunctionName = str + + @dataclasses.dataclass class Config: """Config for documentation.""" @@ -881,6 +886,44 @@ def is_program_listing(para): return len(children) == 1 and children[0].nodeName == 'programlisting' +FunctionParam = Tuple[ + str, # type + str, # parameter name +] + +@dataclasses.dataclass +class FunctionDoc: + """Data structure for function documentation. Also exported as msgpack.""" + + annotations: List[str] + """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map""" + + signature: str + """Function signature with *tags*.""" + + parameters: List[FunctionParam] + """Parameters: (type, name)""" + + parameters_doc: Dict[str, Docstring] + """Parameters documentation. Key is parameter name, value is doc.""" + + doc: List[Docstring] + """Main description for the function. Separated by paragraph.""" + + return_: List[Docstring] + """Return:, or Return (multiple): (@return strings)""" + + seealso: List[Docstring] + """See also: (@see strings)""" + + # for fmt_node_as_vimhelp + desc_node: Element | None = None + brief_desc_node: Element | None = None + + # for INCLUDE_C_DECL + c_decl: str | None = None + + def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='', fmt_vimhelp=False): """Renders (nested) Doxygen nodes as Vim :help text. @@ -946,7 +989,10 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent= return clean_lines('\n'.join(rendered_blocks).strip()) -def extract_from_xml(filename, target, width, fmt_vimhelp): +def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ + Dict[FunctionName, FunctionDoc], + Dict[FunctionName, FunctionDoc], +]: """Extracts Doxygen info as maps without formatting the text. Returns two maps: @@ -958,8 +1004,8 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): """ config: Config = CONFIG[target] - fns = {} # Map of func_name:docstring. - deprecated_fns = {} # Map of func_name:docstring. + fns: Dict[FunctionName, FunctionDoc] = {} + deprecated_fns: Dict[FunctionName, FunctionDoc] = {} dom = minidom.parse(filename) compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) @@ -1084,7 +1130,7 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): # Tracks `xrefsect` titles. As of this writing, used only for separating # deprecated functions. xrefs_all = set() - paras = [] + paras: List[Dict[str, Any]] = [] brief_desc = find_first(member, 'briefdescription') if brief_desc: for child in brief_desc.childNodes: @@ -1103,47 +1149,48 @@ def extract_from_xml(filename, target, width, fmt_vimhelp): desc.toprettyxml(indent=' ', newl='\n')), ' ' * indentation)) - fn = { - 'annotations': list(annotations), - 'signature': signature, - 'parameters': params, - 'parameters_doc': collections.OrderedDict(), - 'doc': [], - 'return': [], - 'seealso': [], - } + fn = FunctionDoc( + annotations=list(annotations), + signature=signature, + parameters=params, + parameters_doc=collections.OrderedDict(), + doc=[], + return_=[], + seealso=[], + ) if fmt_vimhelp: - fn['desc_node'] = desc - fn['brief_desc_node'] = brief_desc + fn.desc_node = desc + fn.brief_desc_node = brief_desc for m in paras: - if 'text' in m: - if not m['text'] == '': - fn['doc'].append(m['text']) + if m.get('text', ''): + fn.doc.append(m['text']) if 'params' in m: # Merge OrderedDicts. - fn['parameters_doc'].update(m['params']) + fn.parameters_doc.update(m['params']) if 'return' in m and len(m['return']) > 0: - fn['return'] += m['return'] + fn.return_ += m['return'] if 'seealso' in m and len(m['seealso']) > 0: - fn['seealso'] += m['seealso'] + fn.seealso += m['seealso'] if INCLUDE_C_DECL: - fn['c_decl'] = c_decl + fn.c_decl = c_decl if 'Deprecated' in str(xrefs_all): deprecated_fns[name] = fn elif name.startswith(config.fn_name_prefix): fns[name] = fn + # sort functions by name (lexicographically) fns = collections.OrderedDict(sorted( fns.items(), - key=lambda key_item_tuple: key_item_tuple[0].lower())) + key=lambda key_item_tuple: key_item_tuple[0].lower(), + )) deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items())) return fns, deprecated_fns -def fmt_doxygen_xml_as_vimhelp(filename, target): +def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: """Entrypoint for generating Vim :help from from Doxygen XML. Returns 2 items: @@ -1154,20 +1201,26 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): fns_txt = {} # Map of func_name:vim-help-text. deprecated_fns_txt = {} # Map of func_name:vim-help-text. + + fns: Dict[FunctionName, FunctionDoc] fns, _ = extract_from_xml(filename, target, text_width, True) - for name, fn in fns.items(): + for fn_name, fn in fns.items(): # Generate Vim :help for parameters. - if fn['desc_node']: - doc = fmt_node_as_vimhelp(fn['desc_node'], fmt_vimhelp=True) - if not doc and fn['brief_desc_node']: - doc = fmt_node_as_vimhelp(fn['brief_desc_node']) - if not doc and name.startswith("nvim__"): + + # Generate body. + doc = '' + if fn.desc_node: + doc = fmt_node_as_vimhelp(fn.desc_node, fmt_vimhelp=True) + if not doc and fn.brief_desc_node: + doc = fmt_node_as_vimhelp(fn.brief_desc_node) + if not doc and fn_name.startswith("nvim__"): continue if not doc: doc = 'TODO: Documentation' - annotations = '\n'.join(fn['annotations']) + # Annotations: put before Parameters + annotations: str = '\n'.join(fn.annotations) if annotations: annotations = ('\n\nAttributes: ~\n' + textwrap.indent(annotations, ' ')) @@ -1177,18 +1230,22 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): else: doc = doc[:i] + annotations + '\n\n' + doc[i:] + # C Declaration: (debug only) if INCLUDE_C_DECL: doc += '\n\nC Declaration: ~\n>\n' - doc += fn['c_decl'] + assert fn.c_decl is not None + doc += fn.c_decl doc += '\n<' - func_doc = fn['signature'] + '\n' + # Start of function documentations. e.g., + # nvim_cmd({*cmd}, {*opts}) *nvim_cmd()* + func_doc = fn.signature + '\n' func_doc += textwrap.indent(clean_lines(doc), ' ' * indentation) # Verbatim handling. func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) - split_lines = func_doc.split('\n') + split_lines: List[str] = func_doc.split('\n') start = 0 while True: try: @@ -1214,12 +1271,14 @@ def fmt_doxygen_xml_as_vimhelp(filename, target): func_doc = "\n".join(map(align_tags, split_lines)) - if (name.startswith(config.fn_name_prefix) - and name != "nvim_error_event"): - fns_txt[name] = func_doc + if (fn_name.startswith(config.fn_name_prefix) + and fn_name != "nvim_error_event"): + fns_txt[fn_name] = func_doc - return ('\n\n'.join(list(fns_txt.values())), - '\n\n'.join(list(deprecated_fns_txt.values()))) + return ( + '\n\n'.join(list(fns_txt.values())), + '\n\n'.join(list(deprecated_fns_txt.values())), + ) def delete_lines_below(filename, tokenstr): @@ -1402,9 +1461,11 @@ def main(doxygen_config, args): with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) - fn_map_full = collections.OrderedDict(sorted(fn_map_full.items())) + fn_map_full = collections.OrderedDict(sorted( + (name, fn_doc.__dict__) for (name, fn_doc) in fn_map_full.items() + )) with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) + fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) # type: ignore if not args.keep_tmpfiles: shutil.rmtree(output_dir) -- cgit From 4e9298ecdf945b4d16c2c6e6e4ed82b97880917c Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Thu, 28 Dec 2023 17:50:05 -0500 Subject: refactor(gen_vimdoc): generate function doc from metadata, not from xml Problem: For function definitions to be included in the vimdoc (formatted) and to be exported as mpack data (unformatted), we had two internal representations of the same function/API metadata in duplicate; one is FunctionDoc (which was previously a dict), and the other is doxygen XML DOM from which vimdoc (functions sections) was generated. Solution: We should have a single path and unified data representation (i.e. FunctionDoc) that contains all the metadata and information about function APIs, from which both of mpack export and vimdoc are generated. I.e., vimdocs are no longer generated directly from doxygen XML nodes, but generated via: (XML DOM Nodes) ------------> FunctionDoc ------> mpack (unformatted) Recursive Internal | Formatting Metadata +---> vimdoc (formatted) This refactoring eliminates the hacky and ugly use of `fmt_vimhelp` in `fmt_node_as_vimhelp()` and all other helper functions! This way, `fmt_node_as_vimhelp()` can simplified as it no longer needs to handle generating of function docs, which needs to be done only in the topmost level of recursion. --- scripts/gen_vimdoc.py | 220 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 151 insertions(+), 69 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 40ece01ba3..eede5b76a2 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -751,7 +751,10 @@ def render_node(n, text, prefix='', indent='', width=text_width - indentation, return text -def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=False): +def para_as_map(parent: Element, + indent: str = '', + width: int = (text_width - indentation), + ): """Extracts a Doxygen XML node to a map. Keys: @@ -787,7 +790,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F kind = '' if is_inline(parent): # Flatten inline text from a tree of non-block nodes. - text = doc_wrap(render_node(parent, "", fmt_vimhelp=fmt_vimhelp), + text = doc_wrap(render_node(parent, ""), indent=indent, width=width) else: prev = None # Previous node @@ -805,8 +808,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F elif kind == 'see': groups['seealso'].append(child) elif kind == 'warning': - text += render_node(child, text, indent=indent, - width=width, fmt_vimhelp=fmt_vimhelp) + text += render_node(child, text, indent=indent, width=width) elif kind == 'since': since_match = re.match(r'^(\d+)', get_text(child)) since = int(since_match.group(1)) if since_match else 0 @@ -827,8 +829,7 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F and ' ' != text[-1]): text += ' ' - text += render_node(child, text, indent=indent, width=width, - fmt_vimhelp=fmt_vimhelp) + text += render_node(child, text, indent=indent, width=width) prev = child chunks['text'] += text @@ -839,17 +840,17 @@ def para_as_map(parent, indent='', width=text_width - indentation, fmt_vimhelp=F update_params_map(child, ret_map=chunks['params'], width=width) for child in groups['note']: chunks['note'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp).rstrip()) + child, '', indent=indent, width=width).rstrip()) for child in groups['return']: chunks['return'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp)) + child, '', indent=indent, width=width)) for child in groups['seealso']: # Example: # # |autocommand| # chunks['seealso'].append(render_node( - child, '', indent=indent, width=width, fmt_vimhelp=fmt_vimhelp)) + child, '', indent=indent, width=width)) xrefs = set() for child in groups['xrefs']: @@ -898,6 +899,9 @@ class FunctionDoc: annotations: List[str] """Attributes, e.g., FUNC_API_REMOTE_ONLY. See annotation_map""" + notes: List[Docstring] + """Notes: (@note strings)""" + signature: str """Function signature with *tags*.""" @@ -916,41 +920,122 @@ class FunctionDoc: seealso: List[Docstring] """See also: (@see strings)""" - # for fmt_node_as_vimhelp - desc_node: Element | None = None - brief_desc_node: Element | None = None + xrefs: List[Docstring] + """XRefs. Currently only used to track Deprecated functions.""" # for INCLUDE_C_DECL c_decl: str | None = None + prerelease: bool = False + + def export_mpack(self) -> Dict[str, Any]: + """Convert a dict to be exported as mpack data.""" + exported = self.__dict__.copy() + del exported['notes'] + del exported['c_decl'] + del exported['prerelease'] + del exported['xrefs'] + exported['return'] = exported.pop('return_') + return exported + + def doc_concatenated(self) -> Docstring: + """Concatenate all the paragraphs in `doc` into a single string, but + remove blank lines before 'programlisting' blocks. #25127 + + BEFORE (without programlisting processing): + ```vimdoc + Example: + + >vim + :echo nvim_get_color_by_name("Pink") + < + ``` + + AFTER: + ```vimdoc + Example: >vim + :echo nvim_get_color_by_name("Pink") + < + ``` + """ + def is_program_listing(paragraph: str) -> bool: + lines = paragraph.strip().split('\n') + return lines[0].startswith('>') and lines[-1] == '<' + + rendered = [] + for paragraph in self.doc: + if is_program_listing(paragraph): + rendered.append(' ') # Example: >vim + elif rendered: + rendered.append('\n\n') + rendered.append(paragraph) + return ''.join(rendered) + + def render(self) -> Docstring: + """Renders function documentation as Vim :help text.""" + rendered_blocks: List[Docstring] = [] + + def fmt_param_doc(m): + """Renders a params map as Vim :help text.""" + max_name_len = max_name(m.keys()) + 4 + out = '' + for name, desc in m.items(): + if name == 'self': + continue + name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + out += '{}{}\n'.format(name, desc) + return out.rstrip() + + # Generate text from the gathered items. + chunks: List[Docstring] = [self.doc_concatenated()] + + notes = [] + if self.prerelease: + notes = [" This API is pre-release (unstable)."] + notes += self.notes + if len(notes) > 0: + chunks.append('\nNote: ~') + for s in notes: + chunks.append(' ' + s) + + if self.parameters_doc: + chunks.append('\nParameters: ~') + chunks.append(fmt_param_doc(self.parameters_doc)) + + if self.return_: + chunks.append('\nReturn (multiple): ~' if len(self.return_) > 1 + else '\nReturn: ~') + for s in self.return_: + chunks.append(' ' + s) + + if self.seealso: + chunks.append('\nSee also: ~') + for s in self.seealso: + chunks.append(' ' + s) + + # Note: xrefs are currently only used to remark "Deprecated: " + # for deprecated functions; visible when INCLUDE_DEPRECATED is set + for s in self.xrefs: + chunks.append('\n' + s) + + rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) + rendered_blocks.append('') -def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent='', - fmt_vimhelp=False): + return clean_lines('\n'.join(rendered_blocks).strip()) + + +def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent=''): """Renders (nested) Doxygen nodes as Vim :help text. + Only handles "text" nodes. Used for individual elements (see render_node()) + and in extract_defgroups(). + NB: Blank lines in a docstring manifest as tags. """ rendered_blocks = [] - def fmt_param_doc(m): - """Renders a params map as Vim :help text.""" - max_name_len = max_name(m.keys()) + 4 - out = '' - for name, desc in m.items(): - if name == 'self': - continue - name = ' • {}'.format('{{{}}}'.format(name).ljust(max_name_len)) - out += '{}{}\n'.format(name, desc) - return out.rstrip() - - def has_nonexcluded_params(m): - """Returns true if any of the given params has at least - one non-excluded item.""" - if fmt_param_doc(m) != '': - return True - for child in parent.childNodes: - para, _ = para_as_map(child, indent, width, fmt_vimhelp) + para, _ = para_as_map(child, indent, width) # 'programlisting' blocks are Markdown code blocks. Do not include # these as a separate paragraph, but append to the last non-empty line @@ -963,25 +1048,6 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent= # Generate text from the gathered items. chunks = [para['text']] - notes = [" This API is pre-release (unstable)."] if para['prerelease'] else [] - notes += para['note'] - if len(notes) > 0: - chunks.append('\nNote: ~') - for s in notes: - chunks.append(s) - if len(para['params']) > 0 and has_nonexcluded_params(para['params']): - chunks.append('\nParameters: ~') - chunks.append(fmt_param_doc(para['params'])) - if len(para['return']) > 0: - chunks.append('\nReturn (multiple): ~' if len(para['return']) > 1 else '\nReturn: ~') - for s in para['return']: - chunks.append(s) - if len(para['seealso']) > 0: - chunks.append('\nSee also: ~') - for s in para['seealso']: - chunks.append(s) - for s in para['xrefs']: - chunks.append(s) rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) rendered_blocks.append('') @@ -989,7 +1055,8 @@ def fmt_node_as_vimhelp(parent: Element, width=text_width - indentation, indent= return clean_lines('\n'.join(rendered_blocks).strip()) -def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ +def extract_from_xml(filename, target, *, + width: int, fmt_vimhelp: bool) -> Tuple[ Dict[FunctionName, FunctionDoc], Dict[FunctionName, FunctionDoc], ]: @@ -1130,18 +1197,20 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ # Tracks `xrefsect` titles. As of this writing, used only for separating # deprecated functions. xrefs_all = set() - paras: List[Dict[str, Any]] = [] + paras: List[Dict[str, Any]] = [] # paras means paragraphs! brief_desc = find_first(member, 'briefdescription') if brief_desc: for child in brief_desc.childNodes: para, xrefs = para_as_map(child) + paras.append(para) xrefs_all.update(xrefs) desc = find_first(member, 'detaileddescription') if desc: + paras_detail = [] # override briefdescription for child in desc.childNodes: para, xrefs = para_as_map(child) - paras.append(para) + paras_detail.append(para) xrefs_all.update(xrefs) log.debug( textwrap.indent( @@ -1149,18 +1218,25 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ desc.toprettyxml(indent=' ', newl='\n')), ' ' * indentation)) + # override briefdescription, if detaileddescription is not empty + # (note: briefdescription can contain some erroneous luadoc + # comments from preceding comments, this is a bug of lua2dox) + if any((para['text'] or para['note'] or para['params'] or + para['return'] or para['seealso'] + ) for para in paras_detail): + paras = paras_detail + fn = FunctionDoc( annotations=list(annotations), + notes=[], signature=signature, parameters=params, parameters_doc=collections.OrderedDict(), doc=[], return_=[], seealso=[], + xrefs=[], ) - if fmt_vimhelp: - fn.desc_node = desc - fn.brief_desc_node = brief_desc for m in paras: if m.get('text', ''): @@ -1172,6 +1248,12 @@ def extract_from_xml(filename, target, width, fmt_vimhelp) -> Tuple[ fn.return_ += m['return'] if 'seealso' in m and len(m['seealso']) > 0: fn.seealso += m['seealso'] + if m.get('prerelease', False): + fn.prerelease = True + if 'note' in m: + fn.notes += m['note'] + if 'xrefs' in m: + fn.xrefs += m['xrefs'] if INCLUDE_C_DECL: fn.c_decl = c_decl @@ -1203,17 +1285,14 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: deprecated_fns_txt = {} # Map of func_name:vim-help-text. fns: Dict[FunctionName, FunctionDoc] - fns, _ = extract_from_xml(filename, target, text_width, True) + fns, _ = extract_from_xml(filename, target, + width=text_width, fmt_vimhelp=True) for fn_name, fn in fns.items(): # Generate Vim :help for parameters. - # Generate body. - doc = '' - if fn.desc_node: - doc = fmt_node_as_vimhelp(fn.desc_node, fmt_vimhelp=True) - if not doc and fn.brief_desc_node: - doc = fmt_node_as_vimhelp(fn.brief_desc_node) + # Generate body from FunctionDoc, not XML nodes + doc = fn.render() if not doc and fn_name.startswith("nvim__"): continue if not doc: @@ -1393,11 +1472,14 @@ def main(doxygen_config, args): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) + # Extract unformatted (*.mpack). - fn_map, _ = extract_from_xml(xmlfile, target, 9999, False) + fn_map, _ = extract_from_xml( + xmlfile, target, width=9999, fmt_vimhelp=False) + # Extract formatted (:help). functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( - os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))), target) + xmlfile, target) if not functions_text and not deprecated_text: continue @@ -1461,11 +1543,11 @@ def main(doxygen_config, args): with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) - fn_map_full = collections.OrderedDict(sorted( - (name, fn_doc.__dict__) for (name, fn_doc) in fn_map_full.items() + fn_map_full_exported = collections.OrderedDict(sorted( + (name, fn_doc.export_mpack()) for (name, fn_doc) in fn_map_full.items() )) with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) # type: ignore + fp.write(msgpack.packb(fn_map_full_exported, use_bin_type=True)) # type: ignore if not args.keep_tmpfiles: shutil.rmtree(output_dir) -- cgit From f74f52a1a5ff668187bf12c397319972764a9704 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Fri, 29 Dec 2023 05:52:31 -0500 Subject: refactor(gen_vimdoc): refactor section and defgroup doc generation Problem: main() has too much logic implemented there, too difficult to read. Solution: Do more OOP, introduce `Section` dataclass that stores information about a "section", with documentation and concrete examples about what each field and variable would mean. Extract all the lines for rendering a section into `section.render()` pulled out of `main()`. --- scripts/gen_vimdoc.py | 185 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 138 insertions(+), 47 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index eede5b76a2..e5f2e61dfd 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -102,6 +102,8 @@ LOG_LEVELS = { text_width = 78 indentation = 4 +SECTION_SEP = '=' * text_width + script_path = os.path.abspath(__file__) base_dir = os.path.dirname(os.path.dirname(script_path)) out_dir = os.path.join(base_dir, 'tmp-{target}-doc') @@ -1378,7 +1380,7 @@ def delete_lines_below(filename, tokenstr): fp.writelines(lines[0:i]) -def extract_defgroups(base: str, dom: Document): +def extract_defgroups(base: str, dom: Document) -> Dict[SectionName, Docstring]: '''Generate module-level (section) docs (@defgroup).''' section_docs = {} @@ -1414,6 +1416,101 @@ def extract_defgroups(base: str, dom: Document): return section_docs +@dataclasses.dataclass +class Section: + """Represents a section. Includes section heading (defgroup) + and all the FunctionDoc that belongs to this section.""" + + name: str + '''Name of the section. Usually derived from basename of lua/c src file. + Example: "Autocmd".''' + + title: str + '''Formatted section config. see config.section_fmt(). + Example: "Autocmd Functions". ''' + + helptag: str + '''see config.helptag_fmt(). Example: *api-autocmd*''' + + @property + def id(self) -> str: + '''section id: Module/Section id matched against @defgroup. + e.g., "*api-autocmd*" => "api-autocmd" + ''' + return self.helptag.strip('*') + + doc: str = "" + '''Section heading docs extracted from @defgroup.''' + + # TODO: Do not carry rendered text, but handle FunctionDoc for better OOP + functions_text: Docstring | None = None + '''(Rendered) doc of all the functions that belong to this section.''' + + deprecated_functions_text: Docstring | None = None + '''(Rendered) doc of all the deprecated functions that belong to this + section.''' + + def __repr__(self): + return f"Section(title='{self.title}', helptag='{self.helptag}')" + + @classmethod + def make_from(cls, filename: str, config: Config, + section_docs: Dict[SectionName, str], + *, + functions_text: Docstring, + deprecated_functions_text: Docstring, + ): + # filename: e.g., 'autocmd.c' + # name: e.g. 'autocmd' + name = os.path.splitext(filename)[0].lower() + + # section name: e.g. "Autocmd" + sectname: SectionName + sectname = name.upper() if name == 'ui' else name.title() + sectname = config.section_name.get(filename, sectname) + + # Formatted (this is what's going to be written in the vimdoc) + # e.g., "Autocmd Functions" + title: str = config.section_fmt(sectname) + + # section tag: e.g., "*api-autocmd*" + section_tag: str = config.helptag_fmt(sectname) + + section = cls(name=sectname, title=title, helptag=section_tag, + functions_text=functions_text, + deprecated_functions_text=deprecated_functions_text, + ) + section.doc = section_docs.get(section.id) or '' + return section + + def render(self, add_header=True) -> str: + """Render as vimdoc.""" + doc = '' + + if add_header: + doc += SECTION_SEP + doc += '\n{}{}'.format( + self.title, + self.helptag.rjust(text_width - len(self.title)) + ) + + if self.doc: + doc += '\n\n' + self.doc + + if self.functions_text: + doc += '\n\n' + self.functions_text + + if INCLUDE_DEPRECATED and self.deprecated_functions_text: + doc += f'\n\n\nDeprecated {self.name} Functions: ~\n\n' + doc += self.deprecated_functions_text + + return doc + + def __bool__(self) -> bool: + """Whether this section has contents. Used for skipping empty ones.""" + return bool(self.doc or self.functions_text) + + def main(doxygen_config, args): """Generates: @@ -1455,14 +1552,16 @@ def main(doxygen_config, args): if p.returncode: sys.exit(p.returncode) - fn_map_full = {} # Collects all functions as each module is processed. - sections = {} - sep = '=' * text_width + # Collects all functions as each module is processed. + fn_map_full: Dict[FunctionName, FunctionDoc] = {} + # key: filename (e.g. autocmd.c) + sections: Dict[str, Section] = {} base = os.path.join(output_dir, 'xml') dom = minidom.parse(os.path.join(base, 'index.xml')) - section_docs = extract_defgroups(base, dom) + # Collect all @defgroups (section headings after the '===...' separator + section_docs: Dict[SectionName, Docstring] = extract_defgroups(base, dom) # Generate docs for all functions in the current module. for compound in dom.getElementsByTagName('compound'): @@ -1470,45 +1569,38 @@ def main(doxygen_config, args): continue filename = get_text(find_first(compound, 'name')) - if filename.endswith('.c') or filename.endswith('.lua'): - xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) + if not ( + filename.endswith('.c') or + filename.endswith('.lua') + ): + continue - # Extract unformatted (*.mpack). - fn_map, _ = extract_from_xml( - xmlfile, target, width=9999, fmt_vimhelp=False) + xmlfile = os.path.join(base, '{}.xml'.format(compound.getAttribute('refid'))) - # Extract formatted (:help). - functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( - xmlfile, target) + # Extract unformatted (*.mpack). + fn_map, _ = extract_from_xml( + xmlfile, target, width=9999, fmt_vimhelp=False) - if not functions_text and not deprecated_text: - continue - else: - filename = os.path.basename(filename) - name = os.path.splitext(filename)[0].lower() - sectname = name.upper() if name == 'ui' else name.title() - sectname = config.section_name.get(filename, sectname) - title = config.section_fmt(sectname) - section_tag = config.helptag_fmt(sectname) - # Module/Section id matched against @defgroup. - # "*api-buffer*" => "api-buffer" - section_id = section_tag.strip('*') - - doc = '' - section_doc = section_docs.get(section_id) - if section_doc: - doc += '\n\n' + section_doc - - if functions_text: - doc += '\n\n' + functions_text - - if INCLUDE_DEPRECATED and deprecated_text: - doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n' - doc += deprecated_text - - if doc: - sections[filename] = (title, section_tag, doc) - fn_map_full.update(fn_map) + # Extract formatted (:help). + functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( + xmlfile, target) + + if not functions_text and not deprecated_text: + continue + + filename = os.path.basename(filename) + + section: Section = Section.make_from( + filename, config, section_docs, + functions_text=functions_text, + deprecated_functions_text=deprecated_text, + ) + + if section: # if not empty + sections[filename] = section + fn_map_full.update(fn_map) + else: + log.debug("Skipping empty section: %s", section) if len(sections) == 0: fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') @@ -1516,21 +1608,20 @@ def main(doxygen_config, args): raise RuntimeError( 'found new modules "{}"; update the "section_order" map'.format( set(sections).difference(config.section_order))) - first_section_tag = sections[config.section_order[0]][1] + first_section_tag = sections[config.section_order[0]].helptag docs = '' for filename in config.section_order: try: - title, section_tag, section_doc = sections.pop(filename) + section: Section = sections.pop(filename) except KeyError: msg(f'warning: empty docs, skipping (target={target}): {filename}') msg(f' existing docs: {sections.keys()}') continue - if filename not in config.append_only: - docs += sep - docs += '\n{}{}'.format(title, section_tag.rjust(text_width - len(title))) - docs += section_doc + + add_sep_and_header = filename not in config.append_only + docs += section.render(add_header=add_sep_and_header) docs += '\n\n\n' docs = docs.rstrip() + '\n\n' -- cgit From 765729a145d3d8204ff68b1da0b5bb45c70262e2 Mon Sep 17 00:00:00 2001 From: Jongwook Choi Date: Fri, 29 Dec 2023 06:32:37 -0500 Subject: fix(gen_vimdoc): INCLUDE_DEPRECATED not generating docs for deprecateds Since some point INCLUDE_DEPRECATED stopped working as it is usually turned off when generating an actual vimdoc. This commit fixes this hidden feature back again (used for devel purposes only). --- scripts/gen_vimdoc.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) (limited to 'scripts/gen_vimdoc.py') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index e5f2e61dfd..804f0fd198 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -89,8 +89,8 @@ else: # DEBUG = ('DEBUG' in os.environ) -INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) -INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) +INCLUDE_C_DECL = os.environ.get('INCLUDE_C_DECL', '0') != '0' +INCLUDE_DEPRECATED = os.environ.get('INCLUDE_DEPRECATED', '0') != '0' log = logging.getLogger(__name__) @@ -168,7 +168,7 @@ CONFIG: Dict[str, Config] = { mode = 'c', filename = 'api.txt', # Section ordering. - section_order=[ + section_order=[x for x in [ 'vim.c', 'vimscript.c', 'command.c', @@ -180,7 +180,8 @@ CONFIG: Dict[str, Config] = { 'tabpage.c', 'autocmd.c', 'ui.c', - ], + 'deprecated.c' if INCLUDE_DEPRECATED else '' + ] if x], files=['src/nvim/api'], file_patterns = '*.h *.c', fn_name_prefix = 'nvim_', @@ -1287,18 +1288,21 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: deprecated_fns_txt = {} # Map of func_name:vim-help-text. fns: Dict[FunctionName, FunctionDoc] - fns, _ = extract_from_xml(filename, target, - width=text_width, fmt_vimhelp=True) + deprecated_fns: Dict[FunctionName, FunctionDoc] + fns, deprecated_fns = extract_from_xml( + filename, target, width=text_width, fmt_vimhelp=True) - for fn_name, fn in fns.items(): + def _handle_fn(fn_name: FunctionName, fn: FunctionDoc, + fns_txt: Dict[FunctionName, Docstring], deprecated=False): # Generate Vim :help for parameters. # Generate body from FunctionDoc, not XML nodes doc = fn.render() if not doc and fn_name.startswith("nvim__"): - continue + return if not doc: - doc = 'TODO: Documentation' + doc = ('TODO: Documentation' if not deprecated + else 'Deprecated.') # Annotations: put before Parameters annotations: str = '\n'.join(fn.annotations) @@ -1356,6 +1360,11 @@ def fmt_doxygen_xml_as_vimhelp(filename, target) -> Tuple[Docstring, Docstring]: and fn_name != "nvim_error_event"): fns_txt[fn_name] = func_doc + for fn_name, fn in fns.items(): + _handle_fn(fn_name, fn, fns_txt) + for fn_name, fn in deprecated_fns.items(): + _handle_fn(fn_name, fn, deprecated_fns_txt, deprecated=True) + return ( '\n\n'.join(list(fns_txt.values())), '\n\n'.join(list(deprecated_fns_txt.values())), @@ -1508,7 +1517,8 @@ class Section: def __bool__(self) -> bool: """Whether this section has contents. Used for skipping empty ones.""" - return bool(self.doc or self.functions_text) + return bool(self.doc or self.functions_text or + (INCLUDE_DEPRECATED and self.deprecated_functions_text)) def main(doxygen_config, args): @@ -1606,8 +1616,11 @@ def main(doxygen_config, args): fail(f'no sections for target: {target} (look for errors near "Preprocessing" log lines above)') if len(sections) > len(config.section_order): raise RuntimeError( - 'found new modules "{}"; update the "section_order" map'.format( - set(sections).difference(config.section_order))) + '{}: found new modules {}; ' + 'update the "section_order" map'.format( + target, + set(sections).difference(config.section_order)) + ) first_section_tag = sections[config.section_order[0]].helptag docs = '' -- cgit