diff options
Diffstat (limited to 'scripts/gen_vimdoc.py')
| -rwxr-xr-x | scripts/gen_vimdoc.py | 795 |
1 files changed, 531 insertions, 264 deletions
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 373a58d11e..a61690e99f 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -1,5 +1,18 @@ #!/usr/bin/env python3 -"""Generates Nvim help docs from C docstrings, by parsing Doxygen XML. +"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. + +Also generates *.mpack files. To inspect the *.mpack structure: + + :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack'))) + + +Flow: + main + extract_from_xml + fmt_node_as_vimhelp \ + para_as_map } recursive + update_params_map / + render_node This would be easier using lxml and XSLT, but: @@ -9,25 +22,22 @@ This would be easier using lxml and XSLT, but: 2. I wouldn't know how to deal with nested indentation in <para> tags using XSLT. -Each function documentation is formatted with the following rules: +Each function :help block is formatted as follows: - - Maximum width of 78 characters (`text_width`). - - Spaces for indentation. - - Function signature and helptag are on the same line. - - Helptag is right aligned. + - Max width of 78 columns (`text_width`). + - Indent with spaces (not tabs). + - Indent of 16 columns for body text. + - Function signature and helptag (right-aligned) on the same line. - Signature and helptag must have a minimum of 8 spaces between them. - - If the signature is too long, it is placed on the line after the - helptag. The signature wraps at `text_width - 8` characters with - subsequent lines indented to the open parenthesis. - - Documentation body will be indented by 16 spaces. + - If the signature is too long, it is placed on the line after the helptag. + Signature wraps at `text_width - 8` characters with subsequent + lines indented to the open parenthesis. - Subsection bodies are indented an additional 4 spaces. - - Documentation body consists of the function description, parameter details, - return description, and C declaration. + - Body consists of function description, parameters, return description, and + C declaration (`INCLUDE_C_DECL`). - Parameters are omitted for the `void` and `Error *` types, or if the parameter is marked as [out]. - Each function documentation is separated by a single line. - -The C declaration is added to the end to show actual argument types. """ import os import re @@ -36,27 +46,33 @@ import shutil import textwrap import subprocess import collections +import msgpack from xml.dom import minidom -if sys.version_info[0] < 3: - print("use Python 3") +MIN_PYTHON_VERSION = (3, 5) + +if sys.version_info < MIN_PYTHON_VERSION: + print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION)) sys.exit(1) DEBUG = ('DEBUG' in os.environ) +TARGET = os.environ.get('TARGET', None) INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) +fmt_vimhelp = False # HACK text_width = 78 script_path = os.path.abspath(__file__) base_dir = os.path.dirname(os.path.dirname(script_path)) -out_dir = os.path.join(base_dir, 'tmp-{mode}-doc') +out_dir = os.path.join(base_dir, 'tmp-{target}-doc') filter_cmd = '%s %s' % (sys.executable, script_path) seen_funcs = set() lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter') CONFIG = { 'api': { + 'mode': 'c', 'filename': 'api.txt', # String used to find the start of the generated part of the doc. 'section_start_token': '*api-global*', @@ -73,18 +89,25 @@ CONFIG = { # file patterns used by doxygen 'file_patterns': '*.h *.c', # Only function with this prefix are considered - 'func_name_prefix': 'nvim_', + 'fn_name_prefix': 'nvim_', # Section name overrides. 'section_name': { 'vim.c': 'Global', }, + # For generated section names. + 'section_fmt': lambda name: f'{name} Functions', + # Section helptag. + 'helptag_fmt': lambda name: f'*api-{name.lower()}*', + # Per-function helptag. + 'fn_helptag_fmt': lambda fstem, name: f'*{name}()*', # Module name overrides (for Lua). 'module_override': {}, # Append the docs for these modules, do not start a new section. 'append_only': [], }, 'lua': { - 'filename': 'if_lua.txt', + 'mode': 'lua', + 'filename': 'lua.txt', 'section_start_token': '*lua-vim*', 'section_order': [ 'vim.lua', @@ -95,8 +118,13 @@ CONFIG = { os.path.join(base_dir, 'runtime/lua/vim/shared.lua'), ]), 'file_patterns': '*.lua', - 'func_name_prefix': '', - 'section_name': {}, + 'fn_name_prefix': '', + 'section_name': { + 'lsp.lua': 'core', + }, + 'section_fmt': lambda name: f'Lua module: {name.lower()}', + 'helptag_fmt': lambda name: f'*lua-{name.lower()}*', + 'fn_helptag_fmt': lambda fstem, name: f'*{fstem}.{name}()*', 'module_override': { # `shared` functions are exposed on the `vim` module. 'shared': 'vim', @@ -105,6 +133,45 @@ CONFIG = { 'shared.lua', ], }, + 'lsp': { + 'mode': 'lua', + 'filename': 'lsp.txt', + 'section_start_token': '*lsp-core*', + 'section_order': [ + 'lsp.lua', + 'protocol.lua', + 'buf.lua', + 'callbacks.lua', + 'log.lua', + 'rpc.lua', + 'util.lua' + ], + 'files': ' '.join([ + os.path.join(base_dir, 'runtime/lua/vim/lsp'), + os.path.join(base_dir, 'runtime/lua/vim/lsp.lua'), + ]), + 'file_patterns': '*.lua', + 'fn_name_prefix': '', + 'section_name': {}, + 'section_fmt': lambda name: ( + 'Lua module: vim.lsp' + if name.lower() == 'lsp' + else f'Lua module: vim.lsp.{name.lower()}'), + 'helptag_fmt': lambda name: ( + '*lsp-core*' + if name.lower() == 'lsp' + else f'*lsp-{name.lower()}*'), + 'fn_helptag_fmt': lambda fstem, name: ( + f'*vim.lsp.{name}()*' + if fstem == 'lsp' and name != 'client' + else ( + '*vim.lsp.client*' + # HACK. TODO(justinmk): class/structure support in lua2dox + if 'lsp.client' == f'{fstem}.{name}' + else f'*vim.lsp.{fstem}.{name}()*')), + 'module_override': {}, + 'append_only': [], + }, } param_exclude = ( @@ -122,14 +189,22 @@ annotation_map = { xrefs = set() -def debug_this(s, n): - o = n if isinstance(n, str) else n.toprettyxml(indent=' ', newl='\n') - name = '' if isinstance(n, str) else n.nodeName - if s in o: +# Raises an error with details about `o`, if `cond` is in object `o`, +# or if `cond()` is callable and returns True. +def debug_this(cond, o): + name = '' + if not isinstance(o, str): + try: + name = o.nodeName + o = o.toprettyxml(indent=' ', newl='\n') + except Exception: + pass + if ((callable(cond) and cond()) + or (not callable(cond) and cond) + or (not callable(cond) and cond in o)): raise RuntimeError('xxx: {}\n{}'.format(name, o)) -# XML Parsing Utilities {{{ def find_first(parent, name): """Finds the first matching node within parent.""" sub = parent.getElementsByTagName(name) @@ -138,20 +213,27 @@ def find_first(parent, name): return sub[0] -def get_children(parent, name): - """Yield matching child nodes within parent.""" +def iter_children(parent, name): + """Yields matching child nodes within parent.""" for child in parent.childNodes: if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: yield child def get_child(parent, name): - """Get the first matching child node.""" - for child in get_children(parent, name): + """Gets the first matching child node.""" + for child in iter_children(parent, name): return child return None +def self_or_child(n): + """Gets the first child node, or self.""" + if len(n.childNodes) == 0: + return n + return n.childNodes[0] + + def clean_text(text): """Cleans text. @@ -172,18 +254,21 @@ def is_blank(text): return '' == clean_lines(text) -def get_text(parent, preformatted=False): - """Combine all text in a node.""" - if parent.nodeType == parent.TEXT_NODE: - return parent.data - - out = '' - for node in parent.childNodes: +def get_text(n, preformatted=False): + """Recursively concatenates all text in a node tree.""" + text = '' + if n.nodeType == n.TEXT_NODE: + return n.data + if n.nodeName == 'computeroutput': + for node in n.childNodes: + text += get_text(node) + return '`{}` '.format(text) + for node in n.childNodes: if node.nodeType == node.TEXT_NODE: - out += node.data if preformatted else clean_text(node.data) + text += node.data if preformatted else clean_text(node.data) elif node.nodeType == node.ELEMENT_NODE: - out += ' ' + get_text(node, preformatted) - return out + text += ' ' + get_text(node, preformatted) + return text # Gets the length of the last line in `text`, excluding newline ("\n") char. @@ -203,6 +288,8 @@ def len_lastline_withoutindent(text, indent): # Returns True if node `n` contains only inline (not block-level) elements. def is_inline(n): + # if len(n.childNodes) == 0: + # return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput' for c in n.childNodes: if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput': return False @@ -253,83 +340,78 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None): return result -def has_nonexcluded_params(nodes): - """Returns true if any of the given <parameterlist> elements has at least - one non-excluded item.""" - for n in nodes: - if render_params(n) != '': - return True +def max_name(names): + if len(names) == 0: + return 0 + return max(len(name) for name in names) -def render_params(parent, width=62): - """Renders Doxygen <parameterlist> tag as Vim help text.""" - name_length = 0 - items = [] +def update_params_map(parent, ret_map, width=62): + """Updates `ret_map` with name:desc key-value pairs extracted + from Doxygen XML node `parent`. + """ + params = collections.OrderedDict() for node in parent.childNodes: if node.nodeType == node.TEXT_NODE: continue - name_node = find_first(node, 'parametername') if name_node.getAttribute('direction') == 'out': continue - name = get_text(name_node) if name in param_exclude: continue - - name = '{%s}' % name - name_length = max(name_length, len(name) + 2) - items.append((name.strip(), node)) - - out = '' - for name, node in items: - name = ' {}'.format(name.ljust(name_length)) - + params[name.strip()] = node + max_name_len = max_name(params.keys()) + 8 + # `ret_map` is a name:desc map. + for name, node in params.items(): desc = '' desc_node = get_child(node, 'parameterdescription') if desc_node: - desc = parse_parblock(desc_node, width=width, - indent=(' ' * len(name))) - - out += '{}{}\n'.format(name, desc) - return out.rstrip() + desc = fmt_node_as_vimhelp( + desc_node, width=width, indent=(' ' * max_name_len)) + ret_map[name] = desc + return ret_map def render_node(n, text, prefix='', indent='', width=62): """Renders a node as Vim help text, recursively traversing all descendants.""" + global fmt_vimhelp + global has_seen_preformatted + + def ind(s): + return s if fmt_vimhelp else '' + text = '' # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) # text += (int(not space_preceding) * ' ') - if n.nodeType == n.TEXT_NODE: - # `prefix` is NOT sent to doc_wrap, it was already handled by now. - text += doc_wrap(n.data, indent=indent, width=width) - elif n.nodeName == 'computeroutput': - text += ' `{}` '.format(get_text(n)) - elif n.nodeName == 'preformatted': + if n.nodeName == 'preformatted': o = get_text(n, preformatted=True) ensure_nl = '' if o[-1] == '\n' else '\n' - text += ' >{}{}\n<'.format(ensure_nl, o) + text += '>{}{}\n<'.format(ensure_nl, o) + elif is_inline(n): - for c in n.childNodes: - text += render_node(c, text) - text = doc_wrap(text, indent=indent, width=width) + text = doc_wrap(get_text(n), indent=indent, width=width) elif n.nodeName == 'verbatim': # TODO: currently we don't use this. The "[verbatim]" hint is there as # a reminder that we must decide how to format this if we do use it. text += ' [verbatim] {}'.format(get_text(n)) elif n.nodeName == 'listitem': for c in n.childNodes: - text += ( - indent - + prefix - + render_node(c, text, indent=indent + (' ' * len(prefix)), width=width) + result = render_node( + c, + text, + indent=indent + (' ' * len(prefix)), + width=width ) + + if is_blank(result): + continue + + text += indent + prefix + result elif n.nodeName in ('para', 'heading'): for c in n.childNodes: text += render_node(c, text, indent=indent, width=width) - if is_inline(n): - text = doc_wrap(text, indent=indent, width=width) elif n.nodeName == 'itemizedlist': for c in n.childNodes: text += '{}\n'.format(render_node(c, text, prefix='• ', @@ -355,23 +437,33 @@ def render_node(n, text, prefix='', indent='', width=62): text += '\n' elif (n.nodeName == 'simplesect' and n.getAttribute('kind') in ('return', 'see')): - text += ' ' + text += ind(' ') for c in n.childNodes: text += render_node(c, text, indent=' ', width=width) else: raise RuntimeError('unhandled node type: {}\n{}'.format( n.nodeName, n.toprettyxml(indent=' ', newl='\n'))) + return text -def render_para(parent, indent='', width=62): - """Renders Doxygen <para> containing arbitrary nodes. +def para_as_map(parent, indent='', width=62): + """Extracts a Doxygen XML <para> node to a map. - NB: Blank lines in a docstring manifest as <para> tags. + Keys: + 'text': Text from this <para> element + 'params': <parameterlist> map + 'return': List of @return strings + 'seealso': List of @see strings + 'xrefs': ? """ - if is_inline(parent): - return clean_lines(doc_wrap(render_node(parent, ''), - indent=indent, width=width).strip()) + chunks = { + 'text': '', + 'params': collections.OrderedDict(), + 'return': [], + 'seealso': [], + 'xrefs': [] + } # Ordered dict of ordered lists. groups = collections.OrderedDict([ @@ -386,75 +478,125 @@ def render_para(parent, indent='', width=62): text = '' kind = '' last = '' - for child in parent.childNodes: - if child.nodeName == 'parameterlist': - groups['params'].append(child) - elif child.nodeName == 'xrefsect': - groups['xrefs'].append(child) - elif child.nodeName == 'simplesect': - last = kind - kind = child.getAttribute('kind') - if kind == 'return' or (kind == 'note' and last == 'return'): - groups['return'].append(child) - elif kind == 'see': - groups['seealso'].append(child) - elif kind in ('note', 'warning'): - text += render_node(child, text, indent=indent, width=width) + if is_inline(parent): + # Flatten inline text from a tree of non-block nodes. + text = doc_wrap(render_node(parent, ""), indent=indent, width=width) + else: + prev = None # Previous node + for child in parent.childNodes: + if child.nodeName == 'parameterlist': + groups['params'].append(child) + elif child.nodeName == 'xrefsect': + groups['xrefs'].append(child) + elif child.nodeName == 'simplesect': + last = kind + kind = child.getAttribute('kind') + if kind == 'return' or (kind == 'note' and last == 'return'): + groups['return'].append(child) + elif kind == 'see': + groups['seealso'].append(child) + elif kind in ('note', 'warning'): + text += render_node(child, text, indent=indent, width=width) + else: + raise RuntimeError('unhandled simplesect: {}\n{}'.format( + child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) else: - raise RuntimeError('unhandled simplesect: {}\n{}'.format( - child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) - else: - text += render_node(child, text, indent=indent, width=width) + if (prev is not None + and is_inline(self_or_child(prev)) + and is_inline(self_or_child(child)) + and '' != get_text(self_or_child(child)).strip() + and ' ' != text[-1]): + text += ' ' + + text += render_node(child, text, indent=indent, width=width) + prev = child + + chunks['text'] += text - chunks = [text] - # Generate text from the gathered items. - if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']): - chunks.append('\nParameters: ~') + # Generate map from the gathered items. + if len(groups['params']) > 0: for child in groups['params']: - chunks.append(render_params(child, width=width)) - if len(groups['return']) > 0: - chunks.append('\nReturn: ~') - for child in groups['return']: - chunks.append(render_node( - child, chunks[-1][-1], indent=indent, width=width)) - if len(groups['seealso']) > 0: - chunks.append('\nSee also: ~') - for child in groups['seealso']: - chunks.append(render_node( - child, chunks[-1][-1], indent=indent, width=width)) + update_params_map(child, ret_map=chunks['params'], width=width) + for child in groups['return']: + chunks['return'].append(render_node( + child, '', indent=indent, width=width)) + for child in groups['seealso']: + chunks['seealso'].append(render_node( + child, '', indent=indent, width=width)) for child in groups['xrefs']: - title = get_text(get_child(child, 'xreftitle')) + # XXX: Add a space (or any char) to `title` here, otherwise xrefs + # ("Deprecated" section) acts very weird... + title = get_text(get_child(child, 'xreftitle')) + ' ' xrefs.add(title) - xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width) - chunks.append(doc_wrap(xrefdesc, prefix='{}: '.format(title), - width=width) + '\n') + xrefdesc = get_text(get_child(child, 'xrefdescription')) + chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title), + width=width) + '\n') - return clean_lines('\n'.join(chunks).strip()) + return chunks -def parse_parblock(parent, prefix='', width=62, indent=''): - """Renders a nested block of <para> tags as Vim help text.""" - paragraphs = [] - for child in parent.childNodes: - paragraphs.append(render_para(child, width=width, indent=indent)) - paragraphs.append('') - return clean_lines('\n'.join(paragraphs).strip()) -# }}} - - -def parse_source_xml(filename, mode): - """Collects API functions. +def fmt_node_as_vimhelp(parent, width=62, indent=''): + """Renders (nested) Doxygen <para> nodes as Vim :help text. - Returns two strings: - 1. API functions - 2. Deprecated API functions + NB: Blank lines in a docstring manifest as <para> tags. + """ + rendered_blocks = [] + + def fmt_param_doc(m): + """Renders a params map as Vim :help text.""" + max_name_len = max_name(m.keys()) + 4 + out = '' + for name, desc in m.items(): + name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + out += '{}{}\n'.format(name, desc) + return out.rstrip() + + def has_nonexcluded_params(m): + """Returns true if any of the given params has at least + one non-excluded item.""" + if fmt_param_doc(m) != '': + return True - Caller decides what to do with the deprecated documentation. + for child in parent.childNodes: + para = para_as_map(child, indent, width) + + # Generate text from the gathered items. + chunks = [para['text']] + if len(para['params']) > 0 and has_nonexcluded_params(para['params']): + chunks.append('\nParameters: ~') + chunks.append(fmt_param_doc(para['params'])) + if len(para['return']) > 0: + chunks.append('\nReturn: ~') + for s in para['return']: + chunks.append(s) + if len(para['seealso']) > 0: + chunks.append('\nSee also: ~') + for s in para['seealso']: + chunks.append(s) + for s in para['xrefs']: + chunks.append(s) + + rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) + rendered_blocks.append('') + + return clean_lines('\n'.join(rendered_blocks).strip()) + + +def extract_from_xml(filename, target, width): + """Extracts Doxygen info as maps without formatting the text. + + Returns two maps: + 1. Functions + 2. Deprecated functions + + The `fmt_vimhelp` global controls some special cases for use by + fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) """ global xrefs - xrefs = set() - functions = [] - deprecated_functions = [] + global fmt_vimhelp + xrefs.clear() + fns = {} # Map of func_name:docstring. + deprecated_fns = {} # Map of func_name:docstring. dom = minidom.parse(filename) compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) @@ -489,31 +631,33 @@ def parse_source_xml(filename, mode): annotations = filter(None, map(lambda x: annotation_map.get(x), annotations.split())) - if mode == 'lua': - fstem = compoundname.split('.')[0] - fstem = CONFIG[mode]['module_override'].get(fstem, fstem) - vimtag = '*{}.{}()*'.format(fstem, name) + if not fmt_vimhelp: + pass else: - vimtag = '*{}()*'.format(name) + fstem = '?' + if '.' in compoundname: + fstem = compoundname.split('.')[0] + fstem = CONFIG[target]['module_override'].get(fstem, fstem) + vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name) params = [] type_length = 0 - for param in get_children(member, 'param'): + for param in iter_children(member, 'param'): param_type = get_text(get_child(param, 'type')).strip() param_name = '' declname = get_child(param, 'declname') if declname: param_name = get_text(declname).strip() - elif mode == 'lua': - # that's how it comes out of lua2dox + elif CONFIG[target]['mode'] == 'lua': + # XXX: this is what lua2dox gives us... param_name = param_type param_type = '' if param_name in param_exclude: continue - if param_type.endswith('*'): + if fmt_vimhelp and param_type.endswith('*'): param_type = param_type.strip('* ') param_name = '*' + param_name type_length = max(type_length, len(param_type)) @@ -521,41 +665,106 @@ def parse_source_xml(filename, mode): c_args = [] for param_type, param_name in params: - c_args.append(' ' + ( + c_args.append((' ' if fmt_vimhelp else '') + ( '%s %s' % (param_type.ljust(type_length), param_name)).strip()) - c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, - ',\n'.join(c_args)), - ' ') - prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params if a[0] not in ('void', 'Error')) + if not fmt_vimhelp: + c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) + signature = prefix + suffix + else: + c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, + ',\n'.join(c_args)), + ' ') - # Minimum 8 chars between signature and vimtag - lhs = (text_width - 8) - len(prefix) + # Minimum 8 chars between signature and vimtag + lhs = (width - 8) - len(vimtag) - if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(text_width) + '\n' - signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, - func=True) - else: - signature = prefix + suffix - signature += vimtag.rjust(text_width - len(signature)) + if len(prefix) + len(suffix) > lhs: + signature = vimtag.rjust(width) + '\n' + signature += doc_wrap(suffix, width=width-8, prefix=prefix, + func=True) + else: + signature = prefix + suffix + signature += vimtag.rjust(width - len(signature)) + + paras = [] + brief_desc = find_first(member, 'briefdescription') + if brief_desc: + for child in brief_desc.childNodes: + paras.append(para_as_map(child)) - doc = '' desc = find_first(member, 'detaileddescription') if desc: - doc = parse_parblock(desc) + for child in desc.childNodes: + paras.append(para_as_map(child)) if DEBUG: print(textwrap.indent( re.sub(r'\n\s*\n+', '\n', desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + fn = { + 'annotations': list(annotations), + 'signature': signature, + 'parameters': params, + 'parameters_doc': collections.OrderedDict(), + 'doc': [], + 'return': [], + 'seealso': [], + } + if fmt_vimhelp: + fn['desc_node'] = desc # HACK :( + + for m in paras: + if 'text' in m: + if not m['text'] == '': + fn['doc'].append(m['text']) + if 'params' in m: + # Merge OrderedDicts. + fn['parameters_doc'].update(m['params']) + if 'return' in m and len(m['return']) > 0: + fn['return'] += m['return'] + if 'seealso' in m and len(m['seealso']) > 0: + fn['seealso'] += m['seealso'] + + if INCLUDE_C_DECL: + fn['c_decl'] = c_decl + + if 'Deprecated' in str(xrefs): + deprecated_fns[name] = fn + elif name.startswith(CONFIG[target]['fn_name_prefix']): + fns[name] = fn + + xrefs.clear() + + fns = collections.OrderedDict(sorted(fns.items())) + deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items())) + return (fns, deprecated_fns) + + +def fmt_doxygen_xml_as_vimhelp(filename, target): + """Entrypoint for generating Vim :help from from Doxygen XML. + + Returns 3 items: + 1. Vim help text for functions found in `filename`. + 2. Vim help text for deprecated functions. + """ + global fmt_vimhelp + fmt_vimhelp = True + fns_txt = {} # Map of func_name:vim-help-text. + deprecated_fns_txt = {} # Map of func_name:vim-help-text. + fns, _ = extract_from_xml(filename, target, width=text_width) + + for name, fn in fns.items(): + # Generate Vim :help for parameters. + if fn['desc_node']: + doc = fmt_node_as_vimhelp(fn['desc_node']) if not doc: doc = 'TODO: Documentation' - annotations = '\n'.join(annotations) + annotations = '\n'.join(fn['annotations']) if annotations: annotations = ('\n\nAttributes: ~\n' + textwrap.indent(annotations, ' ')) @@ -567,21 +776,51 @@ def parse_source_xml(filename, mode): if INCLUDE_C_DECL: doc += '\n\nC Declaration: ~\n>\n' - doc += c_decl + doc += fn['c_decl'] doc += '\n<' - func_doc = signature + '\n' + func_doc = fn['signature'] + '\n' func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) + + # Verbatim handling. func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) + split_lines = func_doc.split('\n') + start = 0 + while True: + try: + start = split_lines.index('>', start) + except ValueError: + break + + try: + end = split_lines.index('<', start) + except ValueError: + break + + split_lines[start + 1:end] = [ + (' ' + x).rstrip() + for x in textwrap.dedent( + "\n".join( + split_lines[start+1:end] + ) + ).split("\n") + ] + + start = end + + func_doc = "\n".join(split_lines) + if 'Deprecated' in xrefs: - deprecated_functions.append(func_doc) - elif name.startswith(CONFIG[mode]['func_name_prefix']): - functions.append(func_doc) + deprecated_fns_txt[name] = func_doc + elif name.startswith(CONFIG[target]['fn_name_prefix']): + fns_txt[name] = func_doc xrefs.clear() - return '\n\n'.join(functions), '\n\n'.join(deprecated_functions) + fmt_vimhelp = False + return ('\n\n'.join(list(fns_txt.values())), + '\n\n'.join(list(deprecated_fns_txt.values()))) def delete_lines_below(filename, tokenstr): @@ -590,33 +829,54 @@ def delete_lines_below(filename, tokenstr): """ lines = open(filename).readlines() i = 0 + found = False for i, line in enumerate(lines, 1): if tokenstr in line: + found = True break + if not found: + raise RuntimeError(f'not found: "{tokenstr}"') i = max(0, i - 2) with open(filename, 'wt') as fp: fp.writelines(lines[0:i]) -def gen_docs(config): - """Generate documentation. +def main(config): + """Generates: + + 1. Vim :help docs + 2. *.mpack files for use by API clients Doxygen is called and configured through stdin. """ - for mode in CONFIG: - output_dir = out_dir.format(mode=mode) - p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) + for target in CONFIG: + if TARGET is not None and target != TARGET: + continue + mpack_file = os.path.join( + base_dir, 'runtime', 'doc', + CONFIG[target]['filename'].replace('.txt', '.mpack')) + if os.path.exists(mpack_file): + os.remove(mpack_file) + + output_dir = out_dir.format(target=target) + p = subprocess.Popen( + ['doxygen', '-'], + stdin=subprocess.PIPE, + # silence warnings + # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found + stderr=(subprocess.STDOUT if DEBUG else subprocess.DEVNULL)) p.communicate( config.format( - input=CONFIG[mode]['files'], + input=CONFIG[target]['files'], output=output_dir, filter=filter_cmd, - file_patterns=CONFIG[mode]['file_patterns']) + file_patterns=CONFIG[target]['file_patterns']) .encode('utf8') ) if p.returncode: sys.exit(p.returncode) + fn_map_full = {} # Collects all functions as each module is processed. sections = {} intros = {} sep = '=' * text_width @@ -633,11 +893,21 @@ def gen_docs(config): groupxml = os.path.join(base, '%s.xml' % compound.getAttribute('refid')) - desc = find_first(minidom.parse(groupxml), 'detaileddescription') + group_parsed = minidom.parse(groupxml) + doc_list = [] + brief_desc = find_first(group_parsed, 'briefdescription') + if brief_desc: + for child in brief_desc.childNodes: + doc_list.append(fmt_node_as_vimhelp(child)) + + desc = find_first(group_parsed, 'detaileddescription') if desc: - doc = parse_parblock(desc) + doc = fmt_node_as_vimhelp(desc) + if doc: - intros[groupname] = doc + doc_list.append(doc) + + intros[groupname] = "\n".join(doc_list) for compound in dom.getElementsByTagName('compound'): if compound.getAttribute('kind') != 'file': @@ -645,59 +915,54 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): - functions, deprecated = parse_source_xml( - os.path.join(base, '%s.xml' % - compound.getAttribute('refid')), mode) - - if not functions and not deprecated: + # Extract unformatted (*.mpack). + fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format( + compound.getAttribute('refid'))), target, width=9999) + # Extract formatted (:help). + functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( + os.path.join(base, '{}.xml'.format( + compound.getAttribute('refid'))), target) + + if not functions_text and not deprecated_text: continue - - if functions or deprecated: - name = os.path.splitext(os.path.basename(filename))[0] - if name == 'ui': - name = name.upper() - else: - name = name.title() - + else: + name = os.path.splitext( + os.path.basename(filename))[0].lower() + sectname = name.upper() if name == 'ui' else name.title() doc = '' - - intro = intros.get('api-%s' % name.lower()) + intro = intros.get(f'api-{name}') if intro: doc += '\n\n' + intro - if functions: - doc += '\n\n' + functions + if functions_text: + doc += '\n\n' + functions_text - if INCLUDE_DEPRECATED and deprecated: - doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name - doc += deprecated + if INCLUDE_DEPRECATED and deprecated_text: + doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n' + doc += deprecated_text if doc: filename = os.path.basename(filename) - name = CONFIG[mode]['section_name'].get(filename, name) - - if mode == 'lua': - title = 'Lua module: {}'.format(name.lower()) - helptag = '*lua-{}*'.format(name.lower()) - else: - title = '{} Functions'.format(name) - helptag = '*api-{}*'.format(name.lower()) + sectname = CONFIG[target]['section_name'].get( + filename, sectname) + title = CONFIG[target]['section_fmt'](sectname) + helptag = CONFIG[target]['helptag_fmt'](sectname) sections[filename] = (title, helptag, doc) + fn_map_full.update(fn_map) - if not sections: - return + assert sections + if len(sections) > len(CONFIG[target]['section_order']): + raise RuntimeError( + 'found new modules "{}"; update the "section_order" map'.format( + set(sections).difference(CONFIG[target]['section_order']))) docs = '' i = 0 - for filename in CONFIG[mode]['section_order']: - if filename not in sections: - raise RuntimeError( - 'found new module "{}"; update the "section_order" map'.format( - filename)) + for filename in CONFIG[target]['section_order']: title, helptag, section_doc = sections.pop(filename) i += 1 - if filename not in CONFIG[mode]['append_only']: + if filename not in CONFIG[target]['append_only']: docs += sep docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) @@ -708,12 +973,16 @@ def gen_docs(config): docs += ' vim:tw=78:ts=8:ft=help:norl:\n' doc_file = os.path.join(base_dir, 'runtime', 'doc', - CONFIG[mode]['filename']) + CONFIG[target]['filename']) - delete_lines_below(doc_file, CONFIG[mode]['section_start_token']) + delete_lines_below(doc_file, CONFIG[target]['section_start_token']) with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) + fn_map_full = collections.OrderedDict(sorted(fn_map_full.items())) + with open(mpack_file, 'wb') as fp: + fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) + shutil.rmtree(output_dir) @@ -732,47 +1001,45 @@ def filter_source(filename): fp.read(), flags=re.M)) -# Doxygen Config {{{ -Doxyfile = ''' -OUTPUT_DIRECTORY = {output} -INPUT = {input} -INPUT_ENCODING = UTF-8 -FILE_PATTERNS = {file_patterns} -RECURSIVE = YES -INPUT_FILTER = "{filter}" -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = */private/* -EXCLUDE_SYMBOLS = -EXTENSION_MAPPING = lua=C -EXTRACT_PRIVATE = NO - -GENERATE_HTML = NO -GENERATE_DOCSET = NO -GENERATE_HTMLHELP = NO -GENERATE_QHP = NO -GENERATE_TREEVIEW = NO -GENERATE_LATEX = NO -GENERATE_RTF = NO -GENERATE_MAN = NO -GENERATE_DOCBOOK = NO -GENERATE_AUTOGEN_DEF = NO - -GENERATE_XML = YES -XML_OUTPUT = xml -XML_PROGRAMLISTING = NO - -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = YES -EXPAND_ONLY_PREDEF = NO -MARKDOWN_SUPPORT = YES -''' -# }}} +Doxyfile = textwrap.dedent(''' + OUTPUT_DIRECTORY = {output} + INPUT = {input} + INPUT_ENCODING = UTF-8 + FILE_PATTERNS = {file_patterns} + RECURSIVE = YES + INPUT_FILTER = "{filter}" + EXCLUDE = + EXCLUDE_SYMLINKS = NO + EXCLUDE_PATTERNS = */private/* + EXCLUDE_SYMBOLS = + EXTENSION_MAPPING = lua=C + EXTRACT_PRIVATE = NO + + GENERATE_HTML = NO + GENERATE_DOCSET = NO + GENERATE_HTMLHELP = NO + GENERATE_QHP = NO + GENERATE_TREEVIEW = NO + GENERATE_LATEX = NO + GENERATE_RTF = NO + GENERATE_MAN = NO + GENERATE_DOCBOOK = NO + GENERATE_AUTOGEN_DEF = NO + + GENERATE_XML = YES + XML_OUTPUT = xml + XML_PROGRAMLISTING = NO + + ENABLE_PREPROCESSING = YES + MACRO_EXPANSION = YES + EXPAND_ONLY_PREDEF = NO + MARKDOWN_SUPPORT = YES +''') if __name__ == "__main__": if len(sys.argv) > 1: filter_source(sys.argv[1]) else: - gen_docs(Doxyfile) + main(Doxyfile) -# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : +# vim: set ft=python ts=4 sw=4 tw=79 et : |