diff options
Diffstat (limited to 'scripts/gen_vimdoc.py')
| -rwxr-xr-x | scripts/gen_vimdoc.py | 416 |
1 files changed, 266 insertions, 150 deletions
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 4d71d5e15e..876d46c18f 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -1,5 +1,18 @@ #!/usr/bin/env python3 -"""Generates Nvim help docs from C docstrings, by parsing Doxygen XML. +"""Generates Nvim help docs from C/Lua docstrings, using Doxygen. + +Also generates *.mpack files. To inspect the *.mpack structure: + + :new | put=json_encode(msgpackparse(readfile('runtime/doc/api.mpack'))) + +Flow: + gen_docs + extract_from_xml + fmt_node_as_vimhelp + fmt_params_map_as_vimhelp + render_node + para_as_map + render_node This would be easier using lxml and XSLT, but: @@ -9,25 +22,22 @@ This would be easier using lxml and XSLT, but: 2. I wouldn't know how to deal with nested indentation in <para> tags using XSLT. -Each function documentation is formatted with the following rules: +Each function :help block is formatted as follows: - - Maximum width of 78 characters (`text_width`). - - Spaces for indentation. - - Function signature and helptag are on the same line. - - Helptag is right aligned. + - Max width of 78 columns (`text_width`). + - Indent with spaces (not tabs). + - Indent of 16 columns for body text. + - Function signature and helptag (right-aligned) on the same line. - Signature and helptag must have a minimum of 8 spaces between them. - - If the signature is too long, it is placed on the line after the - helptag. The signature wraps at `text_width - 8` characters with - subsequent lines indented to the open parenthesis. - - Documentation body will be indented by 16 spaces. + - If the signature is too long, it is placed on the line after the helptag. + Signature wraps at `text_width - 8` characters with subsequent + lines indented to the open parenthesis. - Subsection bodies are indented an additional 4 spaces. - - Documentation body consists of the function description, parameter details, - return description, and C declaration. + - Body consists of function description, parameters, return description, and + C declaration (`INCLUDE_C_DECL`). - Parameters are omitted for the `void` and `Error *` types, or if the parameter is marked as [out]. - Each function documentation is separated by a single line. - -The C declaration is added to the end to show actual argument types. """ import os import re @@ -123,14 +133,21 @@ annotation_map = { xrefs = set() -def debug_this(s, n): - o = n if isinstance(n, str) else n.toprettyxml(indent=' ', newl='\n') - name = '' if isinstance(n, str) else n.nodeName - if s in o: +# Raises an error with details about `o`, if `cond` is in object `o`, +# or if `cond()` is callable and returns True. +def debug_this(cond, o): + name = '' + if not isinstance(o, str): + try: + name = o.nodeName + o = o.toprettyxml(indent=' ', newl='\n') + except: + pass + if ((callable(cond) and cond()) + or (not callable(cond) and cond in o)): raise RuntimeError('xxx: {}\n{}'.format(name, o)) -# XML Parsing Utilities {{{ def find_first(parent, name): """Finds the first matching node within parent.""" sub = parent.getElementsByTagName(name) @@ -254,44 +271,42 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None): return result -def has_nonexcluded_params(nodes): - """Returns true if any of the given <parameterlist> elements has at least - one non-excluded item.""" - for n in nodes: - if render_params(n) != '': - return True -def render_params(parent, width=62): - """Renders Doxygen <parameterlist> tag as Vim help text.""" - name_length = 0 - items = [] +def update_params_map(parent, ret_map, width=62): + """Updates `ret_map` with name:desc key-value pairs extracted + from Doxygen XML node `parent`. + """ + params = [] for node in parent.childNodes: if node.nodeType == node.TEXT_NODE: continue - name_node = find_first(node, 'parametername') if name_node.getAttribute('direction') == 'out': continue - name = get_text(name_node) if name in param_exclude: continue - - name = '{%s}' % name - name_length = max(name_length, len(name) + 2) - items.append((name.strip(), node)) - - out = '' - for name, node in items: - name = ' {}'.format(name.ljust(name_length)) - + params.append((name.strip(), node)) + # `ret_map` is a name:desc map. + for name, node in params: desc = '' desc_node = get_child(node, 'parameterdescription') if desc_node: - desc = parse_parblock(desc_node, width=width, + desc = fmt_node_as_vimhelp(desc_node, width=width, indent=(' ' * len(name))) + ret_map[name] = desc + return ret_map + +def fmt_params_map_as_vimhelp(m, width=62): + """Renders a params map as Vim :help text.""" + max_name_len = 0 + for name, desc in m.items(): + max_name_len = max(max_name_len, len(name) + 4) + out = '' + for name, desc in m.items(): + name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) out += '{}{}\n'.format(name, desc) return out.rstrip() @@ -365,14 +380,27 @@ def render_node(n, text, prefix='', indent='', width=62): return text -def render_para(parent, indent='', width=62): - """Renders Doxygen <para> containing arbitrary nodes. +def para_as_map(parent, indent='', width=62): + """Extracts a Doxygen XML <para> node to a map. - NB: Blank lines in a docstring manifest as <para> tags. + Keys: + 'text': Text from this <para> element + 'params': <parameterlist> map + 'return': List of @return strings + 'seealso': List of @see strings + 'xrefs': ? """ + chunks = { + 'text': '', + 'params': collections.OrderedDict(), + 'return': [], + 'seealso': [], + 'xrefs': [] + } + if is_inline(parent): - return clean_lines(doc_wrap(render_node(parent, ''), - indent=indent, width=width).strip()) + chunks['text'] = clean_lines(doc_wrap(render_node(parent, ''), + indent=indent, width=width).strip()) # Ordered dict of ordered lists. groups = collections.OrderedDict([ @@ -407,55 +435,80 @@ def render_para(parent, indent='', width=62): else: text += render_node(child, text, indent=indent, width=width) - chunks = [text] - # Generate text from the gathered items. - if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']): - chunks.append('\nParameters: ~') + chunks['text'] = text + + # Generate map from the gathered items. + if len(groups['params']) > 0: for child in groups['params']: - chunks.append(render_params(child, width=width)) - if len(groups['return']) > 0: - chunks.append('\nReturn: ~') - for child in groups['return']: - chunks.append(render_node( - child, chunks[-1][-1], indent=indent, width=width)) - if len(groups['seealso']) > 0: - chunks.append('\nSee also: ~') - for child in groups['seealso']: - chunks.append(render_node( - child, chunks[-1][-1], indent=indent, width=width)) + update_params_map(child, ret_map=chunks['params'], width=width) + for child in groups['return']: + chunks['return'].append(render_node( + child, '', indent=indent, width=width).lstrip()) + for child in groups['seealso']: + chunks['seealso'].append(render_node( + child, '', indent=indent, width=width)) for child in groups['xrefs']: - title = get_text(get_child(child, 'xreftitle')) + # XXX: Add a space (or any char) to `title` here, otherwise xrefs + # ("Deprecated" section) acts very weird... + title = get_text(get_child(child, 'xreftitle')) + ' ' xrefs.add(title) - xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width) - chunks.append(doc_wrap(xrefdesc, prefix='{}: '.format(title), - width=width) + '\n') - - return clean_lines('\n'.join(chunks).strip()) - - -def parse_parblock(parent, prefix='', width=62, indent=''): - """Renders a nested block of <para> tags as Vim help text.""" - paragraphs = [] - for child in parent.childNodes: - paragraphs.append(render_para(child, width=width, indent=indent)) - paragraphs.append('') - return clean_lines('\n'.join(paragraphs).strip()) -# }}} + xrefdesc = get_text(get_child(child, 'xrefdescription')) + chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title), + width=width) + '\n') + return chunks -def parse_source_xml(filename, mode): - """Collects API functions. - Returns two strings: - 1. API functions - 2. Deprecated API functions +def fmt_node_as_vimhelp(parent, width=62, indent=''): + """Renders (nested) Doxygen <para> nodes as Vim :help text. - Caller decides what to do with the deprecated documentation. + NB: Blank lines in a docstring manifest as <para> tags. + """ + rendered_blocks = [] + for child in parent.childNodes: + para = para_as_map(child, indent, width) + + def has_nonexcluded_params(m): + """Returns true if any of the given params has at least + one non-excluded item.""" + if fmt_params_map_as_vimhelp(m) != '': + return True + + # Generate text from the gathered items. + chunks = [para['text']] + if len(para['params']) > 0 and has_nonexcluded_params(para['params']): + chunks.append('\nParameters: ~') + chunks.append(fmt_params_map_as_vimhelp(para['params'], width=width)) + if len(para['return']) > 0: + chunks.append('\nReturn: ~') + for s in para['return']: + chunks.append(s) + if len(para['seealso']) > 0: + chunks.append('\nSee also: ~') + for s in para['seealso']: + chunks.append(s) + for s in para['xrefs']: + chunks.append(s) + + rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) + rendered_blocks.append('') + return clean_lines('\n'.join(rendered_blocks).strip()) + + +def extract_from_xml(filename, mode, fmt_vimhelp): + """Extracts Doxygen info as maps without formatting the text. + + Returns two maps: + 1. Functions + 2. Deprecated functions + + The `fmt_vimhelp` parameter controls some special cases for use by + fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) """ global xrefs - xrefs = set() + xrefs.clear() functions = {} # Map of func_name:docstring. - deprecated_functions = [] + deprecated_functions = {} # Map of func_name:docstring. dom = minidom.parse(filename) compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) @@ -490,7 +543,9 @@ def parse_source_xml(filename, mode): annotations = filter(None, map(lambda x: annotation_map.get(x), annotations.split())) - if mode == 'lua': + if not fmt_vimhelp: + pass + elif mode == 'lua': fstem = compoundname.split('.')[0] fstem = CONFIG[mode]['module_override'].get(fstem, fstem) vimtag = '*{}.{}()*'.format(fstem, name) @@ -514,7 +569,7 @@ def parse_source_xml(filename, mode): if param_name in param_exclude: continue - if param_type.endswith('*'): + if fmt_vimhelp and param_type.endswith('*'): param_type = param_type.strip('* ') param_name = '*' + param_name type_length = max(type_length, len(param_type)) @@ -522,41 +577,97 @@ def parse_source_xml(filename, mode): c_args = [] for param_type, param_name in params: - c_args.append(' ' + ( + c_args.append((' ' if fmt_vimhelp else '') + ( '%s %s' % (param_type.ljust(type_length), param_name)).strip()) - c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, - ',\n'.join(c_args)), - ' ') - prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params if a[0] not in ('void', 'Error')) + if not fmt_vimhelp: + c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) + signature = prefix + suffix + else: + c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, + ',\n'.join(c_args)), + ' ') - # Minimum 8 chars between signature and vimtag - lhs = (text_width - 8) - len(prefix) + # Minimum 8 chars between signature and vimtag + lhs = (text_width - 8) - len(prefix) - if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(text_width) + '\n' - signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, - func=True) - else: - signature = prefix + suffix - signature += vimtag.rjust(text_width - len(signature)) + if len(prefix) + len(suffix) > lhs: + signature = vimtag.rjust(text_width) + '\n' + signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, + func=True) + else: + signature = prefix + suffix + signature += vimtag.rjust(text_width - len(signature)) - doc = '' + paras = [] desc = find_first(member, 'detaileddescription') if desc: - doc = parse_parblock(desc) + for child in desc.childNodes: + paras.append(para_as_map(child)) #, width=width, indent=indent)) if DEBUG: print(textwrap.indent( re.sub(r'\n\s*\n+', '\n', desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + fn = { + 'annotations': list(annotations), + 'signature': signature, + 'parameters': params, + 'parameters_doc': collections.OrderedDict(), + 'doc': [], + 'return': [], + 'seealso': [], + } + if fmt_vimhelp: + fn['desc_node'] = desc # HACK :( + + for m in paras: + if 'text' in m: + if not m['text'] == '': + fn['doc'].append(m['text']) + if 'params' in m: + # Merge OrderedDicts. + fn['parameters_doc'].update(m['params']) + if 'return' in m and len(m['return']) > 0: + fn['return'] += m['return'] + if 'seealso' in m and len(m['seealso']) > 0: + fn['seealso'] += m['seealso'] + + if INCLUDE_C_DECL: + fn['c_decl'] = c_decl + + if 'Deprecated' in xrefs: + deprecated_functions[name] = fn + elif name.startswith(CONFIG[mode]['func_name_prefix']): + functions[name] = fn + + xrefs.clear() + + return (functions, deprecated_functions) + + +def fmt_doxygen_xml_as_vimhelp(filename, mode): + """Formats functions from doxygen XML into Vim :help format. + + Returns two strings: + 1. Functions in Vim :help format + 2. Deprecated functions (handled by caller, or ignored) + """ + functions = {} # Map of func_name:docstring. + deprecated_functions = {} # Map of func_name:docstring. + fns, deprecated_fns = extract_from_xml(filename, mode, True) + + for name, fn in fns.items(): + # Generate Vim :help for parameters. + if fn['desc_node']: + doc = fmt_node_as_vimhelp(fn['desc_node']) if not doc: doc = 'TODO: Documentation' - annotations = '\n'.join(annotations) + annotations = '\n'.join(fn['annotations']) if annotations: annotations = ('\n\nAttributes: ~\n' + textwrap.indent(annotations, ' ')) @@ -568,10 +679,10 @@ def parse_source_xml(filename, mode): if INCLUDE_C_DECL: doc += '\n\nC Declaration: ~\n>\n' - doc += c_decl + doc += fn['c_decl'] doc += '\n<' - func_doc = signature + '\n' + func_doc = fn['signature'] + '\n' func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) @@ -583,7 +694,7 @@ def parse_source_xml(filename, mode): xrefs.clear() return ('\n\n'.join(list(functions.values())), - '\n\n'.join(deprecated_functions), + '\n\n'.join(deprecated_fns), functions) @@ -602,15 +713,16 @@ def delete_lines_below(filename, tokenstr): def gen_docs(config): - """Generate documentation. + """Generate formatted Vim :help docs and unformatted *.mpack files for use + by API clients. Doxygen is called and configured through stdin. """ for mode in CONFIG: functions = {} # Map of func_name:docstring. mpack_file = os.path.join( - base_dir, 'runtime', 'doc', - CONFIG[mode]['filename'].replace('.txt', '.mpack')) + base_dir, 'runtime', 'doc', + CONFIG[mode]['filename'].replace('.txt', '.mpack')) if os.path.exists(mpack_file): os.remove(mpack_file) @@ -627,6 +739,7 @@ def gen_docs(config): if p.returncode: sys.exit(p.returncode) + fn_map_full = {} # Collects all functions as each module is processed. sections = {} intros = {} sep = '=' * text_width @@ -645,7 +758,7 @@ def gen_docs(config): desc = find_first(minidom.parse(groupxml), 'detaileddescription') if desc: - doc = parse_parblock(desc) + doc = fmt_node_as_vimhelp(desc) if doc: intros[groupname] = doc @@ -655,7 +768,10 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): - functions_text, deprecated_text, fns = parse_source_xml( + fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format( + compound.getAttribute('refid'))), mode, False) + + functions_text, deprecated_text, fns = fmt_doxygen_xml_as_vimhelp( os.path.join(base, '{}.xml'.format( compound.getAttribute('refid'))), mode) # Collect functions from all modules (for the current `mode`). @@ -694,6 +810,7 @@ def gen_docs(config): title = '{} Functions'.format(name) helptag = '*api-{}*'.format(name.lower()) sections[filename] = (title, helptag, doc) + fn_map_full.update(fn_map) if not sections: return @@ -724,8 +841,9 @@ def gen_docs(config): delete_lines_below(doc_file, CONFIG[mode]['section_start_token']) with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) + with open(mpack_file, 'wb') as fp: - fp.write(msgpack.packb(functions, use_bin_type=True)) + fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) shutil.rmtree(output_dir) @@ -745,42 +863,40 @@ def filter_source(filename): fp.read(), flags=re.M)) -# Doxygen Config {{{ -Doxyfile = ''' -OUTPUT_DIRECTORY = {output} -INPUT = {input} -INPUT_ENCODING = UTF-8 -FILE_PATTERNS = {file_patterns} -RECURSIVE = YES -INPUT_FILTER = "{filter}" -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = */private/* -EXCLUDE_SYMBOLS = -EXTENSION_MAPPING = lua=C -EXTRACT_PRIVATE = NO - -GENERATE_HTML = NO -GENERATE_DOCSET = NO -GENERATE_HTMLHELP = NO -GENERATE_QHP = NO -GENERATE_TREEVIEW = NO -GENERATE_LATEX = NO -GENERATE_RTF = NO -GENERATE_MAN = NO -GENERATE_DOCBOOK = NO -GENERATE_AUTOGEN_DEF = NO - -GENERATE_XML = YES -XML_OUTPUT = xml -XML_PROGRAMLISTING = NO - -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = YES -EXPAND_ONLY_PREDEF = NO -MARKDOWN_SUPPORT = YES -''' -# }}} +Doxyfile = textwrap.dedent(''' + OUTPUT_DIRECTORY = {output} + INPUT = {input} + INPUT_ENCODING = UTF-8 + FILE_PATTERNS = {file_patterns} + RECURSIVE = YES + INPUT_FILTER = "{filter}" + EXCLUDE = + EXCLUDE_SYMLINKS = NO + EXCLUDE_PATTERNS = */private/* + EXCLUDE_SYMBOLS = + EXTENSION_MAPPING = lua=C + EXTRACT_PRIVATE = NO + + GENERATE_HTML = NO + GENERATE_DOCSET = NO + GENERATE_HTMLHELP = NO + GENERATE_QHP = NO + GENERATE_TREEVIEW = NO + GENERATE_LATEX = NO + GENERATE_RTF = NO + GENERATE_MAN = NO + GENERATE_DOCBOOK = NO + GENERATE_AUTOGEN_DEF = NO + + GENERATE_XML = YES + XML_OUTPUT = xml + XML_PROGRAMLISTING = NO + + ENABLE_PREPROCESSING = YES + MACRO_EXPANSION = YES + EXPAND_ONLY_PREDEF = NO + MARKDOWN_SUPPORT = YES +''') if __name__ == "__main__": if len(sys.argv) > 1: @@ -788,4 +904,4 @@ if __name__ == "__main__": else: gen_docs(Doxyfile) -# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : +# vim: set ft=python ts=4 sw=4 tw=79 et : |