diff options
Diffstat (limited to 'scripts/gen_api_vimdoc.py')
| -rw-r--r-- | scripts/gen_api_vimdoc.py | 514 |
1 files changed, 514 insertions, 0 deletions
diff --git a/scripts/gen_api_vimdoc.py b/scripts/gen_api_vimdoc.py new file mode 100644 index 0000000000..d7165187f4 --- /dev/null +++ b/scripts/gen_api_vimdoc.py @@ -0,0 +1,514 @@ +#!/usr/bin/env python +"""Parses Doxygen XML output to generate Neovim's API documentation. + +This would be easier using lxml and XSLT, but: + + 1. This should avoid needing Python dependencies, especially ones that are + C modules that have library dependencies (lxml requires libxml and + libxslt). + 2. I wouldn't know how to deal with nested indentation in <para> tags using + XSLT. + +Each function documentation is formatted with the following rules: + + - Maximum width of 78 characters (`text_width`). + - Spaces for indentation. + - Function signature and helptag are on the same line. + - Helptag is right aligned. + - Signature and helptag must have a minimum of 8 spaces between them. + - If the signature is too long, it is placed on the line after the + helptag. The signature wraps at `text_width - 8` characters with + subsequent lines indented to the open parenthesis. + - Documentation body will be indented by 16 spaces. + - Subsection bodies are indented an additional 4 spaces. + - Documentation body consists of the function description, parameter details, + return description, and C declaration. + - Parameters are omitted for the `void` and `Error *` types, or if the + parameter is marked as [out]. + - Each function documentation is separated by a single line. + +The C declaration is added to the end to show actual argument types. +""" +import os +import re +import sys +import shutil +import textwrap +import subprocess + +from xml.dom import minidom + +# Text at the top of the doc file. +preamble = ''' +Note: This documentation is generated from Neovim's API source code. +''' + +doc_filename = 'api-funcs.txt' + +# Section name overrides. +section_name = { + 'vim.c': 'Global', +} + +# Section ordering. +section_order = ( + 'vim.c', + 'buffer.c', + 'window.c', + 'tabpage.c', + 'ui.c', +) + +param_exclude = ( + 'channel_id', +) + +text_width = 78 +script_path = os.path.abspath(__file__) +base_dir = os.path.dirname(os.path.dirname(script_path)) +src_dir = os.path.join(base_dir, 'src/nvim/api') +out_dir = os.path.join(base_dir, 'tmp/api_doc') +filter_cmd = '%s %s' % (sys.executable, script_path) +seen_funcs = set() + +# Tracks `xrefsect` titles. As of this writing, used only for separating +# deprecated functions. +xrefs = set() + + +# XML Parsing Utilities {{{ +def find_first(parent, name): + """Finds the first matching node within parent.""" + sub = parent.getElementsByTagName(name) + if not sub: + return None + return sub[0] + + +def get_children(parent, name): + """Yield matching child nodes within parent.""" + for child in parent.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: + yield child + + +def get_child(parent, name): + """Get the first matching child node.""" + for child in get_children(parent, name): + return child + return None + + +def clean_text(text): + """Cleans text. + + Only cleans superfluous whitespace at the moment. + """ + return ' '.join(text.split()).strip() + + +def clean_lines(text): + """Removes superfluous lines. + + The beginning and end of the string is trimmed. Empty lines are collapsed. + """ + return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text)) + + +def get_text(parent): + """Combine all text in a node.""" + if parent.nodeType == parent.TEXT_NODE: + return parent.data + + out = '' + for node in parent.childNodes: + if node.nodeType == node.TEXT_NODE: + out += clean_text(node.data) + elif node.nodeType == node.ELEMENT_NODE: + out += ' ' + get_text(node) + return out + + +def doc_wrap(text, prefix='', width=70, func=False): + """Wraps text to `width`. + + The first line is prefixed with `prefix`, and subsequent lines are aligned. + If `func` is True, only wrap at commas. + """ + if not width: + return text + + indent_space = ' ' * len(prefix) + + if func: + lines = [prefix] + for part in text.split(', '): + if part[-1] not in ');': + part += ', ' + if len(lines[-1]) + len(part) > width: + lines.append(indent_space) + lines[-1] += part + return '\n'.join(x.rstrip() for x in lines).rstrip() + + return '\n'.join(textwrap.wrap(text.strip(), width=width, + initial_indent=prefix, + subsequent_indent=indent_space)) + + +def parse_params(parent, width=62): + """Parse Doxygen `parameterlist`.""" + name_length = 0 + items = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + continue + + name_node = find_first(child, 'parametername') + if name_node.getAttribute('direction') == 'out': + continue + + name = get_text(name_node) + if name in param_exclude: + continue + + name = '{%s}' % name + name_length = max(name_length, len(name) + 2) + + desc = '' + desc_node = get_child(child, 'parameterdescription') + if desc_node: + desc = parse_parblock(desc_node, width=None) + items.append((name.strip(), desc.strip())) + + out = 'Parameters:~\n' + for name, desc in items: + name = ' %s' % name.ljust(name_length) + out += doc_wrap(desc, prefix=name, width=width) + '\n' + return out.strip() + + +def parse_para(parent, width=62): + """Parse doxygen `para` tag. + + I assume <para> is a paragraph block or "a block of text". It can contain + text nodes, or other tags. + """ + line = '' + lines = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + line += child.data + elif child.nodeName == 'computeroutput': + line += '`%s`' % get_text(child) + else: + if line: + lines.append(doc_wrap(line, width=width)) + line = '' + + if child.nodeName == 'parameterlist': + lines.append(parse_params(child, width=width)) + elif child.nodeName == 'xrefsect': + title = get_text(get_child(child, 'xreftitle')) + xrefs.add(title) + xrefdesc = parse_para(get_child(child, 'xrefdescription')) + lines.append(doc_wrap(xrefdesc, prefix='%s: ' % title, + width=width) + '\n') + elif child.nodeName == 'simplesect': + kind = child.getAttribute('kind') + if kind == 'return': + lines.append('%s:~' % kind.title()) + lines.append(doc_wrap(parse_para(child), + prefix=' ', + width=width)) + else: + lines.append(get_text(child)) + + if line: + lines.append(doc_wrap(line, width=width)) + return clean_lines('\n'.join(lines).strip()) + + +def parse_parblock(parent, width=62): + """Parses a nested block of `para` tags. + + Named after the \parblock command, but not directly related. + """ + paragraphs = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + paragraphs.append(doc_wrap(child.data, width=width)) + elif child.nodeName == 'para': + paragraphs.append(parse_para(child, width=width)) + else: + paragraphs.append(doc_wrap(get_text(child), width=width)) + paragraphs.append('') + return clean_lines('\n'.join(paragraphs).strip()) +# }}} + + +def parse_source_xml(filename): + """Collects API functions. + + This returns two strings: + 1. The API functions + 2. The deprecated API functions + + The caller decides what to do with the deprecated documentation. + """ + global xrefs + xrefs = set() + functions = [] + deprecated_functions = [] + + dom = minidom.parse(filename) + for member in dom.getElementsByTagName('memberdef'): + if member.getAttribute('static') == 'yes' or \ + member.getAttribute('kind') != 'function': + continue + + loc = find_first(member, 'location') + if 'private' in loc.getAttribute('file'): + continue + + return_type = get_text(get_child(member, 'type')) + if return_type == '': + continue + + if return_type.startswith(('ArrayOf', 'DictionaryOf')): + parts = return_type.strip('_').split('_') + return_type = '%s(%s)' % (parts[0], ', '.join(parts[1:])) + + name = get_text(get_child(member, 'name')) + + vimtag = '*%s()*' % name + args = [] + type_length = 0 + + for param in get_children(member, 'param'): + arg_type = get_text(get_child(param, 'type')).strip() + arg_name = '' + declname = get_child(param, 'declname') + if declname: + arg_name = get_text(declname).strip() + + if arg_name in param_exclude: + continue + + if arg_type.endswith('*'): + arg_type = arg_type.strip('* ') + arg_name = '*' + arg_name + type_length = max(type_length, len(arg_type)) + args.append((arg_type, arg_name)) + + c_args = [] + for arg_type, arg_name in args: + c_args.append(' ' + ( + '%s %s' % (arg_type.ljust(type_length), arg_name)).strip()) + + c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, + ',\n'.join(c_args)), + ' ') + + prefix = '%s(' % name + suffix = '%s)' % ', '.join('{%s}' % a[1] for a in args + if a[0] not in ('void', 'Error')) + + # Minimum 8 chars between signature and vimtag + lhs = (text_width - 8) - len(prefix) + + if len(prefix) + len(suffix) > lhs: + signature = vimtag.rjust(text_width) + '\n' + signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, + func=True) + else: + signature = prefix + suffix + signature += vimtag.rjust(text_width - len(signature)) + + doc = '' + desc = find_first(member, 'detaileddescription') + if desc: + doc = parse_parblock(desc) + if 'DEBUG' in os.environ: + print(textwrap.indent( + re.sub(r'\n\s*\n+', '\n', + desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + + if not doc: + doc = 'TODO: Documentation' + + if 'INCLUDE_C_DECL' in os.environ: + doc += '\n\nC Declaration:~\n>\n' + doc += c_decl + doc += '\n<' + + func_doc = signature + '\n' + func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) + func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) + + if 'Deprecated' in xrefs: + deprecated_functions.append(func_doc) + else: + functions.append(func_doc) + + xrefs.clear() + + return '\n\n'.join(functions), '\n\n'.join(deprecated_functions) + + +def gen_docs(config): + """Generate documentation. + + Doxygen is called and configured through stdin. + """ + p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) + p.communicate(config.format(input=src_dir, output=out_dir, + filter=filter_cmd).encode('utf8')) + if p.returncode: + sys.exit(p.returncode) + + title_length = 0 + sections = {} + sep = '=' * text_width + + base = os.path.join(out_dir, 'xml') + dom = minidom.parse(os.path.join(base, 'index.xml')) + for compound in dom.getElementsByTagName('compound'): + if compound.getAttribute('kind') != 'file': + continue + + filename = get_text(find_first(compound, 'name')) + if filename.endswith('.c'): + functions, deprecated = parse_source_xml( + os.path.join(base, '%s.xml' % compound.getAttribute('refid'))) + + if not functions and not deprecated: + continue + + if functions or deprecated: + name = os.path.splitext(os.path.basename(filename))[0] + if name == 'ui': + name = name.upper() + else: + name = name.title() + + doc = '' + if functions: + doc += '\n\n' + functions + + if 'INCLUDE_DEPRECATED' in os.environ and deprecated: + doc += '\n\n\nDeprecated %s Functions:~\n\n' % name + doc += deprecated + + if doc: + filename = os.path.basename(filename) + name = section_name.get(filename, name) + title = '%s Functions' % name + helptag = '*api-%s*' % name.lower() + title_length = max(title_length, len(title)) + sections[filename] = (title, helptag, doc) + + if not sections: + return + + title_left = '*%s*' % doc_filename + title_center = 'Neovim API Function Reference' + title_right = '{Nvim}' + margin = max(len(title_left), len(title_right)) + head = (title_left.ljust(margin) + + title_center.center(text_width - margin * 2) + + title_right.rjust(margin)) + '\n' + + head += '\n%s\n\n' % doc_wrap(preamble, width=text_width) + head += 'Contents:\n\n' + + docs = '' + + title_length += len(str(len(section_order))) + 2 + i = 0 + for filename in section_order: + if filename not in sections: + continue + title, helptag, section_doc = sections.pop(filename) + + i += 1 + docs += sep + title = '%d. %s' % (i, title) + head += (title.ljust(title_length) + ' ' + + helptag.replace('*', '|') + '\n') + docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) + docs += section_doc + docs += '\n\n\n' + + if sections: + # In case new API sources are added without updating the order dict. + for title, helptag, section_doc in sections.values(): + i += 1 + docs += sep + title = '%d. %s' % (i, title) + head += (title.ljust(title_length) + ' ' + + helptag.replace('*', '|') + '\n') + docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) + docs += section_doc + docs += '\n\n\n' + + docs = '%s\n%s' % (head, docs) + docs = docs.rstrip() + '\n\n' + docs += ' vim:tw=78:ts=8:ft=help:norl:' + + doc_file = os.path.join(base_dir, 'runtime/doc', doc_filename) + with open(doc_file, 'wb') as fp: + fp.write(docs.encode('utf8')) + + shutil.rmtree(out_dir) + + +def filter_source(filename): + """Filters the source to fix macros that confuse Doxygen.""" + with open(filename, 'rt') as fp: + print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))', + lambda m: m.group(1)+'_'.join( + re.split(r'[^\w]+', m.group(2))), + fp.read(), flags=re.M)) + + +# Doxygen Config {{{ +Doxyfile = ''' +OUTPUT_DIRECTORY = {output} +INPUT = {input} +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = *.h *.c +RECURSIVE = YES +INPUT_FILTER = "{filter}" +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = */private/* +EXCLUDE_SYMBOLS = + +GENERATE_HTML = NO +GENERATE_DOCSET = NO +GENERATE_HTMLHELP = NO +GENERATE_QHP = NO +GENERATE_TREEVIEW = NO +GENERATE_LATEX = NO +GENERATE_RTF = NO +GENERATE_MAN = NO +GENERATE_DOCBOOK = NO +GENERATE_AUTOGEN_DEF = NO + +GENERATE_XML = YES +XML_OUTPUT = xml +XML_PROGRAMLISTING = NO + +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = YES +EXPAND_ONLY_PREDEF = NO +''' +# }}} + +if __name__ == "__main__": + if len(sys.argv) > 1: + filter_source(sys.argv[1]) + else: + gen_docs(Doxyfile) + +# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : |