diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/gen_api_vimdoc.py | 563 | ||||
| -rw-r--r-- | scripts/gen_help_html.py | 74 | ||||
| -rwxr-xr-x | scripts/gen_vimdoc.py | 763 | ||||
| -rwxr-xr-x | scripts/genappimage.sh | 52 | ||||
| -rw-r--r-- | scripts/lua2dox.lua | 666 | ||||
| -rwxr-xr-x | scripts/lua2dox_filter | 87 | ||||
| -rwxr-xr-x | scripts/vim-patch.sh | 47 |
7 files changed, 1610 insertions, 642 deletions
diff --git a/scripts/gen_api_vimdoc.py b/scripts/gen_api_vimdoc.py deleted file mode 100755 index 515964bfe8..0000000000 --- a/scripts/gen_api_vimdoc.py +++ /dev/null @@ -1,563 +0,0 @@ -#!/usr/bin/env python3 -"""Parses Doxygen XML output to generate Neovim's API documentation. - -This would be easier using lxml and XSLT, but: - - 1. This should avoid needing Python dependencies, especially ones that are - C modules that have library dependencies (lxml requires libxml and - libxslt). - 2. I wouldn't know how to deal with nested indentation in <para> tags using - XSLT. - -Each function documentation is formatted with the following rules: - - - Maximum width of 78 characters (`text_width`). - - Spaces for indentation. - - Function signature and helptag are on the same line. - - Helptag is right aligned. - - Signature and helptag must have a minimum of 8 spaces between them. - - If the signature is too long, it is placed on the line after the - helptag. The signature wraps at `text_width - 8` characters with - subsequent lines indented to the open parenthesis. - - Documentation body will be indented by 16 spaces. - - Subsection bodies are indented an additional 4 spaces. - - Documentation body consists of the function description, parameter details, - return description, and C declaration. - - Parameters are omitted for the `void` and `Error *` types, or if the - parameter is marked as [out]. - - Each function documentation is separated by a single line. - -The C declaration is added to the end to show actual argument types. -""" -import os -import re -import sys -import shutil -import textwrap -import subprocess - -from xml.dom import minidom - -if sys.version_info[0] < 3: - print("use Python 3") - sys.exit(1) - -doc_filename = 'api.txt' -# String used to find the start of the generated part of the doc. -section_start_token = '*api-global*' -# Required prefix for API function names. -api_func_name_prefix = 'nvim_' - -# Section name overrides. -section_name = { - 'vim.c': 'Global', -} - -# Section ordering. -section_order = ( - 'vim.c', - 'buffer.c', - 'window.c', - 'tabpage.c', - 'ui.c', -) - -param_exclude = ( - 'channel_id', -) - -# Annotations are displayed as line items after API function descriptions. -annotation_map = { - 'FUNC_API_ASYNC': '{async}', -} - -text_width = 78 -script_path = os.path.abspath(__file__) -base_dir = os.path.dirname(os.path.dirname(script_path)) -src_dir = os.path.join(base_dir, 'src/nvim/api') -out_dir = os.path.join(base_dir, 'tmp-api-doc') -filter_cmd = '%s %s' % (sys.executable, script_path) -seen_funcs = set() - -# Tracks `xrefsect` titles. As of this writing, used only for separating -# deprecated functions. -xrefs = set() - - -# XML Parsing Utilities {{{ -def find_first(parent, name): - """Finds the first matching node within parent.""" - sub = parent.getElementsByTagName(name) - if not sub: - return None - return sub[0] - - -def get_children(parent, name): - """Yield matching child nodes within parent.""" - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: - yield child - - -def get_child(parent, name): - """Get the first matching child node.""" - for child in get_children(parent, name): - return child - return None - - -def clean_text(text): - """Cleans text. - - Only cleans superfluous whitespace at the moment. - """ - return ' '.join(text.split()).strip() - - -def clean_lines(text): - """Removes superfluous lines. - - The beginning and end of the string is trimmed. Empty lines are collapsed. - """ - return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text)) - - -def get_text(parent): - """Combine all text in a node.""" - if parent.nodeType == parent.TEXT_NODE: - return parent.data - - out = '' - for node in parent.childNodes: - if node.nodeType == node.TEXT_NODE: - out += clean_text(node.data) - elif node.nodeType == node.ELEMENT_NODE: - out += ' ' + get_text(node) - return out - - -def doc_wrap(text, prefix='', width=70, func=False): - """Wraps text to `width`. - - The first line is prefixed with `prefix`, and subsequent lines are aligned. - If `func` is True, only wrap at commas. - """ - if not width: - return text - - indent_space = ' ' * len(prefix) - - if func: - lines = [prefix] - for part in text.split(', '): - if part[-1] not in ');': - part += ', ' - if len(lines[-1]) + len(part) > width: - lines.append(indent_space) - lines[-1] += part - return '\n'.join(x.rstrip() for x in lines).rstrip() - - tw = textwrap.TextWrapper(break_long_words = False, - break_on_hyphens = False, - width=width, - initial_indent=prefix, - subsequent_indent=indent_space) - return '\n'.join(tw.wrap(text.strip())) - - -def parse_params(parent, width=62): - """Parse Doxygen `parameterlist`.""" - name_length = 0 - items = [] - for child in parent.childNodes: - if child.nodeType == child.TEXT_NODE: - continue - - name_node = find_first(child, 'parametername') - if name_node.getAttribute('direction') == 'out': - continue - - name = get_text(name_node) - if name in param_exclude: - continue - - name = '{%s}' % name - name_length = max(name_length, len(name) + 2) - - desc = '' - desc_node = get_child(child, 'parameterdescription') - if desc_node: - desc = parse_parblock(desc_node, width=None) - items.append((name.strip(), desc.strip())) - - out = 'Parameters: ~\n' - for name, desc in items: - name = ' %s' % name.ljust(name_length) - out += doc_wrap(desc, prefix=name, width=width) + '\n' - return out.strip() - - -def parse_para(parent, width=62): - """Parse doxygen `para` tag. - - I assume <para> is a paragraph block or "a block of text". It can contain - text nodes, or other tags. - """ - line = '' - lines = [] - for child in parent.childNodes: - if child.nodeType == child.TEXT_NODE: - line += child.data - elif child.nodeName == 'computeroutput': - line += '`%s`' % get_text(child) - else: - if line: - lines.append(doc_wrap(line, width=width)) - line = '' - - if child.nodeName == 'parameterlist': - lines.append(parse_params(child, width=width)) - elif child.nodeName == 'xrefsect': - title = get_text(get_child(child, 'xreftitle')) - xrefs.add(title) - xrefdesc = parse_para(get_child(child, 'xrefdescription')) - lines.append(doc_wrap(xrefdesc, prefix='%s: ' % title, - width=width) + '\n') - elif child.nodeName == 'simplesect': - kind = child.getAttribute('kind') - if kind == 'note': - lines.append('Note:') - lines.append(doc_wrap(parse_para(child), - prefix=' ', - width=width)) - elif kind == 'return': - lines.append('%s: ~' % kind.title()) - lines.append(doc_wrap(parse_para(child), - prefix=' ', - width=width)) - else: - lines.append(get_text(child)) - - if line: - lines.append(doc_wrap(line, width=width)) - return clean_lines('\n'.join(lines).strip()) - - -def parse_parblock(parent, width=62): - """Parses a nested block of `para` tags. - - Named after the \parblock command, but not directly related. - """ - paragraphs = [] - for child in parent.childNodes: - if child.nodeType == child.TEXT_NODE: - paragraphs.append(doc_wrap(child.data, width=width)) - elif child.nodeName == 'para': - paragraphs.append(parse_para(child, width=width)) - else: - paragraphs.append(doc_wrap(get_text(child), width=width)) - paragraphs.append('') - return clean_lines('\n'.join(paragraphs).strip()) -# }}} - - -def parse_source_xml(filename): - """Collects API functions. - - Returns two strings: - 1. API functions - 2. Deprecated API functions - - Caller decides what to do with the deprecated documentation. - """ - global xrefs - xrefs = set() - functions = [] - deprecated_functions = [] - - dom = minidom.parse(filename) - for member in dom.getElementsByTagName('memberdef'): - if member.getAttribute('static') == 'yes' or \ - member.getAttribute('kind') != 'function': - continue - - loc = find_first(member, 'location') - if 'private' in loc.getAttribute('file'): - continue - - return_type = get_text(get_child(member, 'type')) - if return_type == '': - continue - - if return_type.startswith(('ArrayOf', 'DictionaryOf')): - parts = return_type.strip('_').split('_') - return_type = '%s(%s)' % (parts[0], ', '.join(parts[1:])) - - name = get_text(get_child(member, 'name')) - - annotations = get_text(get_child(member, 'argsstring')) - if annotations and ')' in annotations: - annotations = annotations.rsplit(')', 1)[-1].strip() - # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of - # non-void functions. Special-case void functions here. - if name == 'nvim_get_mode' and len(annotations) == 0: - annotations += 'FUNC_API_ASYNC' - annotations = filter(None, map(lambda x: annotation_map.get(x), - annotations.split())) - - vimtag = '*%s()*' % name - args = [] - type_length = 0 - - for param in get_children(member, 'param'): - arg_type = get_text(get_child(param, 'type')).strip() - arg_name = '' - declname = get_child(param, 'declname') - if declname: - arg_name = get_text(declname).strip() - - if arg_name in param_exclude: - continue - - if arg_type.endswith('*'): - arg_type = arg_type.strip('* ') - arg_name = '*' + arg_name - type_length = max(type_length, len(arg_type)) - args.append((arg_type, arg_name)) - - c_args = [] - for arg_type, arg_name in args: - c_args.append(' ' + ( - '%s %s' % (arg_type.ljust(type_length), arg_name)).strip()) - - c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, - ',\n'.join(c_args)), - ' ') - - prefix = '%s(' % name - suffix = '%s)' % ', '.join('{%s}' % a[1] for a in args - if a[0] not in ('void', 'Error')) - - # Minimum 8 chars between signature and vimtag - lhs = (text_width - 8) - len(prefix) - - if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(text_width) + '\n' - signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, - func=True) - else: - signature = prefix + suffix - signature += vimtag.rjust(text_width - len(signature)) - - doc = '' - desc = find_first(member, 'detaileddescription') - if desc: - doc = parse_parblock(desc) - if 'DEBUG' in os.environ: - print(textwrap.indent( - re.sub(r'\n\s*\n+', '\n', - desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) - - if not doc: - doc = 'TODO: Documentation' - - annotations = '\n'.join(annotations) - if annotations: - annotations = ('\n\nAttributes: ~\n' + - textwrap.indent(annotations, ' ')) - i = doc.rfind('Parameters: ~') - if i == -1: - doc += annotations - else: - doc = doc[:i] + annotations + '\n\n' + doc[i:] - - if 'INCLUDE_C_DECL' in os.environ: - doc += '\n\nC Declaration: ~\n>\n' - doc += c_decl - doc += '\n<' - - func_doc = signature + '\n' - func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) - func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) - - if 'Deprecated' in xrefs: - deprecated_functions.append(func_doc) - elif name.startswith(api_func_name_prefix): - functions.append(func_doc) - - xrefs.clear() - - return '\n\n'.join(functions), '\n\n'.join(deprecated_functions) - - -def delete_lines_below(filename, tokenstr): - """Deletes all lines below the line containing `tokenstr`, the line itself, - and one line above it. - """ - lines = open(filename).readlines() - i = 0 - for i, line in enumerate(lines, 1): - if tokenstr in line: - break - i = max(0, i - 2) - with open(filename, 'wt') as fp: - fp.writelines(lines[0:i]) - -def gen_docs(config): - """Generate documentation. - - Doxygen is called and configured through stdin. - """ - p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) - p.communicate(config.format(input=src_dir, output=out_dir, - filter=filter_cmd).encode('utf8')) - if p.returncode: - sys.exit(p.returncode) - - sections = {} - intros = {} - sep = '=' * text_width - - base = os.path.join(out_dir, 'xml') - dom = minidom.parse(os.path.join(base, 'index.xml')) - - # generate docs for section intros - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'group': - continue - - groupname = get_text(find_first(compound, 'name')) - groupxml = os.path.join(base, '%s.xml' % compound.getAttribute('refid')) - - desc = find_first(minidom.parse(groupxml), 'detaileddescription') - if desc: - doc = parse_parblock(desc) - if doc: - intros[groupname] = doc - - for compound in dom.getElementsByTagName('compound'): - if compound.getAttribute('kind') != 'file': - continue - - filename = get_text(find_first(compound, 'name')) - if filename.endswith('.c'): - functions, deprecated = parse_source_xml( - os.path.join(base, '%s.xml' % compound.getAttribute('refid'))) - - if not functions and not deprecated: - continue - - if functions or deprecated: - name = os.path.splitext(os.path.basename(filename))[0] - if name == 'ui': - name = name.upper() - else: - name = name.title() - - doc = '' - - intro = intros.get('api-%s' % name.lower()) - if intro: - doc += '\n\n' + intro - - if functions: - doc += '\n\n' + functions - - if 'INCLUDE_DEPRECATED' in os.environ and deprecated: - doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name - doc += deprecated - - if doc: - filename = os.path.basename(filename) - name = section_name.get(filename, name) - title = '%s Functions' % name - helptag = '*api-%s*' % name.lower() - sections[filename] = (title, helptag, doc) - - if not sections: - return - - docs = '' - - i = 0 - for filename in section_order: - if filename not in sections: - continue - title, helptag, section_doc = sections.pop(filename) - - i += 1 - docs += sep - docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) - docs += section_doc - docs += '\n\n\n' - - if sections: - # In case new API sources are added without updating the order dict. - for title, helptag, section_doc in sections.values(): - i += 1 - docs += sep - docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) - docs += section_doc - docs += '\n\n\n' - - docs = docs.rstrip() + '\n\n' - docs += ' vim:tw=78:ts=8:ft=help:norl:\n' - - doc_file = os.path.join(base_dir, 'runtime/doc', doc_filename) - delete_lines_below(doc_file, section_start_token) - with open(doc_file, 'ab') as fp: - fp.write(docs.encode('utf8')) - shutil.rmtree(out_dir) - - -def filter_source(filename): - """Filters the source to fix macros that confuse Doxygen.""" - with open(filename, 'rt') as fp: - print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))', - lambda m: m.group(1)+'_'.join( - re.split(r'[^\w]+', m.group(2))), - fp.read(), flags=re.M)) - - -# Doxygen Config {{{ -Doxyfile = ''' -OUTPUT_DIRECTORY = {output} -INPUT = {input} -INPUT_ENCODING = UTF-8 -FILE_PATTERNS = *.h *.c -RECURSIVE = YES -INPUT_FILTER = "{filter}" -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = */private/* -EXCLUDE_SYMBOLS = - -GENERATE_HTML = NO -GENERATE_DOCSET = NO -GENERATE_HTMLHELP = NO -GENERATE_QHP = NO -GENERATE_TREEVIEW = NO -GENERATE_LATEX = NO -GENERATE_RTF = NO -GENERATE_MAN = NO -GENERATE_DOCBOOK = NO -GENERATE_AUTOGEN_DEF = NO - -GENERATE_XML = YES -XML_OUTPUT = xml -XML_PROGRAMLISTING = NO - -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = YES -EXPAND_ONLY_PREDEF = NO -''' -# }}} - -if __name__ == "__main__": - if len(sys.argv) > 1: - filter_source(sys.argv[1]) - else: - gen_docs(Doxyfile) - -# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : diff --git a/scripts/gen_help_html.py b/scripts/gen_help_html.py index 6cca81d250..dbdeb3c162 100644 --- a/scripts/gen_help_html.py +++ b/scripts/gen_help_html.py @@ -1,5 +1,10 @@ # Converts Vim/Nvim documentation to HTML. # +# USAGE: +# 1. python3 scripts/gen_help_html.py runtime/doc/ ~/neovim.github.io/t/ +# 3. cd ~/neovim.github.io/ && jekyll serve --host 0.0.0.0 +# 2. Visit http://localhost:4000/t/help.txt.html +# # Adapted from https://github.com/c4rlo/vimhelp/ # License: MIT # @@ -23,7 +28,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import re, urllib.parse +import os +import re +import urllib.parse +import datetime +import sys from itertools import chain HEAD = """\ @@ -32,6 +41,17 @@ HEAD = """\ <html> <head> <meta http-equiv="Content-type" content="text/html; charset={encoding}"/> +<style> +.h {{ + font-weight: bold; +}} +h1 {{ + font-family: sans-serif; +}} +pre {{ + font-family: sans-serif; +}} +</style> <title>Nvim: {filename}</title> """ @@ -39,22 +59,19 @@ HEAD_END = '</head>\n<body>\n' INTRO = """ <h1>Nvim help files</h1> -<p>HTML export of the <a href="https://neovim.io/">Nvim</a> help pages{vers-note}. -Updated <a href="https://github.com/neovim/bot-ci" class="d">automatically</a> from the <a -href="https://github.com/vim/vim/tree/master/runtime/doc" class="d">Nvim source repository</a>. -Also includes the <a href="vim_faq.txt.html">Vim FAQ</a>, pulled from its -<a href="https://github.com/chrisbra/vim_faq" class="d">source repository</a>.</p> +<p> +<a href="https://neovim.io/">Nvim</a> help pages{vers-note}. +Updated <a href="https://github.com/neovim/bot-ci" class="d">automatically</a> +from the <a href="https://github.com/neovim/neovim" class="d">Nvim source</a>. +</p> """ -VERSION_NOTE = ", current as of Vim {version}" +VERSION_NOTE = ", current as of Nvim {version}" SITENAVI_LINKS = """ -Quick links: -<a href="/">help overview</a> · -<a href="quickref.txt.html">quick reference</a> · -<a href="usr_toc.txt.html">user manual toc</a> · -<a href="{helptxt}#reference_toc">reference manual toc</a> · -<a href="vim_faq.txt.html">faq</a> +<a href="quickref.txt.html">Quick reference</a> · +<a href="usr_toc.txt.html">User manual</a> · +<a href="{helptxt}#reference_toc">Reference manual</a> · """ SITENAVI_LINKS_PLAIN = SITENAVI_LINKS.format(helptxt='help.txt.html') @@ -77,15 +94,14 @@ TEXTSTART = """ FOOTER = '</pre>' FOOTER2 = """ -<p id="footer">This site is maintained by Carlo Teubner (<i>(my first name) dot (my last name) at gmail dot com</i>).</p> +<p id="footer">Generated {generated_date} from <code>{commit}</code></p> </div> </div> </body> </html> -""" - -VIM_FAQ_LINE = '<a href="vim_faq.txt.html#vim_faq.txt" class="l">' \ - 'vim_faq.txt</a> Frequently Asked Questions\n' +""".format( + generated_date='{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()), + commit='?') RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)') @@ -127,6 +143,8 @@ RE_TAGWORD = re.compile( PAT_URL + '|' + PAT_WORD) RE_NEWLINE = re.compile(r'[\r\n]') +# H1 header "=====…" +# H2 header "-----…" RE_HRULE = re.compile(r'[-=]{3,}.*[-=]{3,3}$') RE_EG_START = re.compile(r'(?:.* )?>$') RE_EG_END = re.compile(r'\S') @@ -211,13 +229,20 @@ class VimH2H(object): inexample = 0 filename = str(filename) is_help_txt = (filename == 'help.txt') - faq_line = False + last = '' for line in RE_NEWLINE.split(contents): line = line.rstrip('\r\n') line_tabs = line line = line.expandtabs() + if last == 'h1': + out.extend(('</pre>')) # XXX + out.extend(('<h1>', line.rstrip(), '</h1>\n')) + out.extend(('<pre>')) + last = '' + continue if RE_HRULE.match(line): - out.extend(('<span class="h">', line, '</span>\n')) + # out.extend(('<span class="h">', line, '</span>\n')) + last = 'h1' continue if inexample == 2: if RE_EG_END.match(line): @@ -234,8 +259,6 @@ class VimH2H(object): m = RE_SECTION.match(line) out.extend((r'<span class="c">', m.group(0), r'</span>')) line = line[m.end():] - if is_help_txt and RE_LOCAL_ADD.match(line_tabs): - faq_line = True lastpos = 0 for match in RE_TAGWORD.finditer(line): pos = match.start() @@ -278,9 +301,6 @@ class VimH2H(object): out.append(html_escape[line[lastpos:]]) out.append('\n') if inexample == 1: inexample = 2 - if faq_line: - out.append(VIM_FAQ_LINE) - faq_line = False header = [] header.append(HEAD.format(encoding=encoding, filename=filename)) @@ -310,9 +330,6 @@ html_escape = HtmlEscCache() -import sys, os, os.path -#import cProfile -sys.path.append('.') def slurp(filename): try: @@ -352,4 +369,3 @@ def main(): of.close() main() -#cProfile.run('main()') diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py new file mode 100755 index 0000000000..e491ff59b0 --- /dev/null +++ b/scripts/gen_vimdoc.py @@ -0,0 +1,763 @@ +#!/usr/bin/env python3 +"""Generates Nvim help docs from C docstrings, by parsing Doxygen XML. + +This would be easier using lxml and XSLT, but: + + 1. This should avoid needing Python dependencies, especially ones that are + C modules that have library dependencies (lxml requires libxml and + libxslt). + 2. I wouldn't know how to deal with nested indentation in <para> tags using + XSLT. + +Each function documentation is formatted with the following rules: + + - Maximum width of 78 characters (`text_width`). + - Spaces for indentation. + - Function signature and helptag are on the same line. + - Helptag is right aligned. + - Signature and helptag must have a minimum of 8 spaces between them. + - If the signature is too long, it is placed on the line after the + helptag. The signature wraps at `text_width - 8` characters with + subsequent lines indented to the open parenthesis. + - Documentation body will be indented by 16 spaces. + - Subsection bodies are indented an additional 4 spaces. + - Documentation body consists of the function description, parameter details, + return description, and C declaration. + - Parameters are omitted for the `void` and `Error *` types, or if the + parameter is marked as [out]. + - Each function documentation is separated by a single line. + +The C declaration is added to the end to show actual argument types. +""" +import os +import re +import sys +import shutil +import textwrap +import subprocess +import collections +import pprint + +from xml.dom import minidom + +if sys.version_info[0] < 3: + print("use Python 3") + sys.exit(1) + +DEBUG = ('DEBUG' in os.environ) +INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) +INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) + +text_width = 78 +script_path = os.path.abspath(__file__) +base_dir = os.path.dirname(os.path.dirname(script_path)) +out_dir = os.path.join(base_dir, 'tmp-{mode}-doc') +filter_cmd = '%s %s' % (sys.executable, script_path) +seen_funcs = set() +lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter') + +CONFIG = { + 'api': { + 'filename': 'api.txt', + # String used to find the start of the generated part of the doc. + 'section_start_token': '*api-global*', + # Section ordering. + 'section_order' : [ + 'vim.c', + 'buffer.c', + 'window.c', + 'tabpage.c', + 'ui.c', + ], + # List of files/directories for doxygen to read, separated by blanks + 'files': os.path.join(base_dir, 'src/nvim/api'), + # file patterns used by doxygen + 'file_patterns': '*.h *.c', + # Only function with this prefix are considered + 'func_name_prefix': 'nvim_', + # Section name overrides. + 'section_name': { + 'vim.c': 'Global', + }, + # Module name overrides (for Lua). + 'module_override': {}, + # Append the docs for these modules, do not start a new section. + 'append_only' : [], + }, + 'lua': { + 'filename': 'if_lua.txt', + 'section_start_token': '*lua-vim*', + 'section_order' : [ + 'vim.lua', + 'shared.lua', + ], + 'files': ' '.join([ + os.path.join(base_dir, 'src/nvim/lua/vim.lua'), + os.path.join(base_dir, 'runtime/lua/vim/shared.lua'), + ]), + 'file_patterns': '*.lua', + 'func_name_prefix': '', + 'section_name': {}, + 'module_override': { + 'shared': 'vim', # `shared` functions are exposed on the `vim` module. + }, + 'append_only' : [ + 'shared.lua', + ], + }, +} + +param_exclude = ( + 'channel_id', +) + +# Annotations are displayed as line items after API function descriptions. +annotation_map = { + 'FUNC_API_ASYNC': '{async}', +} + + +# Tracks `xrefsect` titles. As of this writing, used only for separating +# deprecated functions. +xrefs = set() + +def debug_this(s, n): + o = n if isinstance(n, str) else n.toprettyxml(indent=' ', newl='\n') + name = '' if isinstance(n, str) else n.nodeName + if s in o: + raise RuntimeError('xxx: {}\n{}'.format(name, o)) + + +# XML Parsing Utilities {{{ +def find_first(parent, name): + """Finds the first matching node within parent.""" + sub = parent.getElementsByTagName(name) + if not sub: + return None + return sub[0] + + +def get_children(parent, name): + """Yield matching child nodes within parent.""" + for child in parent.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: + yield child + + +def get_child(parent, name): + """Get the first matching child node.""" + for child in get_children(parent, name): + return child + return None + + +def clean_text(text): + """Cleans text. + + Only cleans superfluous whitespace at the moment. + """ + return ' '.join(text.split()).strip() + + +def clean_lines(text): + """Removes superfluous lines. + + The beginning and end of the string is trimmed. Empty lines are collapsed. + """ + return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text)) + + +def is_blank(text): + return '' == clean_lines(text) + + +def get_text(parent, preformatted=False): + """Combine all text in a node.""" + if parent.nodeType == parent.TEXT_NODE: + return parent.data + + out = '' + for node in parent.childNodes: + if node.nodeType == node.TEXT_NODE: + out += node.data if preformatted else clean_text(node.data) + elif node.nodeType == node.ELEMENT_NODE: + out += ' ' + get_text(node, preformatted) + return out + + +# Gets the length of the last line in `text`, excluding newline ("\n") char. +def len_lastline(text): + lastnl = text.rfind('\n') + if -1 == lastnl: + return len(text) + if '\n' == text[-1]: + return lastnl - (1+ text.rfind('\n', 0, lastnl)) + return len(text) - (1 + lastnl) + + +def len_lastline_withoutindent(text, indent): + n = len_lastline(text) + return (n - len(indent)) if n > len(indent) else 0 + + +# Returns True if node `n` contains only inline (not block-level) elements. +def is_inline(n): + for c in n.childNodes: + if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput': + return False + if not is_inline(c): + return False + return True + +def doc_wrap(text, prefix='', width=70, func=False, indent=None): + """Wraps text to `width`. + + First line is prefixed with `prefix`, subsequent lines are aligned. + If `func` is True, only wrap at commas. + """ + if not width: + # return prefix + text + return text + + # Whitespace used to indent all lines except the first line. + indent = ' ' * len(prefix) if indent is None else indent + indent_only = (prefix == '' and indent is not None) + + if func: + lines = [prefix] + for part in text.split(', '): + if part[-1] not in ');': + part += ', ' + if len(lines[-1]) + len(part) > width: + lines.append(indent) + lines[-1] += part + return '\n'.join(x.rstrip() for x in lines).rstrip() + + # XXX: Dummy prefix to force TextWrapper() to wrap the first line. + if indent_only: + prefix = indent + + tw = textwrap.TextWrapper(break_long_words = False, + break_on_hyphens = False, + width=width, + initial_indent=prefix, + subsequent_indent=indent) + result = '\n'.join(tw.wrap(text.strip())) + + # XXX: Remove the dummy prefix. + if indent_only: + result = result[len(indent):] + + return result + + +def has_nonexcluded_params(nodes): + """Returns true if any of the given <parameterlist> elements has at least + one non-excluded item.""" + for n in nodes: + if render_params(n) != '': + return True + + +def render_params(parent, width=62): + """Renders Doxygen <parameterlist> tag as Vim help text.""" + name_length = 0 + items = [] + for node in parent.childNodes: + if node.nodeType == node.TEXT_NODE: + continue + + name_node = find_first(node, 'parametername') + if name_node.getAttribute('direction') == 'out': + continue + + name = get_text(name_node) + if name in param_exclude: + continue + + name = '{%s}' % name + name_length = max(name_length, len(name) + 2) + items.append((name.strip(), node)) + + out = '' + for name, node in items: + name = ' {}'.format(name.ljust(name_length)) + + desc = '' + desc_node = get_child(node, 'parameterdescription') + if desc_node: + desc = parse_parblock(desc_node, width=width, + indent=(' ' * len(name))) + + out += '{}{}\n'.format(name, desc) + return out.rstrip() + +# Renders a node as Vim help text, recursively traversing all descendants. +def render_node(n, text, prefix='', indent='', width=62): + text = '' + # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) + # text += (int(not space_preceding) * ' ') + + if n.nodeType == n.TEXT_NODE: + # `prefix` is NOT sent to doc_wrap, it was already handled by now. + text += doc_wrap(n.data, indent=indent, width=width) + elif n.nodeName == 'computeroutput': + text += ' `{}` '.format(get_text(n)) + elif n.nodeName == 'preformatted': + o = get_text(n, preformatted=True) + ensure_nl = '' if o[-1] == '\n' else '\n' + text += ' >{}{}\n<'.format(ensure_nl, o) + elif is_inline(n): + for c in n.childNodes: + text += render_node(c, text) + text = doc_wrap(text, indent=indent, width=width) + elif n.nodeName == 'verbatim': + # TODO: currently we don't use this. The "[verbatim]" hint is there as + # a reminder that we must decide how to format this if we do use it. + text += ' [verbatim] {}'.format(get_text(n)) + elif n.nodeName == 'listitem': + for c in n.childNodes: + text += indent + prefix + render_node(c, text, indent=indent+(' ' * len(prefix)), width=width) + elif n.nodeName in ('para', 'heading'): + for c in n.childNodes: + text += render_node(c, text, indent=indent, width=width) + if is_inline(n): + text = doc_wrap(text, indent=indent, width=width) + elif n.nodeName == 'itemizedlist': + for c in n.childNodes: + text += '{}\n'.format(render_node(c, text, prefix='- ', + indent=indent, width=width)) + elif n.nodeName == 'orderedlist': + i = 1 + for c in n.childNodes: + if is_blank(get_text(c)): + text += '\n' + continue + text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i), + indent=indent, width=width)) + i = i + 1 + elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'): + text += 'Note:\n ' + for c in n.childNodes: + text += render_node(c, text, indent=' ', width=width) + text += '\n' + elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'): + text += 'Warning:\n ' + for c in n.childNodes: + text += render_node(c, text, indent=' ', width=width) + text += '\n' + elif (n.nodeName == 'simplesect' + and n.getAttribute('kind') in ('return', 'see')): + text += ' ' + for c in n.childNodes: + text += render_node(c, text, indent=' ', width=width) + else: + raise RuntimeError('unhandled node type: {}\n{}'.format( + n.nodeName, n.toprettyxml(indent=' ', newl='\n'))) + return text + +def render_para(parent, indent='', width=62): + """Renders Doxygen <para> containing arbitrary nodes. + + NB: Blank lines in a docstring manifest as <para> tags. + """ + if is_inline(parent): + return clean_lines(doc_wrap(render_node(parent, ''), + indent=indent, width=width).strip()) + + # Ordered dict of ordered lists. + groups = collections.OrderedDict([ + ('params', []), + ('return', []), + ('seealso', []), + ('xrefs', []), + ]) + + # Gather nodes into groups. Mostly this is because we want "parameterlist" + # nodes to appear together. + text = '' + kind = '' + last = '' + for child in parent.childNodes: + if child.nodeName == 'parameterlist': + groups['params'].append(child) + elif child.nodeName == 'xrefsect': + groups['xrefs'].append(child) + elif child.nodeName == 'simplesect': + last = kind + kind = child.getAttribute('kind') + if kind == 'return' or (kind == 'note' and last == 'return'): + groups['return'].append(child) + elif kind == 'see': + groups['seealso'].append(child) + elif kind in ('note', 'warning'): + text += render_node(child, text, indent=indent, width=width) + else: + raise RuntimeError('unhandled simplesect: {}\n{}'.format( + child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) + else: + text += render_node(child, text, indent=indent, width=width) + + chunks = [text] + # Generate text from the gathered items. + if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']): + chunks.append('\nParameters: ~') + for child in groups['params']: + chunks.append(render_params(child, width=width)) + if len(groups['return']) > 0: + chunks.append('\nReturn: ~') + for child in groups['return']: + chunks.append(render_node(child, chunks[-1][-1], indent=indent, width=width)) + if len(groups['seealso']) > 0: + chunks.append('\nSee also: ~') + for child in groups['seealso']: + chunks.append(render_node(child, chunks[-1][-1], indent=indent, width=width)) + for child in groups['xrefs']: + title = get_text(get_child(child, 'xreftitle')) + xrefs.add(title) + xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width) + chunks.append(doc_wrap(xrefdesc, prefix='{}: '.format(title), + width=width) + '\n') + + return clean_lines('\n'.join(chunks).strip()) + + +def parse_parblock(parent, prefix='', width=62, indent=''): + """Renders a nested block of <para> tags as Vim help text.""" + paragraphs = [] + for child in parent.childNodes: + paragraphs.append(render_para(child, width=width, indent=indent)) + paragraphs.append('') + return clean_lines('\n'.join(paragraphs).strip()) +# }}} + + +def parse_source_xml(filename, mode): + """Collects API functions. + + Returns two strings: + 1. API functions + 2. Deprecated API functions + + Caller decides what to do with the deprecated documentation. + """ + global xrefs + xrefs = set() + functions = [] + deprecated_functions = [] + + dom = minidom.parse(filename) + compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) + for member in dom.getElementsByTagName('memberdef'): + if member.getAttribute('static') == 'yes' or \ + member.getAttribute('kind') != 'function' or \ + member.getAttribute('prot') == 'private' or \ + get_text(get_child(member, 'name')).startswith('_'): + continue + + loc = find_first(member, 'location') + if 'private' in loc.getAttribute('file'): + continue + + return_type = get_text(get_child(member, 'type')) + if return_type == '': + continue + + if return_type.startswith(('ArrayOf', 'DictionaryOf')): + parts = return_type.strip('_').split('_') + return_type = '{}({})'.format(parts[0], ', '.join(parts[1:])) + + name = get_text(get_child(member, 'name')) + + annotations = get_text(get_child(member, 'argsstring')) + if annotations and ')' in annotations: + annotations = annotations.rsplit(')', 1)[-1].strip() + # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of + # non-void functions. Special-case void functions here. + if name == 'nvim_get_mode' and len(annotations) == 0: + annotations += 'FUNC_API_ASYNC' + annotations = filter(None, map(lambda x: annotation_map.get(x), + annotations.split())) + + if mode == 'lua': + fstem = compoundname.split('.')[0] + fstem = CONFIG[mode]['module_override'].get(fstem, fstem) + vimtag = '*{}.{}()*'.format(fstem, name) + else: + vimtag = '*{}()*'.format(name) + + params = [] + type_length = 0 + + for param in get_children(member, 'param'): + param_type = get_text(get_child(param, 'type')).strip() + param_name = '' + declname = get_child(param, 'declname') + if declname: + param_name = get_text(declname).strip() + elif mode == 'lua': + # that's how it comes out of lua2dox + param_name = param_type + param_type = '' + + if param_name in param_exclude: + continue + + if param_type.endswith('*'): + param_type = param_type.strip('* ') + param_name = '*' + param_name + type_length = max(type_length, len(param_type)) + params.append((param_type, param_name)) + + c_args = [] + for param_type, param_name in params: + c_args.append(' ' + ( + '%s %s' % (param_type.ljust(type_length), param_name)).strip()) + + c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, + ',\n'.join(c_args)), + ' ') + + prefix = '%s(' % name + suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params + if a[0] not in ('void', 'Error')) + + # Minimum 8 chars between signature and vimtag + lhs = (text_width - 8) - len(prefix) + + if len(prefix) + len(suffix) > lhs: + signature = vimtag.rjust(text_width) + '\n' + signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, + func=True) + else: + signature = prefix + suffix + signature += vimtag.rjust(text_width - len(signature)) + + doc = '' + desc = find_first(member, 'detaileddescription') + if desc: + doc = parse_parblock(desc) + if DEBUG: + print(textwrap.indent( + re.sub(r'\n\s*\n+', '\n', + desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + + if not doc: + doc = 'TODO: Documentation' + + annotations = '\n'.join(annotations) + if annotations: + annotations = ('\n\nAttributes: ~\n' + + textwrap.indent(annotations, ' ')) + i = doc.rfind('Parameters: ~') + if i == -1: + doc += annotations + else: + doc = doc[:i] + annotations + '\n\n' + doc[i:] + + if INCLUDE_C_DECL: + doc += '\n\nC Declaration: ~\n>\n' + doc += c_decl + doc += '\n<' + + func_doc = signature + '\n' + func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) + func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) + + if 'Deprecated' in xrefs: + deprecated_functions.append(func_doc) + elif name.startswith(CONFIG[mode]['func_name_prefix']): + functions.append(func_doc) + + xrefs.clear() + + return '\n\n'.join(functions), '\n\n'.join(deprecated_functions) + + +def delete_lines_below(filename, tokenstr): + """Deletes all lines below the line containing `tokenstr`, the line itself, + and one line above it. + """ + lines = open(filename).readlines() + i = 0 + for i, line in enumerate(lines, 1): + if tokenstr in line: + break + i = max(0, i - 2) + with open(filename, 'wt') as fp: + fp.writelines(lines[0:i]) + +def gen_docs(config): + """Generate documentation. + + Doxygen is called and configured through stdin. + """ + for mode in CONFIG: + output_dir = out_dir.format(mode=mode) + p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) + p.communicate( + config.format( + input=CONFIG[mode]['files'], + output=output_dir, + filter=filter_cmd, + file_patterns=CONFIG[mode]['file_patterns']) + .encode('utf8') + ) + if p.returncode: + sys.exit(p.returncode) + + sections = {} + intros = {} + sep = '=' * text_width + + base = os.path.join(output_dir, 'xml') + dom = minidom.parse(os.path.join(base, 'index.xml')) + + # generate docs for section intros + for compound in dom.getElementsByTagName('compound'): + if compound.getAttribute('kind') != 'group': + continue + + groupname = get_text(find_first(compound, 'name')) + groupxml = os.path.join(base, '%s.xml' % compound.getAttribute('refid')) + + desc = find_first(minidom.parse(groupxml), 'detaileddescription') + if desc: + doc = parse_parblock(desc) + if doc: + intros[groupname] = doc + + for compound in dom.getElementsByTagName('compound'): + if compound.getAttribute('kind') != 'file': + continue + + filename = get_text(find_first(compound, 'name')) + if filename.endswith('.c') or filename.endswith('.lua'): + functions, deprecated = parse_source_xml( + os.path.join(base, '%s.xml' % + compound.getAttribute('refid')), mode) + + if not functions and not deprecated: + continue + + if functions or deprecated: + name = os.path.splitext(os.path.basename(filename))[0] + if name == 'ui': + name = name.upper() + else: + name = name.title() + + doc = '' + + intro = intros.get('api-%s' % name.lower()) + if intro: + doc += '\n\n' + intro + + if functions: + doc += '\n\n' + functions + + if INCLUDE_DEPRECATED and deprecated: + doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name + doc += deprecated + + if doc: + filename = os.path.basename(filename) + name = CONFIG[mode]['section_name'].get(filename, name) + + if mode == 'lua': + title = 'Lua module: {}'.format(name.lower()) + helptag = '*lua-{}*'.format(name.lower()) + else: + title = '{} Functions'.format(name) + helptag = '*api-{}*'.format(name.lower()) + sections[filename] = (title, helptag, doc) + + if not sections: + return + + docs = '' + + i = 0 + for filename in CONFIG[mode]['section_order']: + if filename not in sections: + raise RuntimeError('found new module "{}"; update the "section_order" map'.format(filename)) + title, helptag, section_doc = sections.pop(filename) + i += 1 + if filename not in CONFIG[mode]['append_only']: + docs += sep + docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) + docs += section_doc + docs += '\n\n\n' + + docs = docs.rstrip() + '\n\n' + docs += ' vim:tw=78:ts=8:ft=help:norl:\n' + + doc_file = os.path.join(base_dir, 'runtime', 'doc', + CONFIG[mode]['filename']) + + delete_lines_below(doc_file, CONFIG[mode]['section_start_token']) + with open(doc_file, 'ab') as fp: + fp.write(docs.encode('utf8')) + + shutil.rmtree(output_dir) + + +def filter_source(filename): + name, extension = os.path.splitext(filename) + if extension == '.lua': + p = subprocess.run([lua2dox_filter, filename], stdout=subprocess.PIPE) + op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8')) + print(op) + else: + """Filters the source to fix macros that confuse Doxygen.""" + with open(filename, 'rt') as fp: + print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))', + lambda m: m.group(1)+'_'.join( + re.split(r'[^\w]+', m.group(2))), + fp.read(), flags=re.M)) + + +# Doxygen Config {{{ +Doxyfile = ''' +OUTPUT_DIRECTORY = {output} +INPUT = {input} +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = {file_patterns} +RECURSIVE = YES +INPUT_FILTER = "{filter}" +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = */private/* +EXCLUDE_SYMBOLS = +EXTENSION_MAPPING = lua=C +EXTRACT_PRIVATE = NO + +GENERATE_HTML = NO +GENERATE_DOCSET = NO +GENERATE_HTMLHELP = NO +GENERATE_QHP = NO +GENERATE_TREEVIEW = NO +GENERATE_LATEX = NO +GENERATE_RTF = NO +GENERATE_MAN = NO +GENERATE_DOCBOOK = NO +GENERATE_AUTOGEN_DEF = NO + +GENERATE_XML = YES +XML_OUTPUT = xml +XML_PROGRAMLISTING = NO + +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = YES +EXPAND_ONLY_PREDEF = NO +MARKDOWN_SUPPORT = YES +''' +# }}} + +if __name__ == "__main__": + if len(sys.argv) > 1: + filter_source(sys.argv[1]) + else: + gen_docs(Doxyfile) + +# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh index c0ef2a8591..cc88ab5559 100755 --- a/scripts/genappimage.sh +++ b/scripts/genappimage.sh @@ -37,29 +37,24 @@ VERSION=$("$ROOT_DIR"/build/bin/nvim --version | head -n 1 | grep -o 'v.*') cd "$APP_BUILD_DIR" -curl -Lo "$APP_BUILD_DIR"/appimage_functions.sh https://github.com/AppImage/AppImages/raw/master/functions.sh -. ./appimage_functions.sh - -# Copy desktop and icon file to AppDir for AppRun to pick them up. -# get_apprun -# get_desktop -cp "$ROOT_DIR/runtime/nvim.desktop" "$APP_DIR/" -cp "$ROOT_DIR/runtime/nvim.png" "$APP_DIR/" +# Only downloads linuxdeploy if the remote file is different from local +if [ -e "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage ]; then + curl -Lo "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage \ + -z "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage \ + https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage +else + curl -Lo "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage \ + https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage +fi + +chmod +x "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage + +# metainfo is not packaged automatically by linuxdeploy mkdir "$APP_DIR/usr/share/metainfo/" cp "$ROOT_DIR/runtime/nvim.appdata.xml" "$APP_DIR/usr/share/metainfo/" cd "$APP_DIR" -# copy dependencies -copy_deps -# Move the libraries to usr/bin -move_lib - -# Delete stuff that should not go into the AppImage. -# Delete dangerous libraries; see -# https://github.com/AppImage/AppImages/blob/master/excludelist -delete_blacklisted - ######################################################################## # AppDir complete. Now package it as an AppImage. ######################################################################## @@ -78,21 +73,24 @@ chmod 755 AppRun cd "$APP_BUILD_DIR" # Get out of AppImage directory. +# Set the name of the file generated by appimage +export OUTPUT=nvim.appimage + +# If it's a release generate the zsync file +if [ -n "$TAG" ]; then + export UPDATE_INFORMATION="gh-releases-zsync|neovim|neovim|$TAG|nvim.appimage.zsync" +fi + # Generate AppImage. # - Expects: $ARCH, $APP, $VERSION env vars # - Expects: ./$APP.AppDir/ directory -# - Produces: ../out/$APP-$VERSION.glibc$GLIBC_NEEDED-$ARCH.AppImage -if [ -n "$TAG" ]; then - generate_type2_appimage -u "gh-releases-zsync|neovim|neovim|$TAG|nvim.appimage.zsync" -else - generate_type2_appimage -fi +# - Produces: ./nvim.appimage +./linuxdeploy-x86_64.AppImage --appdir $APP.AppDir -d $ROOT_DIR/runtime/nvim.desktop -i \ +"$ROOT_DIR/runtime/nvim.png" --output appimage # Moving the final executable to a different folder so it isn't in the # way for a subsequent build. -mv "$ROOT_DIR"/out/*.AppImage* "$ROOT_DIR"/build/bin -# Remove the (now empty) folder the AppImage was built in -rmdir "$ROOT_DIR"/out +mv "$ROOT_DIR"/build/nvim.appimage* "$ROOT_DIR"/build/bin echo 'genappimage.sh: finished' diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua new file mode 100644 index 0000000000..ae76a36d50 --- /dev/null +++ b/scripts/lua2dox.lua @@ -0,0 +1,666 @@ +--[[-------------------------------------------------------------------------- +-- Copyright (C) 2012 by Simon Dales -- +-- simon@purrsoft.co.uk -- +-- -- +-- This program is free software; you can redistribute it and/or modify -- +-- it under the terms of the GNU General Public License as published by -- +-- the Free Software Foundation; either version 2 of the License, or -- +-- (at your option) any later version. -- +-- -- +-- This program is distributed in the hope that it will be useful, -- +-- but WITHOUT ANY WARRANTY; without even the implied warranty of -- +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- +-- GNU General Public License for more details. -- +-- -- +-- You should have received a copy of the GNU General Public License -- +-- along with this program; if not, write to the -- +-- Free Software Foundation, Inc., -- +-- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -- +----------------------------------------------------------------------------]] +--[[! +\file +\brief a hack lua2dox converter +]] + +--[[! +\mainpage + +Introduction +------------ + +A hack lua2dox converter +Version 0.2 + +This lets us make Doxygen output some documentation to let +us develop this code. + +It is partially cribbed from the functionality of lua2dox +(http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm). +Found on CPAN when looking for something else; kinda handy. + +Improved from lua2dox to make the doxygen output more friendly. +Also it runs faster in lua rather than Perl. + +Because this Perl based system is called "lua2dox"., I have decided to add ".lua" to the name +to keep the two separate. + +Running +------- + +<ol> +<li> Ensure doxygen is installed on your system and that you are familiar with its use. +Best is to try to make and document some simple C/C++/PHP to see what it produces. +You can experiment with the enclosed example code. + +<li> Run "doxygen -g" to create a default Doxyfile. + +Then alter it to let it recognise lua. Add the two following lines: + +\code{.bash} +FILE_PATTERNS = *.lua + +FILTER_PATTERNS = *.lua=lua2dox_filter +\endcode + + +Either add them to the end or find the appropriate entry in Doxyfile. + +There are other lines that you might like to alter, but see futher documentation for details. + +<li> When Doxyfile is edited run "doxygen" + +The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. +It only has to be good enough for doxygen to see it as legal. +Therefore our lua interpreter is fairly limited, but "good enough". + +One limitation is that each line is treated separately (except for long comments). +The implication is that class and function declarations must be on the same line. +Some functions can have their parameter lists extended over multiple lines to make it look neat. +Managing this where there are also some comments is a bit more coding than I want to do at this stage, +so it will probably not document accurately if we do do this. + +However I have put in a hack that will insert the "missing" close paren. +The effect is that you will get the function documented, but not with the parameter list you might expect. +</ol> + +Installation +------------ + +Here for linux or unix-like, for any other OS you need to refer to other documentation. + +This file is "lua2dox.lua". It gets called by "lua2dox_filter"(bash). +Somewhere in your path (e.g. "~/bin" or "/usr/local/bin") put a link to "lua2dox_filter". + +Documentation +------------- + +Read the external documentation that should be part of this package. +For example look for the "README" and some .PDFs. + +]] + +-- we won't use our library code, so this becomes more portable + +-- require 'elijah_fix_require' +-- require 'elijah_class' +-- +--! \brief ``declare'' as class +--! +--! use as: +--! \code{.lua} +--! TWibble = class() +--! function TWibble.init(this,Str) +--! this.str = Str +--! -- more stuff here +--! end +--! \endcode +--! +function class(BaseClass, ClassInitialiser) + local newClass = {} -- a new class newClass + if not ClassInitialiser and type(BaseClass) == 'function' then + ClassInitialiser = BaseClass + BaseClass = nil + elseif type(BaseClass) == 'table' then + -- our new class is a shallow copy of the base class! + for i,v in pairs(BaseClass) do + newClass[i] = v + end + newClass._base = BaseClass + end + -- the class will be the metatable for all its newInstanceects, + -- and they will look up their methods in it. + newClass.__index = newClass + + -- expose a constructor which can be called by <classname>(<args>) + local classMetatable = {} + classMetatable.__call = + function(class_tbl, ...) + local newInstance = {} + setmetatable(newInstance,newClass) + --if init then + -- init(newInstance,...) + if class_tbl.init then + class_tbl.init(newInstance,...) + else + -- make sure that any stuff from the base class is initialized! + if BaseClass and BaseClass.init then + BaseClass.init(newInstance, ...) + end + end + return newInstance + end + newClass.init = ClassInitialiser + newClass.is_a = + function(this, klass) + local thisMetatable = getmetatable(this) + while thisMetatable do + if thisMetatable == klass then + return true + end + thisMetatable = thisMetatable._base + end + return false + end + setmetatable(newClass, classMetatable) + return newClass +end + +-- require 'elijah_clock' + +--! \class TCore_Clock +--! \brief a clock +TCore_Clock = class() + +--! \brief get the current time +function TCore_Clock.GetTimeNow() + if os.gettimeofday then + return os.gettimeofday() + else + return os.time() + end +end + +--! \brief constructor +function TCore_Clock.init(this,T0) + if T0 then + this.t0 = T0 + else + this.t0 = TCore_Clock.GetTimeNow() + end +end + +--! \brief get time string +function TCore_Clock.getTimeStamp(this,T0) + local t0 + if T0 then + t0 = T0 + else + t0 = this.t0 + end + return os.date('%c %Z',t0) +end + + +--require 'elijah_io' + +--! \class TCore_IO +--! \brief io to console +--! +--! pseudo class (no methods, just to keep documentation tidy) +TCore_IO = class() +-- +--! \brief write to stdout +function TCore_IO_write(Str) + if (Str) then + io.write(Str) + end +end + +--! \brief write to stdout +function TCore_IO_writeln(Str) + if (Str) then + io.write(Str) + end + io.write("\n") +end + + +--require 'elijah_string' + +--! \brief trims a string +function string_trim(Str) + return Str:match("^%s*(.-)%s*$") +end + +--! \brief split a string +--! +--! \param Str +--! \param Pattern +--! \returns table of string fragments +function string_split(Str, Pattern) + local splitStr = {} + local fpat = "(.-)" .. Pattern + local last_end = 1 + local str, e, cap = string.find(Str,fpat, 1) + while str do + if str ~= 1 or cap ~= "" then + table.insert(splitStr,cap) + end + last_end = e+1 + str, e, cap = string.find(Str,fpat, last_end) + end + if last_end <= #Str then + cap = string.sub(Str,last_end) + table.insert(splitStr, cap) + end + return splitStr +end + + +--require 'elijah_commandline' + +--! \class TCore_Commandline +--! \brief reads/parses commandline +TCore_Commandline = class() + +--! \brief constructor +function TCore_Commandline.init(this) + this.argv = arg + this.parsed = {} + this.params = {} +end + +--! \brief get value +function TCore_Commandline.getRaw(this,Key,Default) + local val = this.argv[Key] + if not val then + val = Default + end + return val +end + + +--require 'elijah_debug' + +------------------------------- +--! \brief file buffer +--! +--! an input file buffer +TStream_Read = class() + +--! \brief get contents of file +--! +--! \param Filename name of file to read (or nil == stdin) +function TStream_Read.getContents(this,Filename) + -- get lines from file + local filecontents + if Filename then + -- syphon lines to our table + --TCore_Debug_show_var('Filename',Filename) + filecontents={} + for line in io.lines(Filename) do + table.insert(filecontents,line) + end + else + -- get stuff from stdin as a long string (with crlfs etc) + filecontents=io.read('*a') + -- make it a table of lines + filecontents = TString_split(filecontents,'[\n]') -- note this only works for unix files. + Filename = 'stdin' + end + + if filecontents then + this.filecontents = filecontents + this.contentsLen = #filecontents + this.currentLineNo = 1 + end + + return filecontents +end + +--! \brief get lineno +function TStream_Read.getLineNo(this) + return this.currentLineNo +end + +--! \brief get a line +function TStream_Read.getLine(this) + local line + if this.currentLine then + line = this.currentLine + this.currentLine = nil + else + -- get line + if this.currentLineNo<=this.contentsLen then + line = this.filecontents[this.currentLineNo] + this.currentLineNo = this.currentLineNo + 1 + else + line = '' + end + end + return line +end + +--! \brief save line fragment +function TStream_Read.ungetLine(this,LineFrag) + this.currentLine = LineFrag +end + +--! \brief is it eof? +function TStream_Read.eof(this) + if this.currentLine or this.currentLineNo<=this.contentsLen then + return false + end + return true +end + +--! \brief output stream +TStream_Write = class() + +--! \brief constructor +function TStream_Write.init(this) + this.tailLine = {} +end + +--! \brief write immediately +function TStream_Write.write(this,Str) + TCore_IO_write(Str) +end + +--! \brief write immediately +function TStream_Write.writeln(this,Str) + TCore_IO_writeln(Str) +end + +--! \brief write immediately +function TStream_Write.writelnComment(this,Str) + TCore_IO_write('// ZZ: ') + TCore_IO_writeln(Str) +end + +--! \brief write to tail +function TStream_Write.writelnTail(this,Line) + if not Line then + Line = '' + end + table.insert(this.tailLine,Line) +end + +--! \brief outout tail lines +function TStream_Write.write_tailLines(this) + for k,line in ipairs(this.tailLine) do + TCore_IO_writeln(line) + end + TCore_IO_write('// Lua2DoX new eof') +end + +--! \brief input filter +TLua2DoX_filter = class() + +--! \brief allow us to do errormessages +function TLua2DoX_filter.warning(this,Line,LineNo,Legend) + this.outStream:writelnTail( + '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"' + ) +end + +--! \brief trim comment off end of string +--! +--! If the string has a comment on the end, this trims it off. +--! +local function TString_removeCommentFromLine(Line) + local pos_comment = string.find(Line,'%-%-') + local tailComment + if pos_comment then + Line = string.sub(Line,1,pos_comment-1) + tailComment = string.sub(Line,pos_comment) + end + return Line,tailComment +end + +--! \brief get directive from magic +local function getMagicDirective(Line) + local macro,tail + local macroStr = '[\\@]' + local pos_macro = string.find(Line,macroStr) + if pos_macro then + --! ....\\ macro...stuff + --! ....\@ macro...stuff + local line = string.sub(Line,pos_macro+1) + local space = string.find(line,'%s+') + if space then + macro = string.sub(line,1,space-1) + tail = string_trim(string.sub(line,space+1)) + else + macro = line + tail = '' + end + end + return macro,tail +end + +--! \brief check comment for fn +local function checkComment4fn(Fn_magic,MagicLines) + local fn_magic = Fn_magic + -- TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"') + + local magicLines = string_split(MagicLines,'\n') + + local macro,tail + + for k,line in ipairs(magicLines) do + macro,tail = getMagicDirective(line) + if macro == 'fn' then + fn_magic = tail + -- TCore_IO_writeln('// found fn "' .. fn_magic .. '"') + else + --TCore_IO_writeln('// not found fn "' .. line .. '"') + end + end + + return fn_magic +end +--! \brief run the filter +function TLua2DoX_filter.readfile(this,AppStamp,Filename) + local err + + local inStream = TStream_Read() + local outStream = TStream_Write() + this.outStream = outStream -- save to this obj + + if (inStream:getContents(Filename)) then + -- output the file + local line + local fn_magic -- function name/def from magic comment + + outStream:writelnTail('// #######################') + outStream:writelnTail('// app run:' .. AppStamp) + outStream:writelnTail('// #######################') + outStream:writelnTail() + + while not (err or inStream:eof()) do + line = string_trim(inStream:getLine()) + -- TCore_Debug_show_var('inStream',inStream) + -- TCore_Debug_show_var('line',line ) + if string.sub(line,1,2)=='--' then -- its a comment + if string.sub(line,3,3)=='@' then -- it's a magic comment + local magic = string.sub(line,4) + outStream:writeln('/// @' .. magic) + fn_magic = checkComment4fn(fn_magic,magic) + elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment + local comment = string.sub(line,4) + outStream:writeln('/// '.. comment) + elseif string.sub(line,3,4)=='[[' then -- it's a long comment + line = string.sub(line,5) -- nibble head + local comment = '' + local closeSquare,hitend,thisComment + while (not err) and (not hitend) and (not inStream:eof()) do + closeSquare = string.find(line,']]') + if not closeSquare then -- need to look on another line + thisComment = line .. '\n' + line = inStream:getLine() + else + thisComment = string.sub(line,1,closeSquare-1) + hitend = true + + -- unget the tail of the line + -- in most cases it's empty. This may make us less efficient but + -- easier to program + inStream:ungetLine(string_trim(string.sub(line,closeSquare+2))) + end + comment = comment .. thisComment + end + if string.sub(comment,1,1)=='@' then -- it's a long magic comment + outStream:write('/*' .. comment .. '*/ ') + fn_magic = checkComment4fn(fn_magic,comment) + else -- discard + outStream:write('/* zz:' .. comment .. '*/ ') + fn_magic = nil + end + else + outStream:writeln('// zz:"' .. line .. '"') + fn_magic = nil + end + elseif string.find(line,'^function') or string.find(line,'^local%s+function') then + -- it's a function + local pos_fn = string.find(line,'function') + -- function + -- ....v... + if pos_fn then + -- we've got a function + local fn_type + if string.find(line,'^local%s+') then + fn_type = ''--'static ' -- static functions seem to be excluded + else + fn_type = '' + end + local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8))) + if fn_magic then + fn = fn_magic + fn_magic = nil + end + + if string.sub(fn,1,1)=='(' then + -- it's an anonymous function + outStream:writelnComment(line) + else + -- fn has a name, so is interesting + + -- want to fix for iffy declarations + local open_paren = string.find(fn,'[%({]') + local fn0 = fn + if open_paren then + fn0 = string.sub(fn,1,open_paren-1) + -- we might have a missing close paren + if not string.find(fn,'%)') then + fn = fn .. ' ___MissingCloseParenHere___)' + end + end + + local dot = string.find(fn0,'[%.:]') + if dot then -- it's a method + local klass = string.sub(fn,1,dot-1) + local method = string.sub(fn,dot+1) + --TCore_IO_writeln('function ' .. klass .. '::' .. method .. ftail .. '{}') + --TCore_IO_writeln(klass .. '::' .. method .. ftail .. '{}') + outStream:writeln( + '/*! \\memberof ' .. klass .. ' */ ' + .. method .. '{}' + ) + else + -- add vanilla function + + outStream:writeln(fn_type .. 'function ' .. fn .. '{}') + end + end + else + this:warning(inStream:getLineNo(),'something weird here') + end + fn_magic = nil -- mustn't indavertently use it again + elseif string.find(line,'=%s*class%(') then + -- it's a class declaration + local tailComment + line,tailComment = TString_removeCommentFromLine(line) + local equals = string.find(line,'=') + local klass = string_trim(string.sub(line,1,equals-1)) + local tail = string_trim(string.sub(line,equals+1)) + -- class(wibble wibble) + -- ....v. + local parent = string.sub(tail,7,-2) + if #parent>0 then + parent = ' :public ' .. parent + end + outStream:writeln('class ' .. klass .. parent .. '{};') + else + -- we don't know what this line means, so we can probably just comment it out + if #line>0 then + outStream:writeln('// zz: ' .. line) + else + outStream:writeln() -- keep this line blank + end + end + end + + -- output the tail + outStream:write_tailLines() + else + outStream:writeln('!empty file') + end +end + +--! \brief this application +TApp = class() + +--! \brief constructor +function TApp.init(this) + local t0 = TCore_Clock() + this.timestamp = t0:getTimeStamp() + this.name = 'Lua2DoX' + this.version = '0.2 20130128' + this.copyright = 'Copyright (c) Simon Dales 2012-13' +end + +function TApp.getRunStamp(this) + return this.name .. ' (' .. this.version .. ') ' + .. this.timestamp +end + +function TApp.getVersion(this) + return this.name .. ' (' .. this.version .. ') ' +end + +function TApp.getCopyright(this) + return this.copyright +end + +local This_app = TApp() + +--main +local cl = TCore_Commandline() + +local argv1 = cl:getRaw(2) +if argv1 == '--help' then + TCore_IO_writeln(This_app:getVersion()) + TCore_IO_writeln(This_app:getCopyright()) + TCore_IO_writeln([[ + run as: + lua2dox_filter <param> + -------------- + Param: + <filename> : interprets filename + --version : show version/copyright info + --help : this help text]]) +elseif argv1 == '--version' then + TCore_IO_writeln(This_app:getVersion()) + TCore_IO_writeln(This_app:getCopyright()) +else + -- it's a filter + local appStamp = This_app:getRunStamp() + local filename = argv1 + + local filter = TLua2DoX_filter() + filter:readfile(appStamp,filename) +end + + +--eof diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter new file mode 100755 index 0000000000..6cb16ef060 --- /dev/null +++ b/scripts/lua2dox_filter @@ -0,0 +1,87 @@ +#!/bin/bash + +########################################################################### +# Copyright (C) 2012 by Simon Dales # +# simon@purrsoft.co.uk # +# # +# This program is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 2 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program; if not, write to the # +# Free Software Foundation, Inc., # +# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # +########################################################################### +LANG="" + +##! \brief test executable to see if it exists +test_executable(){ + P_EXE="$1" + ######### + WHICH=`which ${P_EXE}` + if test -z "${WHICH}" + then + echo "not found \"${P_EXE}\"" + else + EXE="${P_EXE}" + fi + } + +##! \brief sets the lua interpreter +set_lua(){ + test_executable 'texlua' + if test -z "${EXE}" + then + test_executable 'lua' + fi + #echo "final EXE=\"${EXE}\"" + } + +##! \brief makes canonical name of file +##! +##! Note that "readlink -f" doesn't work in MacOSX +##! +do_readlink(){ + pushd . > /dev/null + TARGET_FILE=$1 + + cd `dirname $TARGET_FILE` + TARGET_FILE=`basename $TARGET_FILE` + + # Iterate down a (possible) chain of symlinks + while [ -L "$TARGET_FILE" ] + do + TARGET_FILE=`readlink $TARGET_FILE` + cd `dirname $TARGET_FILE` + TARGET_FILE=`basename $TARGET_FILE` + done + + PHYS_DIR=`pwd -P` + RESULT=$PHYS_DIR + popd > /dev/null + } + +##main +set_lua +if test -z "${EXE}" +then + echo "no lua interpreter available" +else + BASENAME=`basename "$0"` + do_readlink "$0" + DIRNAME="${RESULT}" + + LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}" + #echo "lua[${LUASCRIPT}]" + + ${EXE} ${LUASCRIPT} $@ +fi +# +##eof diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 7339fdc691..eeb77dbbdc 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -357,13 +357,12 @@ list_vim_commits() { ( # Prints all (sorted) "vim-patch:xxx" tokens found in the Nvim git log. list_vimpatch_tokens() { - local tokens - # Find all "vim-patch:xxx" tokens in the Nvim git log. - tokens="$(cd "${NVIM_SOURCE_DIR}" && git log -E --grep='vim-patch:[^ ]+' | grep 'vim-patch')" - echo "$tokens" | grep -E 'vim-patch:[^ ,{]{7,}' \ - | sed 's/.*\(vim-patch:[.0-9a-z]\+\).*/\1/' \ + # Use sed…{7,7} to normalize (internal) Git hashes (for tokens caches). + git -C "${NVIM_SOURCE_DIR}" log -E --grep='vim-patch:[^ ,{]{7,}' \ + | grep -oE 'vim-patch:[^ ,{:]{7,}' \ | sort \ - | uniq + | uniq \ + | sed -nE 's/^(vim-patch:([0-9]+\.[^ ]+|[0-9a-z]{7,7})).*/\1/p' } # Prints all patch-numbers (for the current v:version) for which there is @@ -377,29 +376,31 @@ list_vimpatch_numbers() { # Prints a newline-delimited list of Vim commits, for use by scripts. list_missing_vimpatches() { - local tokens vim_commit vim_commits is_missing vim_tag patch_number + local token vim_commit vim_tag patch_number + declare -A tokens # Find all "vim-patch:xxx" tokens in the Nvim git log. - tokens="$(list_vimpatch_tokens)" + for token in $(list_vimpatch_tokens); do + tokens[$token]=1 + done # Get missing Vim commits - vim_commits="$(list_vim_commits)" - for vim_commit in ${vim_commits}; do + for vim_commit in $(list_vim_commits); do # Check for vim-patch:<commit_hash> (usually runtime updates). - is_missing="$(echo "$tokens" | >/dev/null 2>&1 grep "vim\-patch:${vim_commit:0:7}" && echo false || echo true)" - - if ! [ "$is_missing" = "false" ] \ - && vim_tag="$(cd "${VIM_SOURCE_DIR}" && git describe --tags --exact-match "${vim_commit}" 2>/dev/null)" - then - # Vim version number (not commit hash). - # Check for vim-patch:<tag> (not commit hash). - patch_number="${vim_tag:1}" # "v7.4.0001" => "7.4.0001" - is_missing="$(echo "$tokens" | >/dev/null 2>&1 grep "vim\-patch:${patch_number}" && echo false || echo true)" - vim_commit="${vim_tag#v}" + token="vim-patch:${vim_commit:0:7}" + if [[ "${tokens[$token]-}" ]]; then + continue fi - if ! [ "$is_missing" = "false" ]; then - echo "${vim_commit}" + if vim_tag="$(git -C "${VIM_SOURCE_DIR}" describe --tags --exact-match "${vim_commit}" 2>/dev/null)"; then + # Check for vim-patch:<tag> (not commit hash). + patch_number="vim-patch:${vim_tag:1}" # "v7.4.0001" => "7.4.0001" + if [[ "${tokens[$patch_number]-}" ]]; then + continue + fi + echo "$vim_tag" + else + echo "$vim_commit" fi done } @@ -410,7 +411,7 @@ show_vimpatches() { printf "\nVim patches missing from Neovim:\n" list_missing_vimpatches | while read vim_commit; do - if (cd "${VIM_SOURCE_DIR}" && git --no-pager show --color=never --name-only "v${vim_commit}" 2>/dev/null) | grep -q ^runtime; then + if [ -n "$(git -C "$VIM_SOURCE_DIR" diff-tree --no-commit-id --name-only "${vim_commit}" -- runtime)" ]; then printf ' • %s (+runtime)\n' "${vim_commit}" else printf ' • %s\n' "${vim_commit}" |