diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/gen_vimdoc.py | 795 | ||||
| -rw-r--r-- | scripts/lua2dox.lua | 131 | ||||
| -rwxr-xr-x | scripts/lua2dox_filter | 11 | ||||
| -rwxr-xr-x | scripts/release.sh | 6 | ||||
| -rwxr-xr-x | scripts/shadacat.py | 2 | ||||
| -rwxr-xr-x | scripts/update-ts-runtime.sh | 39 | ||||
| -rwxr-xr-x | scripts/update_version_stamp.lua | 55 | ||||
| -rwxr-xr-x | scripts/vim-patch.sh | 226 |
8 files changed, 827 insertions, 438 deletions
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 373a58d11e..a61690e99f 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -1,5 +1,18 @@ #!/usr/bin/env python3 -"""Generates Nvim help docs from C docstrings, by parsing Doxygen XML. +"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen. + +Also generates *.mpack files. To inspect the *.mpack structure: + + :new | put=v:lua.vim.inspect(msgpackparse(readfile('runtime/doc/api.mpack'))) + + +Flow: + main + extract_from_xml + fmt_node_as_vimhelp \ + para_as_map } recursive + update_params_map / + render_node This would be easier using lxml and XSLT, but: @@ -9,25 +22,22 @@ This would be easier using lxml and XSLT, but: 2. I wouldn't know how to deal with nested indentation in <para> tags using XSLT. -Each function documentation is formatted with the following rules: +Each function :help block is formatted as follows: - - Maximum width of 78 characters (`text_width`). - - Spaces for indentation. - - Function signature and helptag are on the same line. - - Helptag is right aligned. + - Max width of 78 columns (`text_width`). + - Indent with spaces (not tabs). + - Indent of 16 columns for body text. + - Function signature and helptag (right-aligned) on the same line. - Signature and helptag must have a minimum of 8 spaces between them. - - If the signature is too long, it is placed on the line after the - helptag. The signature wraps at `text_width - 8` characters with - subsequent lines indented to the open parenthesis. - - Documentation body will be indented by 16 spaces. + - If the signature is too long, it is placed on the line after the helptag. + Signature wraps at `text_width - 8` characters with subsequent + lines indented to the open parenthesis. - Subsection bodies are indented an additional 4 spaces. - - Documentation body consists of the function description, parameter details, - return description, and C declaration. + - Body consists of function description, parameters, return description, and + C declaration (`INCLUDE_C_DECL`). - Parameters are omitted for the `void` and `Error *` types, or if the parameter is marked as [out]. - Each function documentation is separated by a single line. - -The C declaration is added to the end to show actual argument types. """ import os import re @@ -36,27 +46,33 @@ import shutil import textwrap import subprocess import collections +import msgpack from xml.dom import minidom -if sys.version_info[0] < 3: - print("use Python 3") +MIN_PYTHON_VERSION = (3, 5) + +if sys.version_info < MIN_PYTHON_VERSION: + print("requires Python {}.{}+".format(*MIN_PYTHON_VERSION)) sys.exit(1) DEBUG = ('DEBUG' in os.environ) +TARGET = os.environ.get('TARGET', None) INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ) INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ) +fmt_vimhelp = False # HACK text_width = 78 script_path = os.path.abspath(__file__) base_dir = os.path.dirname(os.path.dirname(script_path)) -out_dir = os.path.join(base_dir, 'tmp-{mode}-doc') +out_dir = os.path.join(base_dir, 'tmp-{target}-doc') filter_cmd = '%s %s' % (sys.executable, script_path) seen_funcs = set() lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter') CONFIG = { 'api': { + 'mode': 'c', 'filename': 'api.txt', # String used to find the start of the generated part of the doc. 'section_start_token': '*api-global*', @@ -73,18 +89,25 @@ CONFIG = { # file patterns used by doxygen 'file_patterns': '*.h *.c', # Only function with this prefix are considered - 'func_name_prefix': 'nvim_', + 'fn_name_prefix': 'nvim_', # Section name overrides. 'section_name': { 'vim.c': 'Global', }, + # For generated section names. + 'section_fmt': lambda name: f'{name} Functions', + # Section helptag. + 'helptag_fmt': lambda name: f'*api-{name.lower()}*', + # Per-function helptag. + 'fn_helptag_fmt': lambda fstem, name: f'*{name}()*', # Module name overrides (for Lua). 'module_override': {}, # Append the docs for these modules, do not start a new section. 'append_only': [], }, 'lua': { - 'filename': 'if_lua.txt', + 'mode': 'lua', + 'filename': 'lua.txt', 'section_start_token': '*lua-vim*', 'section_order': [ 'vim.lua', @@ -95,8 +118,13 @@ CONFIG = { os.path.join(base_dir, 'runtime/lua/vim/shared.lua'), ]), 'file_patterns': '*.lua', - 'func_name_prefix': '', - 'section_name': {}, + 'fn_name_prefix': '', + 'section_name': { + 'lsp.lua': 'core', + }, + 'section_fmt': lambda name: f'Lua module: {name.lower()}', + 'helptag_fmt': lambda name: f'*lua-{name.lower()}*', + 'fn_helptag_fmt': lambda fstem, name: f'*{fstem}.{name}()*', 'module_override': { # `shared` functions are exposed on the `vim` module. 'shared': 'vim', @@ -105,6 +133,45 @@ CONFIG = { 'shared.lua', ], }, + 'lsp': { + 'mode': 'lua', + 'filename': 'lsp.txt', + 'section_start_token': '*lsp-core*', + 'section_order': [ + 'lsp.lua', + 'protocol.lua', + 'buf.lua', + 'callbacks.lua', + 'log.lua', + 'rpc.lua', + 'util.lua' + ], + 'files': ' '.join([ + os.path.join(base_dir, 'runtime/lua/vim/lsp'), + os.path.join(base_dir, 'runtime/lua/vim/lsp.lua'), + ]), + 'file_patterns': '*.lua', + 'fn_name_prefix': '', + 'section_name': {}, + 'section_fmt': lambda name: ( + 'Lua module: vim.lsp' + if name.lower() == 'lsp' + else f'Lua module: vim.lsp.{name.lower()}'), + 'helptag_fmt': lambda name: ( + '*lsp-core*' + if name.lower() == 'lsp' + else f'*lsp-{name.lower()}*'), + 'fn_helptag_fmt': lambda fstem, name: ( + f'*vim.lsp.{name}()*' + if fstem == 'lsp' and name != 'client' + else ( + '*vim.lsp.client*' + # HACK. TODO(justinmk): class/structure support in lua2dox + if 'lsp.client' == f'{fstem}.{name}' + else f'*vim.lsp.{fstem}.{name}()*')), + 'module_override': {}, + 'append_only': [], + }, } param_exclude = ( @@ -122,14 +189,22 @@ annotation_map = { xrefs = set() -def debug_this(s, n): - o = n if isinstance(n, str) else n.toprettyxml(indent=' ', newl='\n') - name = '' if isinstance(n, str) else n.nodeName - if s in o: +# Raises an error with details about `o`, if `cond` is in object `o`, +# or if `cond()` is callable and returns True. +def debug_this(cond, o): + name = '' + if not isinstance(o, str): + try: + name = o.nodeName + o = o.toprettyxml(indent=' ', newl='\n') + except Exception: + pass + if ((callable(cond) and cond()) + or (not callable(cond) and cond) + or (not callable(cond) and cond in o)): raise RuntimeError('xxx: {}\n{}'.format(name, o)) -# XML Parsing Utilities {{{ def find_first(parent, name): """Finds the first matching node within parent.""" sub = parent.getElementsByTagName(name) @@ -138,20 +213,27 @@ def find_first(parent, name): return sub[0] -def get_children(parent, name): - """Yield matching child nodes within parent.""" +def iter_children(parent, name): + """Yields matching child nodes within parent.""" for child in parent.childNodes: if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: yield child def get_child(parent, name): - """Get the first matching child node.""" - for child in get_children(parent, name): + """Gets the first matching child node.""" + for child in iter_children(parent, name): return child return None +def self_or_child(n): + """Gets the first child node, or self.""" + if len(n.childNodes) == 0: + return n + return n.childNodes[0] + + def clean_text(text): """Cleans text. @@ -172,18 +254,21 @@ def is_blank(text): return '' == clean_lines(text) -def get_text(parent, preformatted=False): - """Combine all text in a node.""" - if parent.nodeType == parent.TEXT_NODE: - return parent.data - - out = '' - for node in parent.childNodes: +def get_text(n, preformatted=False): + """Recursively concatenates all text in a node tree.""" + text = '' + if n.nodeType == n.TEXT_NODE: + return n.data + if n.nodeName == 'computeroutput': + for node in n.childNodes: + text += get_text(node) + return '`{}` '.format(text) + for node in n.childNodes: if node.nodeType == node.TEXT_NODE: - out += node.data if preformatted else clean_text(node.data) + text += node.data if preformatted else clean_text(node.data) elif node.nodeType == node.ELEMENT_NODE: - out += ' ' + get_text(node, preformatted) - return out + text += ' ' + get_text(node, preformatted) + return text # Gets the length of the last line in `text`, excluding newline ("\n") char. @@ -203,6 +288,8 @@ def len_lastline_withoutindent(text, indent): # Returns True if node `n` contains only inline (not block-level) elements. def is_inline(n): + # if len(n.childNodes) == 0: + # return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput' for c in n.childNodes: if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput': return False @@ -253,83 +340,78 @@ def doc_wrap(text, prefix='', width=70, func=False, indent=None): return result -def has_nonexcluded_params(nodes): - """Returns true if any of the given <parameterlist> elements has at least - one non-excluded item.""" - for n in nodes: - if render_params(n) != '': - return True +def max_name(names): + if len(names) == 0: + return 0 + return max(len(name) for name in names) -def render_params(parent, width=62): - """Renders Doxygen <parameterlist> tag as Vim help text.""" - name_length = 0 - items = [] +def update_params_map(parent, ret_map, width=62): + """Updates `ret_map` with name:desc key-value pairs extracted + from Doxygen XML node `parent`. + """ + params = collections.OrderedDict() for node in parent.childNodes: if node.nodeType == node.TEXT_NODE: continue - name_node = find_first(node, 'parametername') if name_node.getAttribute('direction') == 'out': continue - name = get_text(name_node) if name in param_exclude: continue - - name = '{%s}' % name - name_length = max(name_length, len(name) + 2) - items.append((name.strip(), node)) - - out = '' - for name, node in items: - name = ' {}'.format(name.ljust(name_length)) - + params[name.strip()] = node + max_name_len = max_name(params.keys()) + 8 + # `ret_map` is a name:desc map. + for name, node in params.items(): desc = '' desc_node = get_child(node, 'parameterdescription') if desc_node: - desc = parse_parblock(desc_node, width=width, - indent=(' ' * len(name))) - - out += '{}{}\n'.format(name, desc) - return out.rstrip() + desc = fmt_node_as_vimhelp( + desc_node, width=width, indent=(' ' * max_name_len)) + ret_map[name] = desc + return ret_map def render_node(n, text, prefix='', indent='', width=62): """Renders a node as Vim help text, recursively traversing all descendants.""" + global fmt_vimhelp + global has_seen_preformatted + + def ind(s): + return s if fmt_vimhelp else '' + text = '' # space_preceding = (len(text) > 0 and ' ' == text[-1][-1]) # text += (int(not space_preceding) * ' ') - if n.nodeType == n.TEXT_NODE: - # `prefix` is NOT sent to doc_wrap, it was already handled by now. - text += doc_wrap(n.data, indent=indent, width=width) - elif n.nodeName == 'computeroutput': - text += ' `{}` '.format(get_text(n)) - elif n.nodeName == 'preformatted': + if n.nodeName == 'preformatted': o = get_text(n, preformatted=True) ensure_nl = '' if o[-1] == '\n' else '\n' - text += ' >{}{}\n<'.format(ensure_nl, o) + text += '>{}{}\n<'.format(ensure_nl, o) + elif is_inline(n): - for c in n.childNodes: - text += render_node(c, text) - text = doc_wrap(text, indent=indent, width=width) + text = doc_wrap(get_text(n), indent=indent, width=width) elif n.nodeName == 'verbatim': # TODO: currently we don't use this. The "[verbatim]" hint is there as # a reminder that we must decide how to format this if we do use it. text += ' [verbatim] {}'.format(get_text(n)) elif n.nodeName == 'listitem': for c in n.childNodes: - text += ( - indent - + prefix - + render_node(c, text, indent=indent + (' ' * len(prefix)), width=width) + result = render_node( + c, + text, + indent=indent + (' ' * len(prefix)), + width=width ) + + if is_blank(result): + continue + + text += indent + prefix + result elif n.nodeName in ('para', 'heading'): for c in n.childNodes: text += render_node(c, text, indent=indent, width=width) - if is_inline(n): - text = doc_wrap(text, indent=indent, width=width) elif n.nodeName == 'itemizedlist': for c in n.childNodes: text += '{}\n'.format(render_node(c, text, prefix='• ', @@ -355,23 +437,33 @@ def render_node(n, text, prefix='', indent='', width=62): text += '\n' elif (n.nodeName == 'simplesect' and n.getAttribute('kind') in ('return', 'see')): - text += ' ' + text += ind(' ') for c in n.childNodes: text += render_node(c, text, indent=' ', width=width) else: raise RuntimeError('unhandled node type: {}\n{}'.format( n.nodeName, n.toprettyxml(indent=' ', newl='\n'))) + return text -def render_para(parent, indent='', width=62): - """Renders Doxygen <para> containing arbitrary nodes. +def para_as_map(parent, indent='', width=62): + """Extracts a Doxygen XML <para> node to a map. - NB: Blank lines in a docstring manifest as <para> tags. + Keys: + 'text': Text from this <para> element + 'params': <parameterlist> map + 'return': List of @return strings + 'seealso': List of @see strings + 'xrefs': ? """ - if is_inline(parent): - return clean_lines(doc_wrap(render_node(parent, ''), - indent=indent, width=width).strip()) + chunks = { + 'text': '', + 'params': collections.OrderedDict(), + 'return': [], + 'seealso': [], + 'xrefs': [] + } # Ordered dict of ordered lists. groups = collections.OrderedDict([ @@ -386,75 +478,125 @@ def render_para(parent, indent='', width=62): text = '' kind = '' last = '' - for child in parent.childNodes: - if child.nodeName == 'parameterlist': - groups['params'].append(child) - elif child.nodeName == 'xrefsect': - groups['xrefs'].append(child) - elif child.nodeName == 'simplesect': - last = kind - kind = child.getAttribute('kind') - if kind == 'return' or (kind == 'note' and last == 'return'): - groups['return'].append(child) - elif kind == 'see': - groups['seealso'].append(child) - elif kind in ('note', 'warning'): - text += render_node(child, text, indent=indent, width=width) + if is_inline(parent): + # Flatten inline text from a tree of non-block nodes. + text = doc_wrap(render_node(parent, ""), indent=indent, width=width) + else: + prev = None # Previous node + for child in parent.childNodes: + if child.nodeName == 'parameterlist': + groups['params'].append(child) + elif child.nodeName == 'xrefsect': + groups['xrefs'].append(child) + elif child.nodeName == 'simplesect': + last = kind + kind = child.getAttribute('kind') + if kind == 'return' or (kind == 'note' and last == 'return'): + groups['return'].append(child) + elif kind == 'see': + groups['seealso'].append(child) + elif kind in ('note', 'warning'): + text += render_node(child, text, indent=indent, width=width) + else: + raise RuntimeError('unhandled simplesect: {}\n{}'.format( + child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) else: - raise RuntimeError('unhandled simplesect: {}\n{}'.format( - child.nodeName, child.toprettyxml(indent=' ', newl='\n'))) - else: - text += render_node(child, text, indent=indent, width=width) + if (prev is not None + and is_inline(self_or_child(prev)) + and is_inline(self_or_child(child)) + and '' != get_text(self_or_child(child)).strip() + and ' ' != text[-1]): + text += ' ' + + text += render_node(child, text, indent=indent, width=width) + prev = child + + chunks['text'] += text - chunks = [text] - # Generate text from the gathered items. - if len(groups['params']) > 0 and has_nonexcluded_params(groups['params']): - chunks.append('\nParameters: ~') + # Generate map from the gathered items. + if len(groups['params']) > 0: for child in groups['params']: - chunks.append(render_params(child, width=width)) - if len(groups['return']) > 0: - chunks.append('\nReturn: ~') - for child in groups['return']: - chunks.append(render_node( - child, chunks[-1][-1], indent=indent, width=width)) - if len(groups['seealso']) > 0: - chunks.append('\nSee also: ~') - for child in groups['seealso']: - chunks.append(render_node( - child, chunks[-1][-1], indent=indent, width=width)) + update_params_map(child, ret_map=chunks['params'], width=width) + for child in groups['return']: + chunks['return'].append(render_node( + child, '', indent=indent, width=width)) + for child in groups['seealso']: + chunks['seealso'].append(render_node( + child, '', indent=indent, width=width)) for child in groups['xrefs']: - title = get_text(get_child(child, 'xreftitle')) + # XXX: Add a space (or any char) to `title` here, otherwise xrefs + # ("Deprecated" section) acts very weird... + title = get_text(get_child(child, 'xreftitle')) + ' ' xrefs.add(title) - xrefdesc = render_para(get_child(child, 'xrefdescription'), width=width) - chunks.append(doc_wrap(xrefdesc, prefix='{}: '.format(title), - width=width) + '\n') + xrefdesc = get_text(get_child(child, 'xrefdescription')) + chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title), + width=width) + '\n') - return clean_lines('\n'.join(chunks).strip()) + return chunks -def parse_parblock(parent, prefix='', width=62, indent=''): - """Renders a nested block of <para> tags as Vim help text.""" - paragraphs = [] - for child in parent.childNodes: - paragraphs.append(render_para(child, width=width, indent=indent)) - paragraphs.append('') - return clean_lines('\n'.join(paragraphs).strip()) -# }}} - - -def parse_source_xml(filename, mode): - """Collects API functions. +def fmt_node_as_vimhelp(parent, width=62, indent=''): + """Renders (nested) Doxygen <para> nodes as Vim :help text. - Returns two strings: - 1. API functions - 2. Deprecated API functions + NB: Blank lines in a docstring manifest as <para> tags. + """ + rendered_blocks = [] + + def fmt_param_doc(m): + """Renders a params map as Vim :help text.""" + max_name_len = max_name(m.keys()) + 4 + out = '' + for name, desc in m.items(): + name = ' {}'.format('{{{}}}'.format(name).ljust(max_name_len)) + out += '{}{}\n'.format(name, desc) + return out.rstrip() + + def has_nonexcluded_params(m): + """Returns true if any of the given params has at least + one non-excluded item.""" + if fmt_param_doc(m) != '': + return True - Caller decides what to do with the deprecated documentation. + for child in parent.childNodes: + para = para_as_map(child, indent, width) + + # Generate text from the gathered items. + chunks = [para['text']] + if len(para['params']) > 0 and has_nonexcluded_params(para['params']): + chunks.append('\nParameters: ~') + chunks.append(fmt_param_doc(para['params'])) + if len(para['return']) > 0: + chunks.append('\nReturn: ~') + for s in para['return']: + chunks.append(s) + if len(para['seealso']) > 0: + chunks.append('\nSee also: ~') + for s in para['seealso']: + chunks.append(s) + for s in para['xrefs']: + chunks.append(s) + + rendered_blocks.append(clean_lines('\n'.join(chunks).strip())) + rendered_blocks.append('') + + return clean_lines('\n'.join(rendered_blocks).strip()) + + +def extract_from_xml(filename, target, width): + """Extracts Doxygen info as maps without formatting the text. + + Returns two maps: + 1. Functions + 2. Deprecated functions + + The `fmt_vimhelp` global controls some special cases for use by + fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :) """ global xrefs - xrefs = set() - functions = [] - deprecated_functions = [] + global fmt_vimhelp + xrefs.clear() + fns = {} # Map of func_name:docstring. + deprecated_fns = {} # Map of func_name:docstring. dom = minidom.parse(filename) compoundname = get_text(dom.getElementsByTagName('compoundname')[0]) @@ -489,31 +631,33 @@ def parse_source_xml(filename, mode): annotations = filter(None, map(lambda x: annotation_map.get(x), annotations.split())) - if mode == 'lua': - fstem = compoundname.split('.')[0] - fstem = CONFIG[mode]['module_override'].get(fstem, fstem) - vimtag = '*{}.{}()*'.format(fstem, name) + if not fmt_vimhelp: + pass else: - vimtag = '*{}()*'.format(name) + fstem = '?' + if '.' in compoundname: + fstem = compoundname.split('.')[0] + fstem = CONFIG[target]['module_override'].get(fstem, fstem) + vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name) params = [] type_length = 0 - for param in get_children(member, 'param'): + for param in iter_children(member, 'param'): param_type = get_text(get_child(param, 'type')).strip() param_name = '' declname = get_child(param, 'declname') if declname: param_name = get_text(declname).strip() - elif mode == 'lua': - # that's how it comes out of lua2dox + elif CONFIG[target]['mode'] == 'lua': + # XXX: this is what lua2dox gives us... param_name = param_type param_type = '' if param_name in param_exclude: continue - if param_type.endswith('*'): + if fmt_vimhelp and param_type.endswith('*'): param_type = param_type.strip('* ') param_name = '*' + param_name type_length = max(type_length, len(param_type)) @@ -521,41 +665,106 @@ def parse_source_xml(filename, mode): c_args = [] for param_type, param_name in params: - c_args.append(' ' + ( + c_args.append((' ' if fmt_vimhelp else '') + ( '%s %s' % (param_type.ljust(type_length), param_name)).strip()) - c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, - ',\n'.join(c_args)), - ' ') - prefix = '%s(' % name suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params if a[0] not in ('void', 'Error')) + if not fmt_vimhelp: + c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args)) + signature = prefix + suffix + else: + c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, + ',\n'.join(c_args)), + ' ') - # Minimum 8 chars between signature and vimtag - lhs = (text_width - 8) - len(prefix) + # Minimum 8 chars between signature and vimtag + lhs = (width - 8) - len(vimtag) - if len(prefix) + len(suffix) > lhs: - signature = vimtag.rjust(text_width) + '\n' - signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, - func=True) - else: - signature = prefix + suffix - signature += vimtag.rjust(text_width - len(signature)) + if len(prefix) + len(suffix) > lhs: + signature = vimtag.rjust(width) + '\n' + signature += doc_wrap(suffix, width=width-8, prefix=prefix, + func=True) + else: + signature = prefix + suffix + signature += vimtag.rjust(width - len(signature)) + + paras = [] + brief_desc = find_first(member, 'briefdescription') + if brief_desc: + for child in brief_desc.childNodes: + paras.append(para_as_map(child)) - doc = '' desc = find_first(member, 'detaileddescription') if desc: - doc = parse_parblock(desc) + for child in desc.childNodes: + paras.append(para_as_map(child)) if DEBUG: print(textwrap.indent( re.sub(r'\n\s*\n+', '\n', desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + fn = { + 'annotations': list(annotations), + 'signature': signature, + 'parameters': params, + 'parameters_doc': collections.OrderedDict(), + 'doc': [], + 'return': [], + 'seealso': [], + } + if fmt_vimhelp: + fn['desc_node'] = desc # HACK :( + + for m in paras: + if 'text' in m: + if not m['text'] == '': + fn['doc'].append(m['text']) + if 'params' in m: + # Merge OrderedDicts. + fn['parameters_doc'].update(m['params']) + if 'return' in m and len(m['return']) > 0: + fn['return'] += m['return'] + if 'seealso' in m and len(m['seealso']) > 0: + fn['seealso'] += m['seealso'] + + if INCLUDE_C_DECL: + fn['c_decl'] = c_decl + + if 'Deprecated' in str(xrefs): + deprecated_fns[name] = fn + elif name.startswith(CONFIG[target]['fn_name_prefix']): + fns[name] = fn + + xrefs.clear() + + fns = collections.OrderedDict(sorted(fns.items())) + deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items())) + return (fns, deprecated_fns) + + +def fmt_doxygen_xml_as_vimhelp(filename, target): + """Entrypoint for generating Vim :help from from Doxygen XML. + + Returns 3 items: + 1. Vim help text for functions found in `filename`. + 2. Vim help text for deprecated functions. + """ + global fmt_vimhelp + fmt_vimhelp = True + fns_txt = {} # Map of func_name:vim-help-text. + deprecated_fns_txt = {} # Map of func_name:vim-help-text. + fns, _ = extract_from_xml(filename, target, width=text_width) + + for name, fn in fns.items(): + # Generate Vim :help for parameters. + if fn['desc_node']: + doc = fmt_node_as_vimhelp(fn['desc_node']) if not doc: doc = 'TODO: Documentation' - annotations = '\n'.join(annotations) + annotations = '\n'.join(fn['annotations']) if annotations: annotations = ('\n\nAttributes: ~\n' + textwrap.indent(annotations, ' ')) @@ -567,21 +776,51 @@ def parse_source_xml(filename, mode): if INCLUDE_C_DECL: doc += '\n\nC Declaration: ~\n>\n' - doc += c_decl + doc += fn['c_decl'] doc += '\n<' - func_doc = signature + '\n' + func_doc = fn['signature'] + '\n' func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) + + # Verbatim handling. func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) + split_lines = func_doc.split('\n') + start = 0 + while True: + try: + start = split_lines.index('>', start) + except ValueError: + break + + try: + end = split_lines.index('<', start) + except ValueError: + break + + split_lines[start + 1:end] = [ + (' ' + x).rstrip() + for x in textwrap.dedent( + "\n".join( + split_lines[start+1:end] + ) + ).split("\n") + ] + + start = end + + func_doc = "\n".join(split_lines) + if 'Deprecated' in xrefs: - deprecated_functions.append(func_doc) - elif name.startswith(CONFIG[mode]['func_name_prefix']): - functions.append(func_doc) + deprecated_fns_txt[name] = func_doc + elif name.startswith(CONFIG[target]['fn_name_prefix']): + fns_txt[name] = func_doc xrefs.clear() - return '\n\n'.join(functions), '\n\n'.join(deprecated_functions) + fmt_vimhelp = False + return ('\n\n'.join(list(fns_txt.values())), + '\n\n'.join(list(deprecated_fns_txt.values()))) def delete_lines_below(filename, tokenstr): @@ -590,33 +829,54 @@ def delete_lines_below(filename, tokenstr): """ lines = open(filename).readlines() i = 0 + found = False for i, line in enumerate(lines, 1): if tokenstr in line: + found = True break + if not found: + raise RuntimeError(f'not found: "{tokenstr}"') i = max(0, i - 2) with open(filename, 'wt') as fp: fp.writelines(lines[0:i]) -def gen_docs(config): - """Generate documentation. +def main(config): + """Generates: + + 1. Vim :help docs + 2. *.mpack files for use by API clients Doxygen is called and configured through stdin. """ - for mode in CONFIG: - output_dir = out_dir.format(mode=mode) - p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) + for target in CONFIG: + if TARGET is not None and target != TARGET: + continue + mpack_file = os.path.join( + base_dir, 'runtime', 'doc', + CONFIG[target]['filename'].replace('.txt', '.mpack')) + if os.path.exists(mpack_file): + os.remove(mpack_file) + + output_dir = out_dir.format(target=target) + p = subprocess.Popen( + ['doxygen', '-'], + stdin=subprocess.PIPE, + # silence warnings + # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found + stderr=(subprocess.STDOUT if DEBUG else subprocess.DEVNULL)) p.communicate( config.format( - input=CONFIG[mode]['files'], + input=CONFIG[target]['files'], output=output_dir, filter=filter_cmd, - file_patterns=CONFIG[mode]['file_patterns']) + file_patterns=CONFIG[target]['file_patterns']) .encode('utf8') ) if p.returncode: sys.exit(p.returncode) + fn_map_full = {} # Collects all functions as each module is processed. sections = {} intros = {} sep = '=' * text_width @@ -633,11 +893,21 @@ def gen_docs(config): groupxml = os.path.join(base, '%s.xml' % compound.getAttribute('refid')) - desc = find_first(minidom.parse(groupxml), 'detaileddescription') + group_parsed = minidom.parse(groupxml) + doc_list = [] + brief_desc = find_first(group_parsed, 'briefdescription') + if brief_desc: + for child in brief_desc.childNodes: + doc_list.append(fmt_node_as_vimhelp(child)) + + desc = find_first(group_parsed, 'detaileddescription') if desc: - doc = parse_parblock(desc) + doc = fmt_node_as_vimhelp(desc) + if doc: - intros[groupname] = doc + doc_list.append(doc) + + intros[groupname] = "\n".join(doc_list) for compound in dom.getElementsByTagName('compound'): if compound.getAttribute('kind') != 'file': @@ -645,59 +915,54 @@ def gen_docs(config): filename = get_text(find_first(compound, 'name')) if filename.endswith('.c') or filename.endswith('.lua'): - functions, deprecated = parse_source_xml( - os.path.join(base, '%s.xml' % - compound.getAttribute('refid')), mode) - - if not functions and not deprecated: + # Extract unformatted (*.mpack). + fn_map, _ = extract_from_xml(os.path.join(base, '{}.xml'.format( + compound.getAttribute('refid'))), target, width=9999) + # Extract formatted (:help). + functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp( + os.path.join(base, '{}.xml'.format( + compound.getAttribute('refid'))), target) + + if not functions_text and not deprecated_text: continue - - if functions or deprecated: - name = os.path.splitext(os.path.basename(filename))[0] - if name == 'ui': - name = name.upper() - else: - name = name.title() - + else: + name = os.path.splitext( + os.path.basename(filename))[0].lower() + sectname = name.upper() if name == 'ui' else name.title() doc = '' - - intro = intros.get('api-%s' % name.lower()) + intro = intros.get(f'api-{name}') if intro: doc += '\n\n' + intro - if functions: - doc += '\n\n' + functions + if functions_text: + doc += '\n\n' + functions_text - if INCLUDE_DEPRECATED and deprecated: - doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name - doc += deprecated + if INCLUDE_DEPRECATED and deprecated_text: + doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n' + doc += deprecated_text if doc: filename = os.path.basename(filename) - name = CONFIG[mode]['section_name'].get(filename, name) - - if mode == 'lua': - title = 'Lua module: {}'.format(name.lower()) - helptag = '*lua-{}*'.format(name.lower()) - else: - title = '{} Functions'.format(name) - helptag = '*api-{}*'.format(name.lower()) + sectname = CONFIG[target]['section_name'].get( + filename, sectname) + title = CONFIG[target]['section_fmt'](sectname) + helptag = CONFIG[target]['helptag_fmt'](sectname) sections[filename] = (title, helptag, doc) + fn_map_full.update(fn_map) - if not sections: - return + assert sections + if len(sections) > len(CONFIG[target]['section_order']): + raise RuntimeError( + 'found new modules "{}"; update the "section_order" map'.format( + set(sections).difference(CONFIG[target]['section_order']))) docs = '' i = 0 - for filename in CONFIG[mode]['section_order']: - if filename not in sections: - raise RuntimeError( - 'found new module "{}"; update the "section_order" map'.format( - filename)) + for filename in CONFIG[target]['section_order']: title, helptag, section_doc = sections.pop(filename) i += 1 - if filename not in CONFIG[mode]['append_only']: + if filename not in CONFIG[target]['append_only']: docs += sep docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) @@ -708,12 +973,16 @@ def gen_docs(config): docs += ' vim:tw=78:ts=8:ft=help:norl:\n' doc_file = os.path.join(base_dir, 'runtime', 'doc', - CONFIG[mode]['filename']) + CONFIG[target]['filename']) - delete_lines_below(doc_file, CONFIG[mode]['section_start_token']) + delete_lines_below(doc_file, CONFIG[target]['section_start_token']) with open(doc_file, 'ab') as fp: fp.write(docs.encode('utf8')) + fn_map_full = collections.OrderedDict(sorted(fn_map_full.items())) + with open(mpack_file, 'wb') as fp: + fp.write(msgpack.packb(fn_map_full, use_bin_type=True)) + shutil.rmtree(output_dir) @@ -732,47 +1001,45 @@ def filter_source(filename): fp.read(), flags=re.M)) -# Doxygen Config {{{ -Doxyfile = ''' -OUTPUT_DIRECTORY = {output} -INPUT = {input} -INPUT_ENCODING = UTF-8 -FILE_PATTERNS = {file_patterns} -RECURSIVE = YES -INPUT_FILTER = "{filter}" -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = */private/* -EXCLUDE_SYMBOLS = -EXTENSION_MAPPING = lua=C -EXTRACT_PRIVATE = NO - -GENERATE_HTML = NO -GENERATE_DOCSET = NO -GENERATE_HTMLHELP = NO -GENERATE_QHP = NO -GENERATE_TREEVIEW = NO -GENERATE_LATEX = NO -GENERATE_RTF = NO -GENERATE_MAN = NO -GENERATE_DOCBOOK = NO -GENERATE_AUTOGEN_DEF = NO - -GENERATE_XML = YES -XML_OUTPUT = xml -XML_PROGRAMLISTING = NO - -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = YES -EXPAND_ONLY_PREDEF = NO -MARKDOWN_SUPPORT = YES -''' -# }}} +Doxyfile = textwrap.dedent(''' + OUTPUT_DIRECTORY = {output} + INPUT = {input} + INPUT_ENCODING = UTF-8 + FILE_PATTERNS = {file_patterns} + RECURSIVE = YES + INPUT_FILTER = "{filter}" + EXCLUDE = + EXCLUDE_SYMLINKS = NO + EXCLUDE_PATTERNS = */private/* + EXCLUDE_SYMBOLS = + EXTENSION_MAPPING = lua=C + EXTRACT_PRIVATE = NO + + GENERATE_HTML = NO + GENERATE_DOCSET = NO + GENERATE_HTMLHELP = NO + GENERATE_QHP = NO + GENERATE_TREEVIEW = NO + GENERATE_LATEX = NO + GENERATE_RTF = NO + GENERATE_MAN = NO + GENERATE_DOCBOOK = NO + GENERATE_AUTOGEN_DEF = NO + + GENERATE_XML = YES + XML_OUTPUT = xml + XML_PROGRAMLISTING = NO + + ENABLE_PREPROCESSING = YES + MACRO_EXPANSION = YES + EXPAND_ONLY_PREDEF = NO + MARKDOWN_SUPPORT = YES +''') if __name__ == "__main__": if len(sys.argv) > 1: filter_source(sys.argv[1]) else: - gen_docs(Doxyfile) + main(Doxyfile) -# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : +# vim: set ft=python ts=4 sw=4 tw=79 et : diff --git a/scripts/lua2dox.lua b/scripts/lua2dox.lua index 77cdabcc4b..d4e68f9e45 100644 --- a/scripts/lua2dox.lua +++ b/scripts/lua2dox.lua @@ -17,61 +17,28 @@ -- Free Software Foundation, Inc., -- -- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -- ----------------------------------------------------------------------------]] ---[[! -\file -\brief a hack lua2dox converter -]] --[[! -\mainpage - -Introduction ------------- - -A hack lua2dox converter -Version 0.2 - -This lets us make Doxygen output some documentation to let -us develop this code. +Lua-to-Doxygen converter -It is partially cribbed from the functionality of lua2dox -(http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm). -Found on CPAN when looking for something else; kinda handy. - -Improved from lua2dox to make the doxygen output more friendly. -Also it runs faster in lua rather than Perl. - -Because this Perl based system is called "lua2dox"., I have decided to add ".lua" to the name -to keep the two separate. +Partially from lua2dox +http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm Running ------- -<ol> -<li> Ensure doxygen is installed on your system and that you are familiar with its use. -Best is to try to make and document some simple C/C++/PHP to see what it produces. -You can experiment with the enclosed example code. - -<li> Run "doxygen -g" to create a default Doxyfile. - -Then alter it to let it recognise lua. Add the two following lines: +This file "lua2dox.lua" gets called by "lua2dox_filter" (bash). -\code{.bash} -FILE_PATTERNS = *.lua +Doxygen must be on your system. You can experiment like so: -FILTER_PATTERNS = *.lua=lua2dox_filter -\endcode - - -Either add them to the end or find the appropriate entry in Doxyfile. - -There are other lines that you might like to alter, but see futher documentation for details. - -<li> When Doxyfile is edited run "doxygen" +- Run "doxygen -g" to create a default Doxyfile. +- Then alter it to let it recognise lua. Add the two following lines: + FILE_PATTERNS = *.lua + FILTER_PATTERNS = *.lua=lua2dox_filter +- Then run "doxygen". The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language. It only has to be good enough for doxygen to see it as legal. -Therefore our lua interpreter is fairly limited, but "good enough". One limitation is that each line is treated separately (except for long comments). The implication is that class and function declarations must be on the same line. @@ -81,40 +48,8 @@ so it will probably not document accurately if we do do this. However I have put in a hack that will insert the "missing" close paren. The effect is that you will get the function documented, but not with the parameter list you might expect. -</ol> - -Installation ------------- - -Here for linux or unix-like, for any other OS you need to refer to other documentation. - -This file is "lua2dox.lua". It gets called by "lua2dox_filter"(bash). -Somewhere in your path (e.g. "~/bin" or "/usr/local/bin") put a link to "lua2dox_filter". - -Documentation -------------- - -Read the external documentation that should be part of this package. -For example look for the "README" and some .PDFs. - ]] --- we won't use our library code, so this becomes more portable - --- require 'elijah_fix_require' --- require 'elijah_class' --- ---! \brief ``declare'' as class ---! ---! use as: ---! \code{.lua} ---! TWibble = class() ---! function TWibble.init(this,Str) ---! this.str = Str ---! -- more stuff here ---! end ---! \endcode ---! function class(BaseClass, ClassInitialiser) local newClass = {} -- a new class newClass if not ClassInitialiser and type(BaseClass) == 'function' then @@ -165,8 +100,6 @@ function class(BaseClass, ClassInitialiser) return newClass end --- require 'elijah_clock' - --! \class TCore_Clock --! \brief a clock TCore_Clock = class() @@ -201,9 +134,6 @@ function TCore_Clock.getTimeStamp(this,T0) end ---require 'elijah_io' - ---! \class TCore_IO --! \brief io to console --! --! pseudo class (no methods, just to keep documentation tidy) @@ -225,8 +155,6 @@ function TCore_IO_writeln(Str) end ---require 'elijah_string' - --! \brief trims a string function string_trim(Str) return Str:match("^%s*(.-)%s*$") @@ -257,8 +185,6 @@ function string_split(Str, Pattern) end ---require 'elijah_commandline' - --! \class TCore_Commandline --! \brief reads/parses commandline TCore_Commandline = class() @@ -279,9 +205,6 @@ function TCore_Commandline.getRaw(this,Key,Default) return val end - ---require 'elijah_debug' - ------------------------------- --! \brief file buffer --! @@ -543,7 +466,6 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8))) if fn_magic then fn = fn_magic - fn_magic = nil end if string.sub(fn,1,1)=='(' then @@ -554,49 +476,20 @@ function TLua2DoX_filter.readfile(this,AppStamp,Filename) -- want to fix for iffy declarations local open_paren = string.find(fn,'[%({]') - local fn0 = fn if open_paren then - fn0 = string.sub(fn,1,open_paren-1) -- we might have a missing close paren if not string.find(fn,'%)') then fn = fn .. ' ___MissingCloseParenHere___)' end end - local dot = string.find(fn0,'[%.:]') - if dot then -- it's a method - local klass = string.sub(fn,1,dot-1) - local method = string.sub(fn,dot+1) - --TCore_IO_writeln('function ' .. klass .. '::' .. method .. ftail .. '{}') - --TCore_IO_writeln(klass .. '::' .. method .. ftail .. '{}') - outStream:writeln( - '/*! \\memberof ' .. klass .. ' */ ' - .. method .. '{}' - ) - else - -- add vanilla function - - outStream:writeln(fn_type .. 'function ' .. fn .. '{}') - end + -- add vanilla function + outStream:writeln(fn_type .. 'function ' .. fn .. '{}') end else this:warning(inStream:getLineNo(),'something weird here') end fn_magic = nil -- mustn't indavertently use it again - elseif string.find(line,'=%s*class%(') then - state = 'in_class' -- it's a class declaration - local tailComment - line,tailComment = TString_removeCommentFromLine(line) - local equals = string.find(line,'=') - local klass = string_trim(string.sub(line,1,equals-1)) - local tail = string_trim(string.sub(line,equals+1)) - -- class(wibble wibble) - -- ....v. - local parent = string.sub(tail,7,-2) - if #parent>0 then - parent = ' :public ' .. parent - end - outStream:writeln('class ' .. klass .. parent .. '{};') else state = '' -- unknown if #line>0 then -- we don't know what this line means, so just comment it out diff --git a/scripts/lua2dox_filter b/scripts/lua2dox_filter index 6cb16ef060..61577527c4 100755 --- a/scripts/lua2dox_filter +++ b/scripts/lua2dox_filter @@ -36,7 +36,16 @@ test_executable(){ ##! \brief sets the lua interpreter set_lua(){ - test_executable 'texlua' + if test -z "${EXE}" + then + test_executable 'luajit' + fi + + if test -z "${EXE}" + then + test_executable 'texlua' + fi + if test -z "${EXE}" then test_executable 'lua' diff --git a/scripts/release.sh b/scripts/release.sh index 29d61370ce..5b4902a2d7 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -80,7 +80,7 @@ _do_bump_commit() { <release date="'"${__DATE}"'" version="xxx"/>,' runtime/nvim.appdata.xml rm CMakeLists.txt.bk rm runtime/nvim.appdata.xml.bk - nvim +'/NVIM_VERSION' +1new +'exe "norm! iUpdate version numbers!!!\<CR>"' \ + nvim +'/NVIM_VERSION' +1new +'exe "norm! iUpdate version numbers!!!"' \ -O CMakeLists.txt runtime/nvim.appdata.xml git add CMakeLists.txt runtime/nvim.appdata.xml @@ -93,8 +93,8 @@ fi _do_bump_commit echo " Next steps: - - Double-check NVIM_VERSION_* in CMakeLists.txt - - Double-check runtime/nvim.appdata.xml + - Update runtime/nvim.appdata.xml on _master_ + - Run tests/CI (version_spec.lua)! - Push the tag: git push --follow-tags - Update the 'stable' tag: diff --git a/scripts/shadacat.py b/scripts/shadacat.py index 89846427a5..2b71fc2385 100755 --- a/scripts/shadacat.py +++ b/scripts/shadacat.py @@ -66,7 +66,7 @@ except IndexError: def filt(entry): return True else: _filt = filt - def filt(entry): return eval(_filt, globals(), {'entry': entry}) + def filt(entry): return eval(_filt, globals(), {'entry': entry}) # noqa poswidth = len(str(os.stat(fname).st_size or 1000)) diff --git a/scripts/update-ts-runtime.sh b/scripts/update-ts-runtime.sh new file mode 100755 index 0000000000..1a947e0ac9 --- /dev/null +++ b/scripts/update-ts-runtime.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# +# This script will update the treesitter runtime to the provided commit. +# Usage : +# $0 <tree-sitter commit sha> +set -e + +ts_source_dir="/tmp/tree-sitter" +ts_url="https://github.com/tree-sitter/tree-sitter.git" + +base_dir="$(cd "$(dirname $(dirname $0))" && pwd)" +ts_dest_dir="$base_dir/src/tree_sitter/" +ts_current_commit="$ts_dest_dir/treesitter_commit_hash.txt" + +echo "Updating treesitter runtime from $(cat "$ts_current_commit") to $1..." + +if [ ! -d "$ts_source_dir" ]; then + echo "Cloning treesitter..." + git clone "$ts_url" "$ts_source_dir" +else + echo "Found a non-empty $ts_source_dir directory..." + git -C "$ts_source_dir" fetch +fi + +echo "Checking out $1..." +git -C "$ts_source_dir" checkout $1 + +echo "Removing old files..." +find "$ts_dest_dir" -not -name "LICENSE" -not -name "README.md" -not -type d -delete + +echo "Copying files..." +cp -t "$ts_dest_dir" -r "$ts_source_dir/lib/src"/* +cp -t "$ts_dest_dir" "$ts_source_dir/lib/include/tree_sitter"/* + +echo "$1" > "$ts_current_commit" + +make +TEST_FILE="$base_dir/test/functional/lua/treesitter_spec.lua" make test + diff --git a/scripts/update_version_stamp.lua b/scripts/update_version_stamp.lua new file mode 100755 index 0000000000..11b521fab6 --- /dev/null +++ b/scripts/update_version_stamp.lua @@ -0,0 +1,55 @@ +#!/usr/bin/env lua +-- +-- Script to update the Git version stamp during build. +-- This is called via the custom update_version_stamp target in +-- src/nvim/CMakeLists.txt. +-- +-- arg[1]: file in which to update the version string +-- arg[2]: prefix to use always ("vX.Y.Z") + +local function die(msg) + io.stderr:write(string.format('%s: %s\n', arg[0], msg)) + -- No error, fall back to using generated "-dev" version. + os.exit(0) +end + +local function iswin() + return package.config:sub(1,1) == '\\' +end + +if #arg ~= 2 then + die(string.format("Expected two args, got %d", #arg)) +end + +local versiondeffile = arg[1] +local prefix = arg[2] + +local dev_null = iswin() and 'NUL' or '/dev/null' +local described = io.popen('git describe --first-parent --dirty 2>'..dev_null):read('*l') +if not described then + described = io.popen('git describe --first-parent --tags --always --dirty'):read('*l') +end +if not described then + io.open(versiondeffile, 'w'):write('\n') + die('git-describe failed, using empty include file.') +end + +-- `git describe` annotates the most recent tagged release; for pre-release +-- builds we must replace that with the unreleased version. +local with_prefix = described:gsub("^v%d+%.%d+%.%d+", prefix) +if described == with_prefix then + -- Prepend the prefix always, e.g. with "nightly-12208-g4041b62b9". + with_prefix = prefix .. "-" .. described +end + +-- Read existing include file. +local current = io.open(versiondeffile, 'r') +if current then + current = current:read('*l') +end + +-- Write new include file, if different. +local new = '#define NVIM_VERSION_MEDIUM "'..with_prefix..'"' +if current ~= new then + io.open(versiondeffile, 'w'):write(new .. '\n') +end diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 2cc32f0dd0..9c4349abca 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -5,6 +5,12 @@ set -u # Use privileged mode, which e.g. skips using CDPATH. set -p +# Ensure that the user has a bash that supports -A +if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then + >&2 echo "error: script requires bash 4+ (you have ${BASH_VERSION})." + exit 1 +fi + readonly NVIM_SOURCE_DIR="${NVIM_SOURCE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" readonly VIM_SOURCE_DIR_DEFAULT="${NVIM_SOURCE_DIR}/.vim-src" readonly VIM_SOURCE_DIR="${VIM_SOURCE_DIR:-${VIM_SOURCE_DIR_DEFAULT}}" @@ -23,6 +29,7 @@ usage() { echo " -h Show this message and exit." echo " -l [git-log opts] List missing Vim patches." echo " -L [git-log opts] List missing Vim patches (for scripts)." + echo " -m {vim-revision} List previous (older) missing Vim patches." echo " -M List all merged patch-numbers (at current v:version)." echo " -p {vim-revision} Download and generate a Vim patch. vim-revision" echo " can be a Vim version (8.0.xxx) or a Git hash." @@ -89,7 +96,7 @@ get_vim_sources() { echo "Cloning Vim into: ${VIM_SOURCE_DIR}" git clone https://github.com/vim/vim.git "${VIM_SOURCE_DIR}" cd "${VIM_SOURCE_DIR}" - else + elif [[ "${1-}" == update ]]; then cd "${VIM_SOURCE_DIR}" if ! [ -d ".git" ] \ && ! [ "$(git rev-parse --show-toplevel)" = "${VIM_SOURCE_DIR}" ]; then @@ -103,6 +110,8 @@ get_vim_sources() { else msg_err "Could not update Vim sources; ignoring error." fi + else + cd "${VIM_SOURCE_DIR}" fi } @@ -124,7 +133,7 @@ find_git_remote() { } # Assign variables for a given Vim tag, patch version, or commit. -# Might exit in case it cannot be found. +# Might exit in case it cannot be found, after updating Vim sources. assign_commit_details() { local vim_commit_ref if [[ ${1} =~ v?[0-9]\.[0-9]\.[0-9]{3,4} ]]; then @@ -146,9 +155,14 @@ assign_commit_details() { local munge_commit_line=false fi - vim_commit=$(git -C "${VIM_SOURCE_DIR}" log -1 --format="%H" "${vim_commit_ref}" --) || { - >&2 msg_err "Couldn't find Vim revision '${vim_commit_ref}'." - exit 3 + local get_vim_commit_cmd="git -C ${VIM_SOURCE_DIR} log -1 --format=%H ${vim_commit_ref} --" + vim_commit=$($get_vim_commit_cmd 2>&1) || { + # Update Vim sources. + get_vim_sources update + vim_commit=$($get_vim_commit_cmd 2>&1) || { + >&2 msg_err "Couldn't find Vim revision '${vim_commit_ref}': git error: ${vim_commit}." + exit 3 + } } vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}" @@ -227,7 +241,8 @@ get_vimpatch() { msg_ok "Saved patch to '${NVIM_SOURCE_DIR}/${patch_file}'." } -# shellcheck disable=SC2015 # "Note that A && B || C is not if-then-else." +# shellcheck disable=SC2015 +# ^ "Note that A && B || C is not if-then-else." stage_patch() { get_vimpatch "$1" local try_apply="${2:-}" @@ -298,7 +313,8 @@ git_hub_pr() { git hub pull new -m "$1" } -# shellcheck disable=SC2015 # "Note that A && B || C is not if-then-else." +# shellcheck disable=SC2015 +# ^ "Note that A && B || C is not if-then-else." submit_pr() { require_executable git local push_first @@ -366,7 +382,7 @@ submit_pr() { # Gets all Vim commits since the "start" commit. list_vim_commits() { ( - cd "${VIM_SOURCE_DIR}" && git log --reverse --format='%H' v8.0.0000..HEAD "$@" + cd "${VIM_SOURCE_DIR}" && git log --reverse v8.0.0000..HEAD "$@" ) } # Prints all (sorted) "vim-patch:xxx" tokens found in the Nvim git log. @@ -388,19 +404,69 @@ list_vimpatch_numbers() { done } +declare -A tokens +declare -A vim_commit_tags + +_set_tokens_and_tags() { + set +u # Avoid "unbound variable" with bash < 4.4 below. + if [[ -n "${tokens[*]}" ]]; then + return + fi + set -u + + # Find all "vim-patch:xxx" tokens in the Nvim git log. + for token in $(list_vimpatch_tokens); do + tokens[$token]=1 + done + + # Create an associative array mapping Vim commits to tags. + eval "vim_commit_tags=( + $(git -C "${VIM_SOURCE_DIR}" for-each-ref refs/tags \ + --format '[%(objectname)]=%(refname:strip=2)' \ + --sort='-*authordate' \ + --shell) + )" + # Exit in case of errors from the above eval (empty vim_commit_tags). + if ! (( "${#vim_commit_tags[@]}" )); then + msg_err "Could not get Vim commits/tags." + exit 1 + fi +} + # Prints a newline-delimited list of Vim commits, for use by scripts. +# "$1": use extended format? (with subject) # "$@" is passed to list_vim_commits, as extra arguments to git-log. list_missing_vimpatches() { + local -a missing_vim_patches=() + _set_missing_vimpatches "$@" + set +u # Avoid "unbound variable" with bash < 4.4 below. + for line in "${missing_vim_patches[@]}"; do + printf '%s\n' "$line" + done + set -u +} + +# Sets / appends to missing_vim_patches (useful to avoid a subshell when +# used multiple times to cache tokens/vim_commit_tags). +# "$1": use extended format? (with subject) +# "$@": extra arguments to git-log. +_set_missing_vimpatches() { local token vim_commit vim_tag patch_number - declare -A tokens - declare -A vim_commit_tags declare -a git_log_args + local extended_format=$1; shift + if [[ "$extended_format" == 1 ]]; then + git_log_args=("--format=%H %s") + else + git_log_args=("--format=%H") + fi + # Massage arguments for git-log. declare -A git_log_replacements=( [^\(.*/\)?src/nvim/\(.*\)]="\${BASH_REMATCH[1]}src/\${BASH_REMATCH[2]}" [^\(.*/\)?\.vim-src/\(.*\)]="\${BASH_REMATCH[2]}" ) + local i j for i in "$@"; do for j in "${!git_log_replacements[@]}"; do if [[ "$i" =~ $j ]]; then @@ -411,33 +477,31 @@ list_missing_vimpatches() { git_log_args+=("$i") done - # Find all "vim-patch:xxx" tokens in the Nvim git log. - for token in $(list_vimpatch_tokens); do - tokens[$token]=1 - done - - # Create an associative array mapping Vim commits to tags. - eval "declare -A vim_commit_tags=( - $(git -C "${VIM_SOURCE_DIR}" for-each-ref refs/tags \ - --format '[%(objectname)]=%(refname:strip=2)' \ - --sort='-*authordate' \ - --shell) - )" - # Exit in case of errors from the above eval (empty vim_commit_tags). - if ! (( "${#vim_commit_tags[@]}" )); then - msg_err "Could not get Vim commits/tags." - exit 1 - fi + _set_tokens_and_tags # Get missing Vim commits set +u # Avoid "unbound variable" with bash < 4.4 below. - for vim_commit in $(list_vim_commits "${git_log_args[@]}"); do + local vim_commit info + while IFS=' ' read -r line; do # Check for vim-patch:<commit_hash> (usually runtime updates). - token="vim-patch:${vim_commit:0:7}" + token="vim-patch:${line:0:7}" if [[ "${tokens[$token]-}" ]]; then continue fi + # Get commit hash, and optional info from line. This is used in + # extended mode, and when using e.g. '--format' manually. + vim_commit=${line%% *} + if [[ "$vim_commit" == "$line" ]]; then + info= + else + info=${line#* } + if [[ -n $info ]]; then + # Remove any "patch 8.0.0902: " prefixes, and prefix with ": ". + info=": ${info#patch*: }" + fi + fi + vim_tag="${vim_commit_tags[$vim_commit]-}" if [[ -n "$vim_tag" ]]; then # Check for vim-patch:<tag> (not commit hash). @@ -445,26 +509,26 @@ list_missing_vimpatches() { if [[ "${tokens[$patch_number]-}" ]]; then continue fi - echo "$vim_tag" + missing_vim_patches+=("$vim_tag$info") else - echo "$vim_commit" + missing_vim_patches+=("$vim_commit$info") fi - done + done < <(list_vim_commits "${git_log_args[@]}") set -u } # Prints a human-formatted list of Vim commits, with instructional messages. # Passes "$@" onto list_missing_vimpatches (args for git-log). show_vimpatches() { - get_vim_sources - printf "\nVim patches missing from Neovim:\n" + get_vim_sources update + printf "Vim patches missing from Neovim:\n" local -A runtime_commits for commit in $(git -C "${VIM_SOURCE_DIR}" log --format="%H %D" -- runtime | sed 's/,\? tag: / /g'); do runtime_commits[$commit]=1 done - list_missing_vimpatches "$@" | while read -r vim_commit; do + list_missing_vimpatches 1 "$@" | while read -r vim_commit; do if [[ "${runtime_commits[$vim_commit]-}" ]]; then printf ' • %s (+runtime)\n' "${vim_commit}" else @@ -472,18 +536,73 @@ show_vimpatches() { fi done - printf "Instructions: + cat << EOF - To port one of the above patches to Neovim, execute - this script with the patch revision as argument and - follow the instructions. - - Examples: '%s -p 7.4.487' - '%s -p 1e8ebf870720e7b671f98f22d653009826304c4f' +Instructions: + To port one of the above patches to Neovim, execute this script with the patch revision as argument and follow the instructions, e.g. + '${BASENAME} -p v8.0.1234', or '${BASENAME} -P v8.0.1234' NOTE: Please port the _oldest_ patch if you possibly can. - Out-of-order patches increase the possibility of bugs. -" "${BASENAME}" "${BASENAME}" + You can use '${BASENAME} -l path/to/file' to see what patches are missing for a file. +EOF +} + +list_missing_previous_vimpatches_for_patch() { + local for_vim_patch="${1}" + local vim_commit vim_tag + assign_commit_details "${for_vim_patch}" + + local file + local -a missing_list + local -a fnames + while IFS= read -r line ; do + fnames+=("$line") + done < <(git -C "${VIM_SOURCE_DIR}" diff-tree --no-commit-id --name-only -r "${vim_commit}") + local i=0 + local n=${#fnames[@]} + printf '=== getting missing patches for %d files ===\n' "$n" + if [[ -z "${vim_tag}" ]]; then + printf 'NOTE: "%s" is not a Vim tag - listing all oldest missing patches\n' "${for_vim_patch}" >&2 + fi + for fname in "${fnames[@]}"; do + i=$(( i+1 )) + printf '[%.*d/%d] %s: ' "${#n}" "$i" "$n" "$fname" + + local -a missing_vim_patches=() + _set_missing_vimpatches 1 -- "${fname}" + + set +u # Avoid "unbound variable" with bash < 4.4 below. + local missing_vim_commit_info="${missing_vim_patches[0]}" + if [[ -z "${missing_vim_commit_info}" ]]; then + printf -- "-\n" + else + local missing_vim_commit="${missing_vim_commit_info%%:*}" + if [[ -z "${vim_tag}" ]] || [[ "${missing_vim_commit}" < "${vim_tag}" ]]; then + printf -- "%s\n" "$missing_vim_commit_info" + missing_list+=("$missing_vim_commit_info") + else + printf -- "-\n" + fi + fi + set -u + done + + set +u # Avoid "unbound variable" with bash < 4.4 below. + if [[ -z "${missing_list[*]}" ]]; then + msg_ok 'no missing previous Vim patches' + set -u + return 0 + fi + set -u + + local -a missing_unique + while IFS= read -r line; do + missing_unique+=("$line") + done < <(printf '%s\n' "${missing_list[@]}" | sort -u) + + msg_err "$(printf '%d missing previous Vim patches:' ${#missing_unique[@]})" + printf ' - %s\n' "${missing_unique[@]}" + return 1 } review_commit() { @@ -561,9 +680,11 @@ review_pr() { echo echo "Downloading data for pull request #${pr}." - local pr_commit_urls=( - "$(curl -Ssf "https://api.github.com/repos/neovim/neovim/pulls/${pr}/commits" \ - | jq -r '.[].html_url')") + local -a pr_commit_urls + while IFS= read -r pr_commit_url; do + pr_commit_urls+=("$pr_commit_url") + done < <(curl -Ssf "https://api.github.com/repos/neovim/neovim/pulls/${pr}/commits" \ + | jq -r '.[].html_url') echo "Found ${#pr_commit_urls[@]} commit(s)." @@ -583,7 +704,7 @@ review_pr() { clean_files } -while getopts "hlLMVp:P:g:r:s" opt; do +while getopts "hlLmMVp:P:g:r:s" opt; do case ${opt} in h) usage @@ -596,13 +717,18 @@ while getopts "hlLMVp:P:g:r:s" opt; do ;; L) shift # remove opt - list_missing_vimpatches "$@" + list_missing_vimpatches 0 "$@" exit 0 ;; M) list_vimpatch_numbers exit 0 ;; + m) + shift # remove opt + list_missing_previous_vimpatches_for_patch "$@" + exit 0 + ;; p) stage_patch "${OPTARG}" exit @@ -624,7 +750,7 @@ while getopts "hlLMVp:P:g:r:s" opt; do exit 0 ;; V) - get_vim_sources + get_vim_sources update exit 0 ;; *) |