diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/check-includes.py | 66 | ||||
| -rw-r--r-- | scripts/check_urls.vim | 68 | ||||
| -rwxr-xr-x | scripts/download-unicode-files.sh | 18 | ||||
| -rwxr-xr-x | scripts/gen_api_vimdoc.py | 560 | ||||
| -rw-r--r-- | scripts/gen_events.lua | 65 | ||||
| -rw-r--r-- | scripts/gen_help_html.py | 355 | ||||
| -rwxr-xr-x | scripts/genappimage.sh | 98 | ||||
| -rwxr-xr-x | scripts/gendeclarations.lua | 261 | ||||
| -rw-r--r-- | scripts/genex_cmds.lua | 88 | ||||
| -rw-r--r-- | scripts/genoptions.lua | 184 | ||||
| -rw-r--r-- | scripts/genunicodetables.lua | 239 | ||||
| -rw-r--r-- | scripts/genvimvim.lua | 50 | ||||
| -rwxr-xr-x | scripts/git-log-pretty-since.sh | 28 | ||||
| -rw-r--r-- | scripts/msgpack-gen.lua | 313 | ||||
| -rwxr-xr-x | scripts/pvscheck.sh | 491 | ||||
| -rwxr-xr-x | scripts/release.sh | 82 | ||||
| -rwxr-xr-x | scripts/run-api-tests.exp | 50 | ||||
| -rwxr-xr-x | scripts/shadacat.py | 2 | ||||
| -rwxr-xr-x | scripts/update_terminfo.sh | 94 | ||||
| -rwxr-xr-x | scripts/vim-patch.sh | 448 | ||||
| -rwxr-xr-x | scripts/vimpatch.lua | 67 | ||||
| -rw-r--r-- | scripts/windows.ti | 71 |
22 files changed, 2286 insertions, 1412 deletions
diff --git a/scripts/check-includes.py b/scripts/check-includes.py new file mode 100755 index 0000000000..21308a21aa --- /dev/null +++ b/scripts/check-includes.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +import sys +import re +import os + +from subprocess import Popen, PIPE +from argparse import ArgumentParser + + +GENERATED_INCLUDE_RE = re.compile( + r'^\s*#\s*include\s*"([/a-z_0-9.]+\.generated\.h)"(\s+//.*)?$') + + +def main(argv): + argparser = ArgumentParser() + argparser.add_argument('--generated-includes-dir', action='append', + help='Directory where generated includes are located.') + argparser.add_argument('--file', type=open, help='File to check.') + argparser.add_argument('iwyu_args', nargs='*', + help='IWYU arguments, must go after --.') + args = argparser.parse_args(argv) + + with args.file: + include_dirs = [] + + iwyu = Popen(['include-what-you-use', '-xc'] + args.iwyu_args + ['/dev/stdin'], + stdin=PIPE, stdout=PIPE, stderr=PIPE) + + for line in args.file: + match = GENERATED_INCLUDE_RE.match(line) + if match: + for d in args.generated_includes_dir: + try: + f = open(os.path.join(d, match.group(1))) + except IOError: + continue + else: + with f: + for generated_line in f: + iwyu.stdin.write(generated_line) + break + else: + raise IOError('Failed to find {0}'.format(match.group(1))) + else: + iwyu.stdin.write(line) + + iwyu.stdin.close() + + out = iwyu.stdout.read() + err = iwyu.stderr.read() + + ret = iwyu.wait() + + if ret != 2: + print('IWYU failed with exit code {0}:'.format(ret)) + print('{0} stdout {0}'.format('=' * ((80 - len(' stdout ')) // 2))) + print(out) + print('{0} stderr {0}'.format('=' * ((80 - len(' stderr ')) // 2))) + print(err) + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/check_urls.vim b/scripts/check_urls.vim new file mode 100644 index 0000000000..3580b79475 --- /dev/null +++ b/scripts/check_urls.vim @@ -0,0 +1,68 @@ +" Test for URLs in help documents. +" +" Opens a new window with all found URLS followed by return code from curl +" (anything other than 0 means unreachable) +" +" Written by Christian Brabandt. + +func Test_check_URLs() + if has("win32") + echoerr "Doesn't work on MS-Windows" + return + endif + if executable('curl') + " Note: does not follow redirects! + let s:command = 'curl --silent --fail --output /dev/null --head ' + elseif executable('wget') + " Note: only allow a couple of redirects + let s:command = 'wget --quiet -S --spider --max-redirect=2 --timeout=5 --tries=2 -O /dev/null ' + else + echoerr 'Only works when "curl" or "wget" is available' + return + endif + + let pat='\(https\?\|ftp\)://[^\t* ]\+' + exe 'helpgrep' pat + helpclose + + let urls = map(getqflist(), 'v:val.text') + " do not use submatch(1)! + let urls = map(urls, {key, val -> matchstr(val, pat)}) + " remove examples like user@host (invalid urls) + let urls = filter(urls, 'v:val !~ "@"') + " Remove example URLs which are invalid + let urls = filter(urls, {key, val -> val !~ '\<\(\(my\|some\)\?host\|machine\|hostname\|file\)\>'}) + new + put =urls + " remove some more invalid items + " empty lines + v/./d + " remove # anchors + %s/#.*$//e + " remove trailing stuff (parenthesis, dot, comma, quotes), but only for HTTP + " links + g/^h/s#[.,)'"/>][:.]\?$## + g#^[hf]t\?tp:/\(/\?\.*\)$#d + silent! g/ftp://,$/d + silent! g/=$/d + let a = getline(1,'$') + let a = uniq(sort(a)) + %d + call setline(1, a) + + " Do the testing. + set nomore + %s/.*/\=TestURL(submatch(0))/ + + " highlight the failures + /.* \([0-9]*[1-9]\|[0-9]\{2,}\)$ +endfunc + +func TestURL(url) + " Relies on the return code to determine whether a page is valid + echom printf("Testing URL: %d/%d %s", line('.'), line('$'), a:url) + call system(s:command . shellescape(a:url)) + return printf("%s %d", a:url, v:shell_error) +endfunc + +call Test_check_URLs() diff --git a/scripts/download-unicode-files.sh b/scripts/download-unicode-files.sh index cb15270cf8..5f38d0589a 100755 --- a/scripts/download-unicode-files.sh +++ b/scripts/download-unicode-files.sh @@ -1,11 +1,11 @@ #!/bin/sh set -e - -files="UnicodeData.txt CaseFolding.txt EastAsianWidth.txt" +data_files="UnicodeData.txt CaseFolding.txt EastAsianWidth.txt" +emoji_files="emoji-data.txt" UNIDIR_DEFAULT=unicode -DOWNLOAD_URL_BASE_DEFAULT='http://unicode.org/Public/UNIDATA' +DOWNLOAD_URL_BASE_DEFAULT='http://unicode.org/Public' if test x$1 = 'x--help' ; then echo 'Usage:' @@ -21,8 +21,16 @@ fi UNIDIR=${1:-$UNIDIR_DEFAULT} DOWNLOAD_URL_BASE=${2:-$DOWNLOAD_URL_BASE_DEFAULT} -for filename in $files ; do - curl -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/$filename" +for filename in $data_files ; do + curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/$filename" + ( + cd "$UNIDIR" + git add $filename + ) +done + +for filename in $emoji_files ; do + curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/emoji/latest/$filename" ( cd "$UNIDIR" git add $filename diff --git a/scripts/gen_api_vimdoc.py b/scripts/gen_api_vimdoc.py new file mode 100755 index 0000000000..4e86f15b37 --- /dev/null +++ b/scripts/gen_api_vimdoc.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""Parses Doxygen XML output to generate Neovim's API documentation. + +This would be easier using lxml and XSLT, but: + + 1. This should avoid needing Python dependencies, especially ones that are + C modules that have library dependencies (lxml requires libxml and + libxslt). + 2. I wouldn't know how to deal with nested indentation in <para> tags using + XSLT. + +Each function documentation is formatted with the following rules: + + - Maximum width of 78 characters (`text_width`). + - Spaces for indentation. + - Function signature and helptag are on the same line. + - Helptag is right aligned. + - Signature and helptag must have a minimum of 8 spaces between them. + - If the signature is too long, it is placed on the line after the + helptag. The signature wraps at `text_width - 8` characters with + subsequent lines indented to the open parenthesis. + - Documentation body will be indented by 16 spaces. + - Subsection bodies are indented an additional 4 spaces. + - Documentation body consists of the function description, parameter details, + return description, and C declaration. + - Parameters are omitted for the `void` and `Error *` types, or if the + parameter is marked as [out]. + - Each function documentation is separated by a single line. + +The C declaration is added to the end to show actual argument types. +""" +import os +import re +import sys +import shutil +import textwrap +import subprocess + +from xml.dom import minidom + +if sys.version_info[0] < 3: + print("use Python 3") + sys.exit(1) + +doc_filename = 'api.txt' +# String used to find the start of the generated part of the doc. +section_start_token = '*api-global*' +# Required prefix for API function names. +api_func_name_prefix = 'nvim_' + +# Section name overrides. +section_name = { + 'vim.c': 'Global', +} + +# Section ordering. +section_order = ( + 'vim.c', + 'buffer.c', + 'window.c', + 'tabpage.c', + 'ui.c', +) + +param_exclude = ( + 'channel_id', +) + +# Annotations are displayed as line items after API function descriptions. +annotation_map = { + 'FUNC_API_ASYNC': '{async}', +} + +text_width = 78 +script_path = os.path.abspath(__file__) +base_dir = os.path.dirname(os.path.dirname(script_path)) +src_dir = os.path.join(base_dir, 'src/nvim/api') +out_dir = os.path.join(base_dir, 'tmp-api-doc') +filter_cmd = '%s %s' % (sys.executable, script_path) +seen_funcs = set() + +# Tracks `xrefsect` titles. As of this writing, used only for separating +# deprecated functions. +xrefs = set() + + +# XML Parsing Utilities {{{ +def find_first(parent, name): + """Finds the first matching node within parent.""" + sub = parent.getElementsByTagName(name) + if not sub: + return None + return sub[0] + + +def get_children(parent, name): + """Yield matching child nodes within parent.""" + for child in parent.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: + yield child + + +def get_child(parent, name): + """Get the first matching child node.""" + for child in get_children(parent, name): + return child + return None + + +def clean_text(text): + """Cleans text. + + Only cleans superfluous whitespace at the moment. + """ + return ' '.join(text.split()).strip() + + +def clean_lines(text): + """Removes superfluous lines. + + The beginning and end of the string is trimmed. Empty lines are collapsed. + """ + return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text)) + + +def get_text(parent): + """Combine all text in a node.""" + if parent.nodeType == parent.TEXT_NODE: + return parent.data + + out = '' + for node in parent.childNodes: + if node.nodeType == node.TEXT_NODE: + out += clean_text(node.data) + elif node.nodeType == node.ELEMENT_NODE: + out += ' ' + get_text(node) + return out + + +def doc_wrap(text, prefix='', width=70, func=False): + """Wraps text to `width`. + + The first line is prefixed with `prefix`, and subsequent lines are aligned. + If `func` is True, only wrap at commas. + """ + if not width: + return text + + indent_space = ' ' * len(prefix) + + if func: + lines = [prefix] + for part in text.split(', '): + if part[-1] not in ');': + part += ', ' + if len(lines[-1]) + len(part) > width: + lines.append(indent_space) + lines[-1] += part + return '\n'.join(x.rstrip() for x in lines).rstrip() + + return '\n'.join(textwrap.wrap(text.strip(), width=width, + initial_indent=prefix, + subsequent_indent=indent_space)) + + +def parse_params(parent, width=62): + """Parse Doxygen `parameterlist`.""" + name_length = 0 + items = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + continue + + name_node = find_first(child, 'parametername') + if name_node.getAttribute('direction') == 'out': + continue + + name = get_text(name_node) + if name in param_exclude: + continue + + name = '{%s}' % name + name_length = max(name_length, len(name) + 2) + + desc = '' + desc_node = get_child(child, 'parameterdescription') + if desc_node: + desc = parse_parblock(desc_node, width=None) + items.append((name.strip(), desc.strip())) + + out = 'Parameters: ~\n' + for name, desc in items: + name = ' %s' % name.ljust(name_length) + out += doc_wrap(desc, prefix=name, width=width) + '\n' + return out.strip() + + +def parse_para(parent, width=62): + """Parse doxygen `para` tag. + + I assume <para> is a paragraph block or "a block of text". It can contain + text nodes, or other tags. + """ + line = '' + lines = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + line += child.data + elif child.nodeName == 'computeroutput': + line += '`%s`' % get_text(child) + else: + if line: + lines.append(doc_wrap(line, width=width)) + line = '' + + if child.nodeName == 'parameterlist': + lines.append(parse_params(child, width=width)) + elif child.nodeName == 'xrefsect': + title = get_text(get_child(child, 'xreftitle')) + xrefs.add(title) + xrefdesc = parse_para(get_child(child, 'xrefdescription')) + lines.append(doc_wrap(xrefdesc, prefix='%s: ' % title, + width=width) + '\n') + elif child.nodeName == 'simplesect': + kind = child.getAttribute('kind') + if kind == 'note': + lines.append('Note:') + lines.append(doc_wrap(parse_para(child), + prefix=' ', + width=width)) + elif kind == 'return': + lines.append('%s: ~' % kind.title()) + lines.append(doc_wrap(parse_para(child), + prefix=' ', + width=width)) + else: + lines.append(get_text(child)) + + if line: + lines.append(doc_wrap(line, width=width)) + return clean_lines('\n'.join(lines).strip()) + + +def parse_parblock(parent, width=62): + """Parses a nested block of `para` tags. + + Named after the \parblock command, but not directly related. + """ + paragraphs = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + paragraphs.append(doc_wrap(child.data, width=width)) + elif child.nodeName == 'para': + paragraphs.append(parse_para(child, width=width)) + else: + paragraphs.append(doc_wrap(get_text(child), width=width)) + paragraphs.append('') + return clean_lines('\n'.join(paragraphs).strip()) +# }}} + + +def parse_source_xml(filename): + """Collects API functions. + + Returns two strings: + 1. API functions + 2. Deprecated API functions + + Caller decides what to do with the deprecated documentation. + """ + global xrefs + xrefs = set() + functions = [] + deprecated_functions = [] + + dom = minidom.parse(filename) + for member in dom.getElementsByTagName('memberdef'): + if member.getAttribute('static') == 'yes' or \ + member.getAttribute('kind') != 'function': + continue + + loc = find_first(member, 'location') + if 'private' in loc.getAttribute('file'): + continue + + return_type = get_text(get_child(member, 'type')) + if return_type == '': + continue + + if return_type.startswith(('ArrayOf', 'DictionaryOf')): + parts = return_type.strip('_').split('_') + return_type = '%s(%s)' % (parts[0], ', '.join(parts[1:])) + + name = get_text(get_child(member, 'name')) + + annotations = get_text(get_child(member, 'argsstring')) + if annotations and ')' in annotations: + annotations = annotations.rsplit(')', 1)[-1].strip() + # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of + # non-void functions. Special-case void functions here. + if name == 'nvim_get_mode' and len(annotations) == 0: + annotations += 'FUNC_API_ASYNC' + annotations = filter(None, map(lambda x: annotation_map.get(x), + annotations.split())) + + vimtag = '*%s()*' % name + args = [] + type_length = 0 + + for param in get_children(member, 'param'): + arg_type = get_text(get_child(param, 'type')).strip() + arg_name = '' + declname = get_child(param, 'declname') + if declname: + arg_name = get_text(declname).strip() + + if arg_name in param_exclude: + continue + + if arg_type.endswith('*'): + arg_type = arg_type.strip('* ') + arg_name = '*' + arg_name + type_length = max(type_length, len(arg_type)) + args.append((arg_type, arg_name)) + + c_args = [] + for arg_type, arg_name in args: + c_args.append(' ' + ( + '%s %s' % (arg_type.ljust(type_length), arg_name)).strip()) + + c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, + ',\n'.join(c_args)), + ' ') + + prefix = '%s(' % name + suffix = '%s)' % ', '.join('{%s}' % a[1] for a in args + if a[0] not in ('void', 'Error')) + + # Minimum 8 chars between signature and vimtag + lhs = (text_width - 8) - len(prefix) + + if len(prefix) + len(suffix) > lhs: + signature = vimtag.rjust(text_width) + '\n' + signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, + func=True) + else: + signature = prefix + suffix + signature += vimtag.rjust(text_width - len(signature)) + + doc = '' + desc = find_first(member, 'detaileddescription') + if desc: + doc = parse_parblock(desc) + if 'DEBUG' in os.environ: + print(textwrap.indent( + re.sub(r'\n\s*\n+', '\n', + desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + + if not doc: + doc = 'TODO: Documentation' + + annotations = '\n'.join(annotations) + if annotations: + annotations = ('\n\nAttributes: ~\n' + + textwrap.indent(annotations, ' ')) + i = doc.rfind('Parameters: ~') + if i == -1: + doc += annotations + else: + doc = doc[:i] + annotations + '\n\n' + doc[i:] + + if 'INCLUDE_C_DECL' in os.environ: + doc += '\n\nC Declaration: ~\n>\n' + doc += c_decl + doc += '\n<' + + func_doc = signature + '\n' + func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) + func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) + + if 'Deprecated' in xrefs: + deprecated_functions.append(func_doc) + elif name.startswith(api_func_name_prefix): + functions.append(func_doc) + + xrefs.clear() + + return '\n\n'.join(functions), '\n\n'.join(deprecated_functions) + + +def delete_lines_below(filename, tokenstr): + """Deletes all lines below the line containing `tokenstr`, the line itself, + and one line above it. + """ + lines = open(filename).readlines() + i = 0 + for i, line in enumerate(lines, 1): + if tokenstr in line: + break + i = max(0, i - 2) + with open(filename, 'wt') as fp: + fp.writelines(lines[0:i]) + +def gen_docs(config): + """Generate documentation. + + Doxygen is called and configured through stdin. + """ + p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) + p.communicate(config.format(input=src_dir, output=out_dir, + filter=filter_cmd).encode('utf8')) + if p.returncode: + sys.exit(p.returncode) + + sections = {} + intros = {} + sep = '=' * text_width + + base = os.path.join(out_dir, 'xml') + dom = minidom.parse(os.path.join(base, 'index.xml')) + + # generate docs for section intros + for compound in dom.getElementsByTagName('compound'): + if compound.getAttribute('kind') != 'group': + continue + + groupname = get_text(find_first(compound, 'name')) + groupxml = os.path.join(base, '%s.xml' % compound.getAttribute('refid')) + + desc = find_first(minidom.parse(groupxml), 'detaileddescription') + if desc: + doc = parse_parblock(desc) + if doc: + intros[groupname] = doc + + for compound in dom.getElementsByTagName('compound'): + if compound.getAttribute('kind') != 'file': + continue + + filename = get_text(find_first(compound, 'name')) + if filename.endswith('.c'): + functions, deprecated = parse_source_xml( + os.path.join(base, '%s.xml' % compound.getAttribute('refid'))) + + if not functions and not deprecated: + continue + + if functions or deprecated: + name = os.path.splitext(os.path.basename(filename))[0] + if name == 'ui': + name = name.upper() + else: + name = name.title() + + doc = '' + + intro = intros.get('api-%s' % name.lower()) + if intro: + doc += '\n\n' + intro + + if functions: + doc += '\n\n' + functions + + if 'INCLUDE_DEPRECATED' in os.environ and deprecated: + doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name + doc += deprecated + + if doc: + filename = os.path.basename(filename) + name = section_name.get(filename, name) + title = '%s Functions' % name + helptag = '*api-%s*' % name.lower() + sections[filename] = (title, helptag, doc) + + if not sections: + return + + docs = '' + + i = 0 + for filename in section_order: + if filename not in sections: + continue + title, helptag, section_doc = sections.pop(filename) + + i += 1 + docs += sep + docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) + docs += section_doc + docs += '\n\n\n' + + if sections: + # In case new API sources are added without updating the order dict. + for title, helptag, section_doc in sections.values(): + i += 1 + docs += sep + docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) + docs += section_doc + docs += '\n\n\n' + + docs = docs.rstrip() + '\n\n' + docs += ' vim:tw=78:ts=8:ft=help:norl:\n' + + doc_file = os.path.join(base_dir, 'runtime/doc', doc_filename) + delete_lines_below(doc_file, section_start_token) + with open(doc_file, 'ab') as fp: + fp.write(docs.encode('utf8')) + shutil.rmtree(out_dir) + + +def filter_source(filename): + """Filters the source to fix macros that confuse Doxygen.""" + with open(filename, 'rt') as fp: + print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))', + lambda m: m.group(1)+'_'.join( + re.split(r'[^\w]+', m.group(2))), + fp.read(), flags=re.M)) + + +# Doxygen Config {{{ +Doxyfile = ''' +OUTPUT_DIRECTORY = {output} +INPUT = {input} +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = *.h *.c +RECURSIVE = YES +INPUT_FILTER = "{filter}" +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = */private/* +EXCLUDE_SYMBOLS = + +GENERATE_HTML = NO +GENERATE_DOCSET = NO +GENERATE_HTMLHELP = NO +GENERATE_QHP = NO +GENERATE_TREEVIEW = NO +GENERATE_LATEX = NO +GENERATE_RTF = NO +GENERATE_MAN = NO +GENERATE_DOCBOOK = NO +GENERATE_AUTOGEN_DEF = NO + +GENERATE_XML = YES +XML_OUTPUT = xml +XML_PROGRAMLISTING = NO + +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = YES +EXPAND_ONLY_PREDEF = NO +''' +# }}} + +if __name__ == "__main__": + if len(sys.argv) > 1: + filter_source(sys.argv[1]) + else: + gen_docs(Doxyfile) + +# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : diff --git a/scripts/gen_events.lua b/scripts/gen_events.lua deleted file mode 100644 index 75e0b3da3a..0000000000 --- a/scripts/gen_events.lua +++ /dev/null @@ -1,65 +0,0 @@ -if arg[1] == '--help' then - print('Usage: gen_events.lua src/nvim enum_file event_names_file') - os.exit(0) -end - -local nvimsrcdir = arg[1] -local fileio_enum_file = arg[2] -local names_file = arg[3] - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local auevents = require('auevents') -local events = auevents.events -local aliases = auevents.aliases - -enum_tgt = io.open(fileio_enum_file, 'w') -names_tgt = io.open(names_file, 'w') - -enum_tgt:write('typedef enum auto_event {') -names_tgt:write([[ -static const struct event_name { - size_t len; - char *name; - event_T event; -} event_names[] = {]]) - -for i, event in ipairs(events) do - if i > 1 then - comma = ',\n' - else - comma = '\n' - end - enum_tgt:write(('%s EVENT_%s = %u'):format(comma, event:upper(), i - 1)) - names_tgt:write(('%s {%u, "%s", EVENT_%s}'):format(comma, #event, event, event:upper())) -end - -for alias, event in pairs(aliases) do - names_tgt:write((',\n {%u, "%s", EVENT_%s}'):format(#alias, alias, event:upper())) -end - -names_tgt:write(',\n {0, NULL, (event_T)0}') - -enum_tgt:write('\n} event_T;\n') -names_tgt:write('\n};\n') - -enum_tgt:write(('\n#define NUM_EVENTS %u\n'):format(#events)) -names_tgt:write('\nstatic AutoPat *first_autopat[NUM_EVENTS] = {\n ') -line_len = 1 -for i = 1,((#events) - 1) do - line_len = line_len + #(' NULL,') - if line_len > 80 then - names_tgt:write('\n ') - line_len = 1 + #(' NULL,') - end - names_tgt:write(' NULL,') -end -if line_len + #(' NULL') > 80 then - names_tgt:write('\n NULL') -else - names_tgt:write(' NULL') -end -names_tgt:write('\n};\n') - -enum_tgt:close() -names_tgt:close() diff --git a/scripts/gen_help_html.py b/scripts/gen_help_html.py new file mode 100644 index 0000000000..6cca81d250 --- /dev/null +++ b/scripts/gen_help_html.py @@ -0,0 +1,355 @@ +# Converts Vim/Nvim documentation to HTML. +# +# Adapted from https://github.com/c4rlo/vimhelp/ +# License: MIT +# +# Copyright (c) 2016 Carlo Teubner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import re, urllib.parse +from itertools import chain + +HEAD = """\ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" + "http://www.w3.org/TR/html4/loose.dtd"> +<html> +<head> +<meta http-equiv="Content-type" content="text/html; charset={encoding}"/> +<title>Nvim: {filename}</title> +""" + +HEAD_END = '</head>\n<body>\n' + +INTRO = """ +<h1>Nvim help files</h1> +<p>HTML export of the <a href="https://neovim.io/">Nvim</a> help pages{vers-note}. +Updated <a href="https://github.com/neovim/bot-ci" class="d">automatically</a> from the <a +href="https://github.com/vim/vim/tree/master/runtime/doc" class="d">Nvim source repository</a>. +Also includes the <a href="vim_faq.txt.html">Vim FAQ</a>, pulled from its +<a href="https://github.com/chrisbra/vim_faq" class="d">source repository</a>.</p> +""" + +VERSION_NOTE = ", current as of Vim {version}" + +SITENAVI_LINKS = """ +Quick links: +<a href="/">help overview</a> · +<a href="quickref.txt.html">quick reference</a> · +<a href="usr_toc.txt.html">user manual toc</a> · +<a href="{helptxt}#reference_toc">reference manual toc</a> · +<a href="vim_faq.txt.html">faq</a> +""" + +SITENAVI_LINKS_PLAIN = SITENAVI_LINKS.format(helptxt='help.txt.html') +SITENAVI_LINKS_WEB = SITENAVI_LINKS.format(helptxt='/') + +SITENAVI_PLAIN = '<p>' + SITENAVI_LINKS_PLAIN + '</p>' +SITENAVI_WEB = '<p>' + SITENAVI_LINKS_WEB + '</p>' + +SITENAVI_SEARCH = '<table width="100%"><tbody><tr><td>' + SITENAVI_LINKS_WEB + \ +'</td><td style="text-align: right; max-width: 25vw"><div class="gcse-searchbox">' \ +'</div></td></tr></tbody></table><div class="gcse-searchresults"></div>' + +TEXTSTART = """ +<div id="d1"> +<pre id="sp"> </pre> +<div id="d2"> +<pre> +""" + +FOOTER = '</pre>' + +FOOTER2 = """ +<p id="footer">This site is maintained by Carlo Teubner (<i>(my first name) dot (my last name) at gmail dot com</i>).</p> +</div> +</div> +</body> +</html> +""" + +VIM_FAQ_LINE = '<a href="vim_faq.txt.html#vim_faq.txt" class="l">' \ + 'vim_faq.txt</a> Frequently Asked Questions\n' + +RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)') + +PAT_WORDCHAR = '[!#-)+-{}~\xC0-\xFF]' + +PAT_HEADER = r'(^.*~$)' +PAT_GRAPHIC = r'(^.* `$)' +PAT_PIPEWORD = r'(?<!\\)\|([#-)!+-~]+)\|' +PAT_STARWORD = r'\*([#-)!+-~]+)\*(?:(?=\s)|$)' +PAT_COMMAND = r'`([^` ]+)`' +PAT_OPTWORD = r"('(?:[a-z]{2,}|t_..)')" +PAT_CTRL = r'(CTRL-(?:W_)?(?:\{char\}|<[A-Za-z]+?>|.)?)' +PAT_SPECIAL = r'(<.+?>|\{.+?}|' \ + r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' \ + r'arg|arguments|ident|addr|group)]|' \ + r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])' +PAT_TITLE = r'(Vim version [0-9.a-z]+|VIM REFERENCE.*)' +PAT_NOTE = r'((?<!' + PAT_WORDCHAR + r')(?:note|NOTE|Notes?):?' \ + r'(?!' + PAT_WORDCHAR + r'))' +PAT_URL = r'((?:https?|ftp)://[^\'"<> \t]+[a-zA-Z0-9/])' +PAT_WORD = r'((?<!' + PAT_WORDCHAR + r')' + PAT_WORDCHAR + r'+' \ + r'(?!' + PAT_WORDCHAR + r'))' + +RE_LINKWORD = re.compile( + PAT_OPTWORD + '|' + + PAT_CTRL + '|' + + PAT_SPECIAL) +RE_TAGWORD = re.compile( + PAT_HEADER + '|' + + PAT_GRAPHIC + '|' + + PAT_PIPEWORD + '|' + + PAT_STARWORD + '|' + + PAT_COMMAND + '|' + + PAT_OPTWORD + '|' + + PAT_CTRL + '|' + + PAT_SPECIAL + '|' + + PAT_TITLE + '|' + + PAT_NOTE + '|' + + PAT_URL + '|' + + PAT_WORD) +RE_NEWLINE = re.compile(r'[\r\n]') +RE_HRULE = re.compile(r'[-=]{3,}.*[-=]{3,3}$') +RE_EG_START = re.compile(r'(?:.* )?>$') +RE_EG_END = re.compile(r'\S') +RE_SECTION = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)') +RE_STARTAG = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)') +RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$') + +class Link(object): + __slots__ = 'link_plain_same', 'link_pipe_same', \ + 'link_plain_foreign', 'link_pipe_foreign', \ + 'filename' + + def __init__(self, link_plain_same, link_plain_foreign, + link_pipe_same, link_pipe_foreign, filename): + self.link_plain_same = link_plain_same + self.link_plain_foreign = link_plain_foreign + self.link_pipe_same = link_pipe_same + self.link_pipe_foreign = link_pipe_foreign + self.filename = filename + +class VimH2H(object): + def __init__(self, tags, version=None, is_web_version=True): + self._urls = { } + self._version = version + self._is_web_version = is_web_version + for line in RE_NEWLINE.split(tags): + m = RE_TAGLINE.match(line) + if m: + tag, filename = m.group(1, 2) + self.do_add_tag(filename, tag) + + def add_tags(self, filename, contents): + for match in RE_STARTAG.finditer(contents): + tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/') + self.do_add_tag(str(filename), tag) + + def do_add_tag(self, filename, tag): + tag_quoted = urllib.parse.quote_plus(tag) + def mkpart1(doc): + return '<a href="' + doc + '#' + tag_quoted + '" class="' + part1_same = mkpart1('') + if self._is_web_version and filename == 'help.txt': + doc = '/' + else: + doc = filename + '.html' + part1_foreign = mkpart1(doc) + part2 = '">' + html_escape[tag] + '</a>' + def mklinks(cssclass): + return (part1_same + cssclass + part2, + part1_foreign + cssclass + part2) + cssclass_plain = 'd' + m = RE_LINKWORD.match(tag) + if m: + opt, ctrl, special = m.groups() + if opt is not None: cssclass_plain = 'o' + elif ctrl is not None: cssclass_plain = 'k' + elif special is not None: cssclass_plain = 's' + links_plain = mklinks(cssclass_plain) + links_pipe = mklinks('l') + self._urls[tag] = Link( + links_plain[0], links_plain[1], + links_pipe[0], links_pipe[1], + filename) + + def maplink(self, tag, curr_filename, css_class=None): + links = self._urls.get(tag) + if links is not None: + if links.filename == curr_filename: + if css_class == 'l': return links.link_pipe_same + else: return links.link_plain_same + else: + if css_class == 'l': return links.link_pipe_foreign + else: return links.link_plain_foreign + elif css_class is not None: + return '<span class="' + css_class + '">' + html_escape[tag] + \ + '</span>' + else: return html_escape[tag] + + def to_html(self, filename, contents, encoding): + out = [ ] + + inexample = 0 + filename = str(filename) + is_help_txt = (filename == 'help.txt') + faq_line = False + for line in RE_NEWLINE.split(contents): + line = line.rstrip('\r\n') + line_tabs = line + line = line.expandtabs() + if RE_HRULE.match(line): + out.extend(('<span class="h">', line, '</span>\n')) + continue + if inexample == 2: + if RE_EG_END.match(line): + inexample = 0 + if line[0] == '<': line = line[1:] + else: + out.extend(('<span class="e">', html_escape[line], + '</span>\n')) + continue + if RE_EG_START.match(line_tabs): + inexample = 1 + line = line[0:-1] + if RE_SECTION.match(line_tabs): + m = RE_SECTION.match(line) + out.extend((r'<span class="c">', m.group(0), r'</span>')) + line = line[m.end():] + if is_help_txt and RE_LOCAL_ADD.match(line_tabs): + faq_line = True + lastpos = 0 + for match in RE_TAGWORD.finditer(line): + pos = match.start() + if pos > lastpos: + out.append(html_escape[line[lastpos:pos]]) + lastpos = match.end() + header, graphic, pipeword, starword, command, opt, ctrl, \ + special, title, note, url, word = match.groups() + if pipeword is not None: + out.append(self.maplink(pipeword, filename, 'l')) + elif starword is not None: + out.extend(('<a name="', urllib.parse.quote_plus(starword), + '" class="t">', html_escape[starword], '</a>')) + elif command is not None: + out.extend(('<span class="e">', html_escape[command], + '</span>')) + elif opt is not None: + out.append(self.maplink(opt, filename, 'o')) + elif ctrl is not None: + out.append(self.maplink(ctrl, filename, 'k')) + elif special is not None: + out.append(self.maplink(special, filename, 's')) + elif title is not None: + out.extend(('<span class="i">', html_escape[title], + '</span>')) + elif note is not None: + out.extend(('<span class="n">', html_escape[note], + '</span>')) + elif header is not None: + out.extend(('<span class="h">', html_escape[header[:-1]], + '</span>')) + elif graphic is not None: + out.append(html_escape[graphic[:-2]]) + elif url is not None: + out.extend(('<a class="u" href="', url, '">' + + html_escape[url], '</a>')) + elif word is not None: + out.append(self.maplink(word, filename)) + if lastpos < len(line): + out.append(html_escape[line[lastpos:]]) + out.append('\n') + if inexample == 1: inexample = 2 + if faq_line: + out.append(VIM_FAQ_LINE) + faq_line = False + + header = [] + header.append(HEAD.format(encoding=encoding, filename=filename)) + header.append(HEAD_END) + if self._is_web_version and is_help_txt: + vers_note = VERSION_NOTE.replace('{version}', self._version) \ + if self._version else '' + header.append(INTRO.replace('{vers-note}', vers_note)) + if self._is_web_version: + header.append(SITENAVI_SEARCH) + sitenavi_footer = SITENAVI_WEB + else: + header.append(SITENAVI_PLAIN) + sitenavi_footer = SITENAVI_PLAIN + header.append(TEXTSTART) + return ''.join(chain(header, out, (FOOTER, sitenavi_footer, FOOTER2))) + +class HtmlEscCache(dict): + def __missing__(self, key): + r = key.replace('&', '&') \ + .replace('<', '<') \ + .replace('>', '>') + self[key] = r + return r + +html_escape = HtmlEscCache() + + + +import sys, os, os.path +#import cProfile +sys.path.append('.') + +def slurp(filename): + try: + with open(filename, encoding='UTF-8') as f: + return f.read(), 'UTF-8' + except UnicodeError: + # 'ISO-8859-1' ? + with open(filename, encoding='latin-1') as f: + return f.read(), 'latin-1' + +def usage(): + return "usage: " + sys.argv[0] + " IN_DIR OUT_DIR [BASENAMES...]" + +def main(): + if len(sys.argv) < 3: sys.exit(usage()) + + in_dir = sys.argv[1] + out_dir = sys.argv[2] + basenames = sys.argv[3:] + + print( "Processing tags...") + h2h = VimH2H(slurp(os.path.join(in_dir, 'tags'))[0], is_web_version=False) + + if len(basenames) == 0: + basenames = os.listdir(in_dir) + + for basename in basenames: + if os.path.splitext(basename)[1] != '.txt' and basename != 'tags': + print( "Ignoring " + basename) + continue + print( "Processing " + basename + "...") + path = os.path.join(in_dir, basename) + text, encoding = slurp(path) + outpath = os.path.join(out_dir, basename + '.html') + of = open(outpath, 'w') + of.write(h2h.to_html(basename, text, encoding)) + of.close() + +main() +#cProfile.run('main()') diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh new file mode 100755 index 0000000000..c0ef2a8591 --- /dev/null +++ b/scripts/genappimage.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +######################################################################## +# Package the binaries built as an AppImage +# By Simon Peter 2016 +# For more information, see http://appimage.org/ +######################################################################## + +# App arch, used by generate_appimage. +if [ -z "$ARCH" ]; then + export ARCH="$(arch)" +fi + +TAG=$1 + +# App name, used by generate_appimage. +APP=nvim + +ROOT_DIR="$(git rev-parse --show-toplevel)" +APP_BUILD_DIR="$ROOT_DIR/build" +APP_DIR="$APP.AppDir" + +######################################################################## +# Compile nvim and install it into AppDir +######################################################################## + +# Build and install nvim into the AppImage +make CMAKE_BUILD_TYPE=RelWithDebInfo CMAKE_EXTRA_FLAGS="-DCMAKE_INSTALL_PREFIX=${APP_DIR}/usr -DCMAKE_INSTALL_MANDIR=man" +make install + +######################################################################## +# Get helper functions and move to AppDir +######################################################################## + +# App version, used by generate_appimage. +VERSION=$("$ROOT_DIR"/build/bin/nvim --version | head -n 1 | grep -o 'v.*') + +cd "$APP_BUILD_DIR" + +curl -Lo "$APP_BUILD_DIR"/appimage_functions.sh https://github.com/AppImage/AppImages/raw/master/functions.sh +. ./appimage_functions.sh + +# Copy desktop and icon file to AppDir for AppRun to pick them up. +# get_apprun +# get_desktop +cp "$ROOT_DIR/runtime/nvim.desktop" "$APP_DIR/" +cp "$ROOT_DIR/runtime/nvim.png" "$APP_DIR/" +mkdir "$APP_DIR/usr/share/metainfo/" +cp "$ROOT_DIR/runtime/nvim.appdata.xml" "$APP_DIR/usr/share/metainfo/" + +cd "$APP_DIR" + +# copy dependencies +copy_deps +# Move the libraries to usr/bin +move_lib + +# Delete stuff that should not go into the AppImage. +# Delete dangerous libraries; see +# https://github.com/AppImage/AppImages/blob/master/excludelist +delete_blacklisted + +######################################################################## +# AppDir complete. Now package it as an AppImage. +######################################################################## + +# Appimage set the ARGV0 environment variable. This causes problems in zsh. +# To prevent this, we use wrapper script to unset ARGV0 as AppRun. +# See https://github.com/AppImage/AppImageKit/issues/852 +# +cat << 'EOF' > AppRun +#!/bin/bash + +unset ARGV0 +exec "$(dirname "$(readlink -f "${0}")")/usr/bin/nvim" ${@+"$@"} +EOF +chmod 755 AppRun + +cd "$APP_BUILD_DIR" # Get out of AppImage directory. + +# Generate AppImage. +# - Expects: $ARCH, $APP, $VERSION env vars +# - Expects: ./$APP.AppDir/ directory +# - Produces: ../out/$APP-$VERSION.glibc$GLIBC_NEEDED-$ARCH.AppImage +if [ -n "$TAG" ]; then + generate_type2_appimage -u "gh-releases-zsync|neovim|neovim|$TAG|nvim.appimage.zsync" +else + generate_type2_appimage +fi + +# Moving the final executable to a different folder so it isn't in the +# way for a subsequent build. + +mv "$ROOT_DIR"/out/*.AppImage* "$ROOT_DIR"/build/bin +# Remove the (now empty) folder the AppImage was built in +rmdir "$ROOT_DIR"/out + +echo 'genappimage.sh: finished' diff --git a/scripts/gendeclarations.lua b/scripts/gendeclarations.lua deleted file mode 100755 index ff69b18ae4..0000000000 --- a/scripts/gendeclarations.lua +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/lua - -local fname = arg[1] -local static_fname = arg[2] -local non_static_fname = arg[3] -local preproc_fname = arg[4] - - -local lpeg = require('lpeg') - -local fold = function (func, ...) - local result = nil - for i, v in ipairs({...}) do - if result == nil then - result = v - else - result = func(result, v) - end - end - return result -end - -local folder = function (func) - return function (...) - return fold(func, ...) - end -end - -local lit = lpeg.P -local set = function(...) - return lpeg.S(fold(function (a, b) return a .. b end, ...)) -end -local any_character = lpeg.P(1) -local rng = function(s, e) return lpeg.R(s .. e) end -local concat = folder(function (a, b) return a * b end) -local branch = folder(function (a, b) return a + b end) -local one_or_more = function(v) return v ^ 1 end -local two_or_more = function(v) return v ^ 2 end -local any_amount = function(v) return v ^ 0 end -local one_or_no = function(v) return v ^ -1 end -local look_behind = lpeg.B -local look_ahead = function(v) return #v end -local neg_look_ahead = function(v) return -v end -local neg_look_behind = function(v) return -look_behind(v) end - -local w = branch( - rng('a', 'z'), - rng('A', 'Z'), - lit('_') -) -local aw = branch( - w, - rng('0', '9') -) -local s = set(' ', '\n', '\t') -local raw_word = concat(w, any_amount(aw)) -local right_word = concat( - raw_word, - neg_look_ahead(aw) -) -local word = branch( - concat( - branch(lit('ArrayOf('), lit('DictionaryOf(')), -- typed container macro - one_or_more(any_character - lit(')')), - lit(')') - ), - concat( - neg_look_behind(aw), - right_word - ) -) -local spaces = any_amount(branch( - s, - -- Comments are really handled by preprocessor, so the following is not needed - concat( - lit('/*'), - any_amount(concat( - neg_look_ahead(lit('*/')), - any_character - )), - lit('*/') - ), - concat( - lit('//'), - any_amount(concat( - neg_look_ahead(lit('\n')), - any_character - )), - lit('\n') - ), - -- Linemarker inserted by preprocessor - concat( - lit('# '), - any_amount(concat( - neg_look_ahead(lit('\n')), - any_character - )), - lit('\n') - ) -)) -local typ_part = concat( - word, - any_amount(concat( - spaces, - lit('*') - )), - spaces -) -local typ = one_or_more(typ_part) -local typ_id = two_or_more(typ_part) -local arg = typ_id -- argument name is swallowed by typ -local pattern = concat( - typ_id, -- return type with function name - spaces, - lit('('), - spaces, - one_or_no(branch( -- function arguments - concat( - arg, -- first argument, does not require comma - any_amount(concat( -- following arguments, start with a comma - spaces, - lit(','), - spaces, - arg, - any_amount(concat( - lit('['), - spaces, - any_amount(aw), - spaces, - lit(']') - )) - )), - one_or_no(concat( - spaces, - lit(','), - spaces, - lit('...') - )) - ), - lit('void') -- also accepts just void - )), - spaces, - lit(')'), - any_amount(concat( -- optional attributes - spaces, - lit('FUNC_'), - any_amount(aw), - one_or_no(concat( -- attribute argument - spaces, - lit('('), - any_amount(concat( - neg_look_ahead(lit(')')), - any_character - )), - lit(')') - )) - )), - look_ahead(concat( -- definition must be followed by "{" - spaces, - lit('{') - )) -) - -if fname == '--help' then - print'Usage:' - print() - print' gendeclarations.lua definitions.c static.h non-static.h preprocessor.i' - os.exit() -end - -local preproc_f = io.open(preproc_fname) -local text = preproc_f:read("*all") -preproc_f:close() - - -local header = [[ -#ifndef DEFINE_FUNC_ATTRIBUTES -# define DEFINE_FUNC_ATTRIBUTES -#endif -#include "nvim/func_attr.h" -#undef DEFINE_FUNC_ATTRIBUTES -]] - -local footer = [[ -#include "nvim/func_attr.h" -]] - -local non_static = header -local static = header - -local filepattern = '^#%a* %d+ "[^"]-/?([^"/]+)"' -local curfile - -init = 0 -curfile = nil -neededfile = fname:match('[^/]+$') -while init ~= nil do - init = text:find('\n', init) - if init == nil then - break - end - init = init + 1 - if text:sub(init, init) == '#' then - file = text:match(filepattern, init) - if file ~= nil then - curfile = file - end - elseif curfile == neededfile then - s = init - e = pattern:match(text, init) - if e ~= nil then - local declaration = text:sub(s, e - 1) - -- Comments are really handled by preprocessor, so the following is not - -- needed - declaration = declaration:gsub('/%*.-%*/', '') - declaration = declaration:gsub('//.-\n', '\n') - - declaration = declaration:gsub('# .-\n', '') - - declaration = declaration:gsub('\n', ' ') - declaration = declaration:gsub('%s+', ' ') - declaration = declaration:gsub(' ?%( ?', '(') - -- declaration = declaration:gsub(' ?%) ?', ')') - declaration = declaration:gsub(' ?, ?', ', ') - declaration = declaration:gsub(' ?(%*+) ?', ' %1') - declaration = declaration:gsub(' ?(FUNC_ATTR_)', ' %1') - declaration = declaration:gsub(' $', '') - declaration = declaration .. ';\n' - if text:sub(s, s + 5) == 'static' then - static = static .. declaration - else - non_static = non_static .. declaration - end - init = e - end - end -end - -non_static = non_static .. footer -static = static .. footer - -local F -F = io.open(static_fname, 'w') -F:write(static) -F:close() - --- Before generating the non-static headers, check if the current file(if --- exists) is different from the new one. If they are the same, we won't touch --- the current version to avoid triggering an unnecessary rebuilds of modules --- that depend on this one -F = io.open(non_static_fname, 'r') -if F ~= nil then - if F:read('*a') == non_static then - os.exit(0) - end - io.close(F) -end - -F = io.open(non_static_fname, 'w') -F:write(non_static) -F:close() diff --git a/scripts/genex_cmds.lua b/scripts/genex_cmds.lua deleted file mode 100644 index b1d34fbffd..0000000000 --- a/scripts/genex_cmds.lua +++ /dev/null @@ -1,88 +0,0 @@ -local nvimsrcdir = arg[1] -local includedir = arg[2] -local autodir = arg[3] - -if nvimsrcdir == '--help' then - print ([[ -Usage: - lua genex_cmds.lua src/nvim build/include build/src/nvim/auto - -Will generate files build/include/ex_cmds_enum.generated.h with cmdidx_T -enum and build/src/nvim/auto/ex_cmds_defs.generated.h with main Ex commands -definitions. -]]) - os.exit(0) -end - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local enumfname = includedir .. '/ex_cmds_enum.generated.h' -local defsfname = autodir .. '/ex_cmds_defs.generated.h' - -local enumfile = io.open(enumfname, 'w') -local defsfile = io.open(defsfname, 'w') - -local defs = require('ex_cmds') -local lastchar = nil - -local i -local cmd -local first = true -local prevfirstchar = nil - -local byte_a = string.byte('a') -local byte_z = string.byte('z') - -local cmdidxs = string.format([[ -static const cmdidx_T cmdidxs[%u] = { -]], byte_z - byte_a + 2) - -enumfile:write([[ -typedef enum CMD_index { -]]) -defsfile:write(string.format([[ -static CommandDefinition cmdnames[%u] = { -]], #defs)) -for i, cmd in ipairs(defs) do - local enumname = cmd.enum or ('CMD_' .. cmd.command) - firstchar = string.byte(cmd.command) - if firstchar ~= prevfirstchar then - if (not prevfirstchar - or (byte_a <= firstchar and firstchar <= byte_z) - or (byte_a <= prevfirstchar and prevfirstchar <= byte_z)) then - if not first then - cmdidxs = cmdidxs .. ',\n' - end - cmdidxs = cmdidxs .. ' ' .. enumname - end - prevfirstchar = firstchar - end - if first then - first = false - else - defsfile:write(',\n') - end - enumfile:write(' ' .. enumname .. ',\n') - defsfile:write(string.format([[ - [%s] = { - .cmd_name = (char_u *) "%s", - .cmd_func = &%s, - .cmd_argt = %uL, - .cmd_addr_type = %i - }]], enumname, cmd.command, cmd.func, cmd.flags, cmd.addr_type)) -end -defsfile:write([[ - -}; -]]) -enumfile:write([[ - CMD_SIZE, - CMD_USER = -1, - CMD_USER_BUF = -2 -} cmdidx_T; -]]) -cmdidxs = cmdidxs .. [[ - -}; -]] -defsfile:write(cmdidxs) diff --git a/scripts/genoptions.lua b/scripts/genoptions.lua deleted file mode 100644 index da53d010bd..0000000000 --- a/scripts/genoptions.lua +++ /dev/null @@ -1,184 +0,0 @@ -if arg[1] == '--help' then - print('Usage: genoptions.lua src/nvim options_file') - os.exit(0) -end - -local nvimsrcdir = arg[1] -local options_file = arg[2] - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local opt_fd = io.open(options_file, 'w') - -local w = function(s) - if s:match('^ %.') then - opt_fd:write(s .. ',\n') - else - opt_fd:write(s .. '\n') - end -end - -local options = require('options') - -cstr = options.cstr - -local type_flags={ - bool='P_BOOL', - number='P_NUM', - string='P_STRING', -} - -local redraw_flags={ - statuslines='P_RSTAT', - current_window='P_RWIN', - current_buffer='P_RBUF', - all_windows='P_RALL', - everything='P_RCLR', - curswant='P_CURSWANT', -} - -local list_flags={ - comma='P_COMMA', - onecomma='P_ONECOMMA', - flags='P_FLAGLIST', - flagscomma='P_COMMA|P_FLAGLIST', -} - -local get_flags = function(o) - local ret = {type_flags[o.type]} - local add_flag = function(f) - ret[1] = ret[1] .. '|' .. f - end - if o.list then - add_flag(list_flags[o.list]) - end - if o.redraw then - for _, r_flag in ipairs(o.redraw) do - add_flag(redraw_flags[r_flag]) - end - end - for _, flag_desc in ipairs({ - {'alloced'}, - {'expand'}, - {'nodefault'}, - {'no_mkrc'}, - {'vi_def'}, - {'vim'}, - {'secure'}, - {'gettext'}, - {'noglob'}, - {'normal_fname_chars', 'P_NFNAME'}, - {'pri_mkrc'}, - {'deny_in_modelines', 'P_NO_ML'}, - {'deny_duplicates', 'P_NODUP'}, - }) do - local key_name = flag_desc[1] - local def_name = flag_desc[2] or ('P_' .. key_name:upper()) - if o[key_name] then - add_flag(def_name) - end - end - return ret[1] -end - -local get_cond -get_cond = function(c, base_string) - local cond_string = base_string or '#if ' - if type(c) == 'table' then - cond_string = cond_string .. get_cond(c[1], '') - for i, subc in ipairs(c) do - if i > 1 then - cond_string = cond_string .. ' && ' .. get_cond(subc, '') - end - end - elseif c:sub(1, 1) == '!' then - cond_string = cond_string .. '!defined(' .. c:sub(2) .. ')' - else - cond_string = cond_string .. 'defined(' .. c .. ')' - end - return cond_string -end - -value_dumpers = { - ['function']=function(v) return v() end, - string=cstr, - boolean=function(v) return v and 'true' or 'false' end, - number=function(v) return ('%iL'):format(v) end, - ['nil']=function(v) return '0L' end, -} - -local get_value = function(v) - return '(char_u *) ' .. value_dumpers[type(v)](v) -end - -local get_defaults = function(d) - return '{' .. get_value(d.vi) .. ', ' .. get_value(d.vim) .. '}' -end - -local defines = {} - -local dump_option = function(i, o) - w(' [' .. ('%u'):format(i - 1) .. ']={') - w(' .fullname=' .. cstr(o.full_name)) - if o.abbreviation then - w(' .shortname=' .. cstr(o.abbreviation)) - end - w(' .flags=' .. get_flags(o)) - if o.enable_if then - w(get_cond(o.enable_if)) - end - if o.varname then - w(' .var=(char_u *)&' .. o.varname) - elseif #o.scope == 1 and o.scope[1] == 'window' then - w(' .var=VAR_WIN') - end - if o.enable_if then - w('#endif') - end - if #o.scope == 1 and o.scope[1] == 'global' then - w(' .indir=PV_NONE') - else - assert (#o.scope == 1 or #o.scope == 2) - assert (#o.scope == 1 or o.scope[1] == 'global') - local min_scope = o.scope[#o.scope] - local varname = o.pv_name or o.varname or ( - 'p_' .. (o.abbreviation or o.full_name)) - local pv_name = ( - 'OPT_' .. min_scope:sub(1, 3):upper() .. '(' .. ( - min_scope:sub(1, 1):upper() .. 'V_' .. varname:sub(3):upper() - ) .. ')' - ) - if #o.scope == 2 then - pv_name = 'OPT_BOTH(' .. pv_name .. ')' - end - defines['PV_' .. varname:sub(3):upper()] = pv_name - w(' .indir=' .. pv_name) - end - if o.defaults then - if o.defaults.condition then - w(get_cond(o.defaults.condition)) - end - w(' .def_val=' .. get_defaults(o.defaults.if_true)) - if o.defaults.condition then - if o.defaults.if_false then - w('#else') - w(' .def_val=' .. get_defaults(o.defaults.if_false)) - end - w('#endif') - end - end - w(' },') -end - -w('static vimoption_T options[] = {') -for i, o in ipairs(options.options) do - dump_option(i, o) -end -w(' [' .. ('%u'):format(#options.options) .. ']={.fullname=NULL}') -w('};') -w('') - -for k, v in pairs(defines) do - w('#define ' .. k .. ' ' .. v) -end -opt_fd:close() diff --git a/scripts/genunicodetables.lua b/scripts/genunicodetables.lua deleted file mode 100644 index 36339e2fc6..0000000000 --- a/scripts/genunicodetables.lua +++ /dev/null @@ -1,239 +0,0 @@ --- Script creates the following tables in unicode_tables.generated.h: --- --- 1. doublewidth and ambiguous tables: sorted list of non-overlapping closed --- intervals. Codepoints in these intervals have double (W or F) or ambiguous --- (A) east asian width respectively. --- 2. combining table: same as the above, but characters inside are combining --- characters (i.e. have general categories equal to Mn, Mc or Me). --- 3. foldCase, toLower and toUpper tables used to convert characters to --- folded/lower/upper variants. In these tables first two values are --- character ranges: like in previous tables they are sorted and must be --- non-overlapping. Third value means step inside the range: e.g. if it is --- 2 then interval applies only to first, third, fifth, … character in range. --- Fourth value is number that should be added to the codepoint to yield --- folded/lower/upper codepoint. -if arg[1] == '--help' then - print('Usage:') - print(' genunicodetables.lua UnicodeData.txt CaseFolding.txt ' .. - 'EastAsianWidth.txt') - print(' unicode_tables.generated.h') - os.exit(0) -end - -local unicodedata_fname = arg[1] -local casefolding_fname = arg[2] -local eastasianwidth_fname = arg[3] - -local utf_tables_fname = arg[4] - -local split_on_semicolons = function(s) - local ret = {} - local idx = 1 - while idx <= #s + 1 do - item = s:match('^[^;]*', idx) - idx = idx + #item + 1 - if idx <= #s + 1 then - assert(s:sub(idx - 1, idx - 1) == ';') - end - item = item:gsub('^%s*', '') - item = item:gsub('%s*$', '') - table.insert(ret, item) - end - return ret -end - -local fp_lines_to_lists = function(fp, n, has_comments) - local ret = {} - local line - local i = 0 - while true do - i = i + 1 - line = fp:read('*l') - if not line then - break - end - if (not has_comments - or (line:sub(1, 1) ~= '#' and not line:match('^%s*$'))) then - local l = split_on_semicolons(line) - if #l ~= n then - io.stderr:write(('Found %s items in line %u, expected %u\n'):format( - #l, i, n)) - io.stderr:write('Line: ' .. line .. '\n') - return nil - end - table.insert(ret, l) - end - end - return ret -end - -local parse_data_to_props = function(ud_fp) - return fp_lines_to_lists(ud_fp, 15, false) -end - -local parse_fold_props = function(cf_fp) - return fp_lines_to_lists(cf_fp, 4, true) -end - -local parse_width_props = function(eaw_fp) - return fp_lines_to_lists(eaw_fp, 2, true) -end - -local make_range = function(start, end_, step, add) - if step and add then - return (' {0x%x, 0x%x, %d, %d},\n'):format( - start, end_, step == 0 and -1 or step, add) - else - return (' {0x%04x, 0x%04x},\n'):format(start, end_) - end -end - -local build_convert_table = function(ut_fp, props, cond_func, nl_index, - table_name) - ut_fp:write('static const convertStruct ' .. table_name .. '[] = {\n') - local start = -1 - local end_ = -1 - local step = 0 - local add = -1 - for _, p in ipairs(props) do - if cond_func(p) then - local n = tonumber(p[1], 16) - local nl = tonumber(p[nl_index], 16) - if start >= 0 and add == (nl - n) and (step == 0 or n - end_ == step) then - -- Continue with the same range. - step = n - end_ - end_ = n - else - if start >= 0 then - -- Produce previous range. - ut_fp:write(make_range(start, end_, step, add)) - end - start = n - end_ = n - step = 0 - add = nl - n - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_, step, add)) - end - ut_fp:write('};\n') -end - -local build_case_table = function(ut_fp, dataprops, table_name, index) - local cond_func = function(p) - return p[index] ~= '' - end - return build_convert_table(ut_fp, dataprops, cond_func, index, - 'to' .. table_name) -end - -local build_fold_table = function(ut_fp, foldprops) - local cond_func = function(p) - return (p[2] == 'C' or p[2] == 'S') - end - return build_convert_table(ut_fp, foldprops, cond_func, 3, 'foldCase') -end - -local build_combining_table = function(ut_fp, dataprops) - ut_fp:write('static const struct interval combining[] = {\n') - local start = -1 - local end_ = -1 - for _, p in ipairs(dataprops) do - if (({Mn=true, Mc=true, Me=true})[p[3]]) then - local n = tonumber(p[1], 16) - if start >= 0 and end_ + 1 == n then - -- Continue with the same range. - end_ = n - else - if start >= 0 then - -- Produce previous range. - ut_fp:write(make_range(start, end_)) - end - start = n - end_ = n - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_)) - end - ut_fp:write('};\n') -end - -local build_width_table = function(ut_fp, dataprops, widthprops, widths, - table_name) - ut_fp:write('static const struct interval ' .. table_name .. '[] = {\n') - local start = -1 - local end_ = -1 - local dataidx = 1 - for _, p in ipairs(widthprops) do - if widths[p[2]:sub(1, 1)] then - local rng_start, rng_end = p[1]:find('%.%.') - local n, n_last - if rng_start then - -- It is a range. We don’t check for composing char then. - n = tonumber(p[1]:sub(1, rng_start - 1), 16) - n_last = tonumber(p[1]:sub(rng_end + 1), 16) - else - n = tonumber(p[1], 16) - n_last = n - end - local dn - while true do - dn = tonumber(dataprops[dataidx][1], 16) - if dn >= n then - break - end - dataidx = dataidx + 1 - end - if dn ~= n and n_last == n then - io.stderr:write('Cannot find character ' .. n .. ' in data table.\n') - end - -- Only use the char when it’s not a composing char. - -- But use all chars from a range. - local dp = dataprops[dataidx] - if (n_last > n) or (not (({Mn=true, Mc=true, Me=true})[dp[3]])) then - if start >= 0 and end_ + 1 == n then - -- Continue with the same range. - else - if start >= 0 then - ut_fp:write(make_range(start, end_)) - end - start = n - end - end_ = n_last - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_)) - end - ut_fp:write('};\n') -end - -local ud_fp = io.open(unicodedata_fname, 'r') -local dataprops = parse_data_to_props(ud_fp) -ud_fp:close() - -local ut_fp = io.open(utf_tables_fname, 'w') - -build_case_table(ut_fp, dataprops, 'Lower', 14) -build_case_table(ut_fp, dataprops, 'Upper', 13) -build_combining_table(ut_fp, dataprops) - -local cf_fp = io.open(casefolding_fname, 'r') -local foldprops = parse_fold_props(cf_fp) -cf_fp:close() - -build_fold_table(ut_fp, foldprops) - -local eaw_fp = io.open(eastasianwidth_fname, 'r') -local widthprops = parse_width_props(eaw_fp) -eaw_fp:close() - -build_width_table(ut_fp, dataprops, widthprops, {W=true, F=true}, 'doublewidth') -build_width_table(ut_fp, dataprops, widthprops, {A=true}, 'ambiguous') - -ut_fp:close() diff --git a/scripts/genvimvim.lua b/scripts/genvimvim.lua index 9135c8e3ab..806533f2ff 100644 --- a/scripts/genvimvim.lua +++ b/scripts/genvimvim.lua @@ -1,17 +1,20 @@ +mpack = require('mpack') + if arg[1] == '--help' then - print('Usage: genoptions.lua src/nvim runtime/syntax/vim/generated.vim') + print('Usage: lua genvimvim.lua src/nvim runtime/syntax/vim/generated.vim') os.exit(0) end local nvimsrcdir = arg[1] local syntax_file = arg[2] +local funcs_file = arg[3] package.path = nvimsrcdir .. '/?.lua;' .. package.path local lld = {} local syn_fd = io.open(syntax_file, 'w') lld.line_length = 0 -local w = function(s) +local function w(s) syn_fd:write(s) if s:find('\n') then lld.line_length = #(s:gsub('.*\n', '')) @@ -24,7 +27,7 @@ local options = require('options') local auevents = require('auevents') local ex_cmds = require('ex_cmds') -local cmd_kw = function(prev_cmd, cmd) +local function cmd_kw(prev_cmd, cmd) if not prev_cmd then return cmd:sub(1, 1) .. '[' .. cmd:sub(2) .. ']' else @@ -40,6 +43,15 @@ local cmd_kw = function(prev_cmd, cmd) end end +-- Exclude these from the vimCommand keyword list, they are handled specially +-- in syntax/vim.vim (vimAugroupKey, vimAutoCmd). #9327 +local function is_autocmd_cmd(cmd) + return (cmd == 'augroup' + or cmd == 'autocmd' + or cmd == 'doautocmd' + or cmd == 'doautoall') +end + vimcmd_start = 'syn keyword vimCommand contained ' w(vimcmd_start) local prev_cmd = nil @@ -48,7 +60,7 @@ for _, cmd_desc in ipairs(ex_cmds) do w('\n' .. vimcmd_start) end local cmd = cmd_desc.command - if cmd:match('%w') and cmd ~= 'z' then + if cmd:match('%w') and cmd ~= 'z' and not is_autocmd_cmd(cmd) then w(' ' .. cmd_kw(prev_cmd, cmd)) end prev_cmd = cmd @@ -82,7 +94,7 @@ local vimau_start = 'syn keyword vimAutoEvent contained ' w('\n\n' .. vimau_start) for _, au in ipairs(auevents.events) do - if not auevents.neovim_specific[au] then + if not auevents.nvim_specific[au] then if lld.line_length > 850 then w('\n' .. vimau_start) end @@ -90,7 +102,7 @@ for _, au in ipairs(auevents.events) do end end for au, _ in pairs(auevents.aliases) do - if not auevents.neovim_specific[au] then + if not auevents.nvim_specific[au] then if lld.line_length > 850 then w('\n' .. vimau_start) end @@ -101,7 +113,7 @@ end local nvimau_start = 'syn keyword nvimAutoEvent contained ' w('\n\n' .. nvimau_start) -for au, _ in pairs(auevents.neovim_specific) do +for au, _ in pairs(auevents.nvim_specific) do if lld.line_length > 850 then w('\n' .. nvimau_start) end @@ -111,28 +123,16 @@ end w('\n\nsyn case match') local vimfun_start = 'syn keyword vimFuncName contained ' w('\n\n' .. vimfun_start) -eval_fd = io.open(nvimsrcdir .. '/eval.c', 'r') +funcs = mpack.unpack(io.open(funcs_file):read("*all")) local started = 0 -for line in eval_fd:lines() do - if line == '} functions[] =' then - started = 1 - elseif started == 1 then - assert (line == '{') - started = 2 - elseif started == 2 then - if line == '};' then - break - end - local func_name = line:match('^ {"(%w+)",') - if func_name then - if lld.line_length > 850 then - w('\n' .. vimfun_start) - end - w(' ' .. func_name) +for name, def in pairs(funcs) do + if name then + if lld.line_length > 850 then + w('\n' .. vimfun_start) end + w(' ' .. name) end end -eval_fd:close() w('\n') syn_fd:close() diff --git a/scripts/git-log-pretty-since.sh b/scripts/git-log-pretty-since.sh index d8e3282fb3..a0aa4354b6 100755 --- a/scripts/git-log-pretty-since.sh +++ b/scripts/git-log-pretty-since.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -# Shows a log with changes grouped next to their merge-commit. +# Prints a nicely-formatted commit history. +# - Commits are grouped below their merge-commit. +# - Issue numbers are moved next to the commit-id. # # Parameters: # $1 "since" commit @@ -19,14 +21,32 @@ is_merge_commit() { git log $1^2 >/dev/null 2>&1 && return 0 || return 1 } -for commit in $(git log --format='%H' --first-parent --since $__SINCE); do +# Removes parens from issue/ticket/PR numbers. +# +# Example: +# in: 3340e08becbf foo (#9423) +# out: 3340e08becbf foo #9423 +_deparen() { + sed 's/(\(\#[0-9]\{3,\}\))/\1/g' +} + +# Cleans up issue/ticket/PR numbers in the commit descriptions. +# +# Example: +# in: 3340e08becbf foo (#9423) +# out: 3340e08becbf #9423 foo +_format_ticketnums() { + nvim -Es +'g/\v(#[0-9]{3,})/norm! ngEldE0ep' +'%p' | _deparen +} + +for commit in $(git log --format='%H' --first-parent "$__SINCE"..HEAD); do if is_merge_commit ${commit} ; then if [ -z "$__INVMATCH" ] || ! git log --oneline ${commit}^1..${commit}^2 \ - | grep -E "$__INVMATCH" >/dev/null 2>&1 ; then + | >/dev/null 2>&1 grep -E "$__INVMATCH" ; then git log -1 --oneline ${commit} git log --format=' %h %s' ${commit}^1..${commit}^2 fi else git log -1 --oneline ${commit} fi -done +done | _format_ticketnums diff --git a/scripts/msgpack-gen.lua b/scripts/msgpack-gen.lua deleted file mode 100644 index 2da3c174f9..0000000000 --- a/scripts/msgpack-gen.lua +++ /dev/null @@ -1,313 +0,0 @@ -lpeg = require('lpeg') -mpack = require('mpack') - --- lpeg grammar for building api metadata from a set of header files. It --- ignores comments and preprocessor commands and parses a very small subset --- of C prototypes with a limited set of types -P, R, S = lpeg.P, lpeg.R, lpeg.S -C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg - -any = P(1) -- (consume one character) -letter = R('az', 'AZ') + S('_$') -alpha = letter + R('09') -nl = P('\r\n') + P('\n') -not_nl = any - nl -ws = S(' \t') + nl -fill = ws ^ 0 -c_comment = P('//') * (not_nl ^ 0) -c_preproc = P('#') * (not_nl ^ 0) -typed_container = - (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')') -c_id = ( - typed_container + - (letter * (alpha ^ 0)) -) -c_void = P('void') -c_param_type = ( - ((P('Error') * fill * P('*') * fill) * Cc('error')) + - (C(c_id) * (ws ^ 1)) - ) -c_type = (C(c_void) * (ws ^ 1)) + c_param_type -c_param = Ct(c_param_type * C(c_id)) -c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0) -c_params = Ct(c_void + c_param_list) -c_proto = Ct( - Cg(c_type, 'return_type') * Cg(c_id, 'name') * - fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') * - Cg(Cc(false), 'async') * - (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) * - (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) * - fill * P(';') - ) -grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1) - --- we need at least 2 arguments since the last one is the output file -assert(#arg >= 1) -functions = {} - --- names of all headers relative to the source root (for inclusion in the --- generated file) -headers = {} --- output file(dispatch function + metadata serialized with msgpack) -outputf = arg[#arg] - --- read each input file, parse and append to the api metadata -for i = 1, #arg - 1 do - local full_path = arg[i] - local parts = {} - for part in string.gmatch(full_path, '[^/]+') do - parts[#parts + 1] = part - end - headers[#headers + 1] = parts[#parts - 1]..'/'..parts[#parts] - - local input = io.open(full_path, 'rb') - local tmp = grammar:match(input:read('*all')) - for i = 1, #tmp do - local fn = tmp[i] - if not fn.noexport then - functions[#functions + 1] = tmp[i] - if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then - -- this function should receive the channel id - fn.receives_channel_id = true - -- remove the parameter since it won't be passed by the api client - table.remove(fn.parameters, 1) - end - if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then - -- function can fail if the last parameter type is 'Error' - fn.can_fail = true - -- remove the error parameter, msgpack has it's own special field - -- for specifying errors - fn.parameters[#fn.parameters] = nil - end - end - end - input:close() -end - - --- start building the output -output = io.open(outputf, 'wb') - -output:write([[ -#include <inttypes.h> -#include <stdbool.h> -#include <stdint.h> -#include <assert.h> -#include <msgpack.h> - -#include "nvim/map.h" -#include "nvim/log.h" -#include "nvim/vim.h" -#include "nvim/msgpack_rpc/helpers.h" -#include "nvim/msgpack_rpc/defs.h" -#include "nvim/api/private/helpers.h" -#include "nvim/api/private/defs.h" -]]) - -for i = 1, #headers do - if headers[i]:sub(-12) ~= '.generated.h' then - output:write('\n#include "nvim/'..headers[i]..'"') - end -end - -output:write([[ - - -static const uint8_t msgpack_metadata[] = { - -]]) --- serialize the API metadata using msgpack and embed into the resulting --- binary for easy querying by clients -packed = mpack.pack(functions) -for i = 1, #packed do - output:write(string.byte(packed, i)..', ') - if i % 10 == 0 then - output:write('\n ') - end -end -output:write([[ -}; - -void msgpack_rpc_init_function_metadata(Dictionary *metadata) -{ - msgpack_unpacked unpacked; - msgpack_unpacked_init(&unpacked); - if (msgpack_unpack_next(&unpacked, - (const char *)msgpack_metadata, - sizeof(msgpack_metadata), - NULL) != MSGPACK_UNPACK_SUCCESS) { - abort(); - } - Object functions; - msgpack_rpc_to_object(&unpacked.data, &functions); - msgpack_unpacked_destroy(&unpacked); - PUT(*metadata, "functions", functions); -} - -]]) - -local function real_type(type) - local rv = type - if typed_container:match(rv) then - if rv:match('Array') then - rv = 'Array' - else - rv = 'Dictionary' - end - end - return rv -end - --- start the handler functions. Visit each function metadata to build the --- handler function with code generated for validating arguments and calling to --- the real API. -for i = 1, #functions do - local fn = functions[i] - local args = {} - - output:write('static Object handle_'..fn.name..'(uint64_t channel_id, uint64_t request_id, Array args, Error *error)') - output:write('\n{') - output:write('\n Object ret = NIL;') - -- Declare/initialize variables that will hold converted arguments - for j = 1, #fn.parameters do - local param = fn.parameters[j] - local converted = 'arg_'..j - output:write('\n '..param[1]..' '..converted..' api_init_'..string.lower(real_type(param[1]))..';') - end - output:write('\n') - output:write('\n if (args.size != '..#fn.parameters..') {') - output:write('\n snprintf(error->msg, sizeof(error->msg), "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);') - output:write('\n error->set = true;') - output:write('\n goto cleanup;') - output:write('\n }\n') - - -- Validation/conversion for each argument - for j = 1, #fn.parameters do - local converted, convert_arg, param, arg - param = fn.parameters[j] - converted = 'arg_'..j - local rt = real_type(param[1]) - if rt ~= 'Object' then - output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {') - output:write('\n '..converted..' = args.items['..(j - 1)..'].data.'..rt:lower()..';') - if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then - -- accept positive integers for Buffers, Windows and Tabpages - output:write('\n } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer > 0) {') - output:write('\n '..converted..' = (unsigned)args.items['..(j - 1)..'].data.integer;') - end - output:write('\n } else {') - output:write('\n snprintf(error->msg, sizeof(error->msg), "Wrong type for argument '..j..', expecting '..param[1]..'");') - output:write('\n error->set = true;') - output:write('\n goto cleanup;') - output:write('\n }\n') - else - output:write('\n '..converted..' = args.items['..(j - 1)..'];\n') - end - - args[#args + 1] = converted - end - - -- function call - local call_args = table.concat(args, ', ') - output:write('\n ') - if fn.return_type ~= 'void' then - -- has a return value, prefix the call with a declaration - output:write(fn.return_type..' rv = ') - end - - -- write the function name and the opening parenthesis - output:write(fn.name..'(') - - if fn.receives_channel_id then - -- if the function receives the channel id, pass it as first argument - if #args > 0 or fn.can_fail then - output:write('channel_id, '..call_args) - else - output:write('channel_id') - end - else - output:write(call_args) - end - - if fn.can_fail then - -- if the function can fail, also pass a pointer to the local error object - if #args > 0 then - output:write(', error);\n') - else - output:write('error);\n') - end - -- and check for the error - output:write('\n if (error->set) {') - output:write('\n goto cleanup;') - output:write('\n }\n') - else - output:write(');\n') - end - - if fn.return_type ~= 'void' then - output:write('\n ret = '..string.upper(real_type(fn.return_type))..'_OBJ(rv);') - end - -- Now generate the cleanup label for freeing memory allocated for the - -- arguments - output:write('\n\ncleanup:'); - - for j = 1, #fn.parameters do - local param = fn.parameters[j] - output:write('\n api_free_'..string.lower(real_type(param[1]))..'(arg_'..j..');') - end - output:write('\n return ret;\n}\n\n'); -end - --- Generate a function that initializes method names with handler functions -output:write([[ -static Map(String, MsgpackRpcRequestHandler) *methods = NULL; - -void msgpack_rpc_add_method_handler(String method, MsgpackRpcRequestHandler handler) -{ - map_put(String, MsgpackRpcRequestHandler)(methods, method, handler); -} - -void msgpack_rpc_init_method_table(void) -{ - methods = map_new(String, MsgpackRpcRequestHandler)(); - -]]) - --- Keep track of the maximum method name length in order to avoid walking --- strings longer than that when searching for a method handler -local max_fname_len = 0 -for i = 1, #functions do - local fn = functions[i] - output:write(' msgpack_rpc_add_method_handler('.. - '(String) {.data = "'..fn.name..'", '.. - '.size = sizeof("'..fn.name..'") - 1}, '.. - '(MsgpackRpcRequestHandler) {.fn = handle_'.. fn.name.. - ', .async = '..tostring(fn.async)..'});\n') - - if #fn.name > max_fname_len then - max_fname_len = #fn.name - end -end - -output:write('\n}\n\n') - -output:write([[ -MsgpackRpcRequestHandler msgpack_rpc_get_handler_for(const char *name, - size_t name_len) -{ - String m = { - .data=(char *)name, - .size=MIN(name_len, ]]..max_fname_len..[[) - }; - MsgpackRpcRequestHandler rv = - map_get(String, MsgpackRpcRequestHandler)(methods, m); - - if (!rv.fn) { - rv.fn = msgpack_rpc_handle_missing_method; - } - - return rv; -} -]]) - -output:close() diff --git a/scripts/pvscheck.sh b/scripts/pvscheck.sh new file mode 100755 index 0000000000..371af7c7e5 --- /dev/null +++ b/scripts/pvscheck.sh @@ -0,0 +1,491 @@ +#!/bin/sh + +# Assume that "local" is available. +# shellcheck disable=SC2039 + +set -e +# Note: -u causes problems with posh, it barks at “undefined” $@ when no +# arguments provided. +test -z "$POSH_VERSION" && set -u + +log_info() { + >&2 printf "pvscheck.sh: %s\n" "$@" +} + +get_jobs_num() { + if [ -n "${TRAVIS:-}" ] ; then + # HACK: /proc/cpuinfo on Travis CI is misleading, so hardcode 1. + echo 1 + else + echo $(( $(grep -c "^processor" /proc/cpuinfo) + 1 )) + fi +} + +help() { + echo 'Usage:' + echo ' pvscheck.sh [--pvs URL] [--deps] [--environment-cc]' + echo ' [target-directory [branch]]' + echo ' pvscheck.sh [--pvs URL] [--recheck] [--environment-cc] [--update]' + echo ' [target-directory]' + echo ' pvscheck.sh [--pvs URL] --only-analyse [target-directory]' + echo ' pvscheck.sh [--pvs URL] --pvs-install {target-directory}' + echo ' pvscheck.sh --patch [--only-build]' + echo + echo ' --pvs: Fetch pvs-studio from URL.' + echo + echo ' --pvs detect: Auto-detect latest version (by scraping viva64.com).' + echo + echo ' --deps: (for regular run) Use top-level Makefile and build deps.' + echo ' Without this it assumes all dependencies are already' + echo ' installed.' + echo + echo ' --environment-cc: (for regular run and --recheck) Do not export' + echo ' CC=clang. Build is still run with CFLAGS=-O0.' + echo + echo ' --only-build: (for --patch) Only patch files in ./build directory.' + echo + echo ' --pvs-install: Only install PVS-studio to the specified location.' + echo + echo ' --patch: patch sources in the current directory.' + echo ' Does not patch already patched files.' + echo ' Does not run analysis.' + echo + echo ' --recheck: run analysis on a prepared target directory.' + echo + echo ' --update: when rechecking first do a pull.' + echo + echo ' --only-analyse: run analysis on a prepared target directory ' + echo ' without building Neovim.' + echo + echo ' target-directory: Directory where build should occur.' + echo ' Default: ../neovim-pvs' + echo + echo ' branch: Branch to check.' + echo ' Default: master.' +} + +getopts_error() { + local msg="$1" ; shift + local do_help= + if test "$msg" = "--help" ; then + msg="$1" ; shift + do_help=1 + fi + printf '%s\n' "$msg" >&2 + if test -n "$do_help" ; then + printf '\n' >&2 + help >&2 + fi + echo 'return 1' + return 1 +} + +# Usage `eval "$(getopts_long long_defs -- positionals_defs -- "$@")"` +# +# long_defs: list of pairs of arguments like `longopt action`. +# positionals_defs: list of arguments like `action`. +# +# `action` is a space-separated commands: +# +# store_const [const] [varname] [default] +# Store constant [const] (default 1) (note: eval’ed) if argument is present +# (long options only). Assumes long option accepts no arguments. +# store [varname] [default] +# Store value. Assumes long option needs an argument. +# run {func} [varname] [default] +# Run function {func} and store its output to the [varname]. Assumes no +# arguments accepted (long options only). +# modify {func} [varname] [default] +# Like run, but assumes a single argument, passed to function {func} as $1. +# +# All actions stores empty value if neither [varname] nor [default] are +# present. [default] is evaled by top-level `eval`, so be careful. Also note +# that no arguments may contain spaces, including [default] and [const]. +getopts_long() { + local positional= + local opt_bases="" + while test $# -gt 0 ; do + local arg="$1" ; shift + local opt_base= + local act= + local opt_name= + if test -z "$positional" ; then + if test "$arg" = "--" ; then + positional=0 + continue + fi + act="$1" ; shift + opt_name="$(echo "$arg" | tr '-' '_')" + opt_base="longopt_$opt_name" + else + if test "$arg" = "--" ; then + break + fi + : $(( positional+=1 )) + act="$arg" + opt_name="arg_$positional" + opt_base="positional_$positional" + fi + opt_bases="$opt_bases $opt_base" + eval "local varname_$opt_base=$opt_name" + local i=0 + for act_subarg in $act ; do + eval "local act_$(( i+=1 ))_$opt_base=\"\$act_subarg\"" + done + done + # Process options + local positional=0 + local force_positional= + while test $# -gt 0 ; do + local argument="$1" ; shift + local opt_base= + local has_equal= + local equal_arg= + local is_positional= + if test "$argument" = "--" ; then + force_positional=1 + continue + elif test -z "$force_positional" && test "${argument#--}" != "$argument" + then + local opt_name="${argument#--}" + local opt_name_striparg="${opt_name%%=*}" + if test "$opt_name" = "$opt_name_striparg" ; then + has_equal=0 + else + has_equal=1 + equal_arg="${argument#*=}" + opt_name="$opt_name_striparg" + fi + # Use trailing x to prevent stripping newlines + opt_name="$(printf '%sx' "$opt_name" | tr '-' '_')" + opt_name="${opt_name%x}" + if test -n "$(printf '%sx' "$opt_name" | tr -d 'a-z_')" ; then + getopts_error "Option contains invalid characters: $opt_name" + fi + opt_base="longopt_$opt_name" + else + : $(( positional+=1 )) + opt_base="positional_$positional" + is_positional=1 + fi + if test -n "$opt_base" ; then + eval "local occurred_$opt_base=1" + + eval "local act_1=\"\${act_1_$opt_base:-}\"" + eval "local varname=\"\${varname_$opt_base:-}\"" + local need_val= + local func= + case "$act_1" in + (store_const) + eval "local const=\"\${act_2_${opt_base}:-1}\"" + eval "local varname=\"\${act_3_${opt_base}:-$varname}\"" + printf 'local %s=%s\n' "$varname" "$const" + ;; + (store) + eval "varname=\"\${act_2_${opt_base}:-$varname}\"" + need_val=1 + ;; + (run) + eval "func=\"\${act_2_${opt_base}}\"" + eval "varname=\"\${act_3_${opt_base}:-$varname}\"" + printf 'local %s="$(%s)"\n' "$varname" "$func" + ;; + (modify) + eval "func=\"\${act_2_${opt_base}}\"" + eval "varname=\"\${act_3_${opt_base}:-$varname}\"" + need_val=1 + ;; + ("") + getopts_error --help "Wrong argument: $argument" + ;; + esac + if test -n "$need_val" ; then + local val= + if test -z "$is_positional" ; then + if test $has_equal = 1 ; then + val="$equal_arg" + else + if test $# -eq 0 ; then + getopts_error "Missing argument for $opt_name" + fi + val="$1" ; shift + fi + else + val="$argument" + fi + local escaped_val="'$(printf "%s" "$val" | sed "s/'/'\\\\''/g")'" + case "$act_1" in + (store) + printf 'local %s=%s\n' "$varname" "$escaped_val" + ;; + (modify) + printf 'local %s="$(%s %s)"\n' "$varname" "$func" "$escaped_val" + ;; + esac + fi + fi + done + # Print default values when no values were provided + local opt_base= + for opt_base in $opt_bases ; do + eval "local occurred=\"\${occurred_$opt_base:-}\"" + if test -n "$occurred" ; then + continue + fi + eval "local act_1=\"\$act_1_$opt_base\"" + eval "local varname=\"\$varname_$opt_base\"" + case "$act_1" in + (store) + eval "local varname=\"\${act_2_${opt_base}:-$varname}\"" + eval "local default=\"\${act_3_${opt_base}:-}\"" + printf 'local %s=%s\n' "$varname" "$default" + ;; + (store_const|run|modify) + eval "local varname=\"\${act_3_${opt_base}:-$varname}\"" + eval "local default=\"\${act_4_${opt_base}:-}\"" + printf 'local %s=%s\n' "$varname" "$default" + ;; + esac + done +} + +get_pvs_comment() { + local tgt="$1" ; shift + + cat > "$tgt/pvs-comment" << EOF +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +EOF +} + +install_pvs() {( + local tgt="$1" ; shift + local pvs_url="$1" ; shift + + cd "$tgt" + + if test -d pvs-studio ; then + log_info 'install_pvs: "pvs-studio" directory already exists, skipping install' + return 0 + fi + + mkdir pvs-studio + cd pvs-studio + + curl -L -o pvs-studio.tar.gz "$pvs_url" + tar xzf pvs-studio.tar.gz + rm pvs-studio.tar.gz + local pvsdir="$(find . -maxdepth 1 -mindepth 1)" + find "$pvsdir" -maxdepth 1 -mindepth 1 -exec mv '{}' . \; + rmdir "$pvsdir" +)} + +create_compile_commands() {( + local tgt="$1" ; shift + local deps="$1" ; shift + local environment_cc="$1" ; shift + + if test -z "$environment_cc" ; then + export CC=clang + fi + export CFLAGS=' -O0 ' + + if test -z "$deps" ; then + mkdir -p "$tgt/build" + ( + cd "$tgt/build" + + cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$PWD/root" + make -j"$(get_jobs_num)" + ) + else + ( + cd "$tgt" + + make -j"$(get_jobs_num)" CMAKE_EXTRA_FLAGS=" -DCMAKE_INSTALL_PREFIX=$PWD/root -DCMAKE_BUILD_TYPE=Debug " + ) + fi + find "$tgt/build/src/nvim/auto" -name '*.test-include.c' -delete +)} + +# Warning: realdir below only cares about directories unlike realpath. +# +# realpath is not available in Ubuntu trusty yet. +realdir() {( + local dir="$1" + local add="" + while ! cd "$dir" 2>/dev/null ; do + add="${dir##*/}/$add" + local new_dir="${dir%/*}" + if test "$new_dir" = "$dir" ; then + return 1 + fi + dir="$new_dir" + done + printf '%s\n' "$PWD/$add" +)} + +patch_sources() {( + local tgt="$1" ; shift + local only_bulid="${1}" ; shift + + get_pvs_comment "$tgt" + + local sh_script=' + pvs_comment="$(cat pvs-comment ; echo -n EOS)" + filehead="$(head -c $(( ${#pvs_comment} - 3 )) "$1" ; echo -n EOS)" + if test "x$filehead" != "x$pvs_comment" ; then + cat pvs-comment "$1" > "$1.tmp" + mv "$1.tmp" "$1" + fi + ' + + cd "$tgt" + + if test "$only_build" != "--only-build" ; then + find \ + src/nvim test/functional/fixtures test/unit/fixtures \ + -name '*.c' \ + -exec /bin/sh -c "$sh_script" - '{}' \; + fi + + find \ + build/src/nvim/auto build/config \ + -name '*.c' -not -name '*.test-include.c' \ + -exec /bin/sh -c "$sh_script" - '{}' \; + + rm pvs-comment +)} + +run_analysis() {( + local tgt="$1" ; shift + + cd "$tgt" + + # pvs-studio-analyzer exits with a non-zero exit code when there are detected + # errors, so ignore its return + pvs-studio-analyzer \ + analyze \ + --threads "$(get_jobs_num)" \ + --output-file PVS-studio.log \ + --verbose \ + --file build/compile_commands.json \ + --sourcetree-root . || true + + rm -rf PVS-studio.{xml,err,tsk,html.d} + local plog_args="PVS-studio.log --srcRoot . --excludedCodes V011" + plog-converter $plog_args --renderTypes xml --output PVS-studio.xml + plog-converter $plog_args --renderTypes errorfile --output PVS-studio.err + plog-converter $plog_args --renderTypes tasklist --output PVS-studio.tsk + plog-converter $plog_args --renderTypes fullhtml --output PVS-studio.html.d +)} + +detect_url() { + local url="${1:-detect}" + if test "$url" = detect ; then + curl --silent -L 'https://www.viva64.com/en/pvs-studio-download/' \ + | grep -o 'https\{0,1\}://[^"<>]\{1,\}/pvs-studio[^/"<>]*-x86_64\.tgz' \ + || echo FAILED + else + printf '%s' "$url" + fi +} + +do_check() { + local tgt="$1" ; shift + local branch="$1" ; shift + local pvs_url="$1" ; shift + local deps="$1" ; shift + local environment_cc="$1" ; shift + + if test -z "$pvs_url" || test "$pvs_url" = FAILED ; then + pvs_url="$(detect_url detect)" + if test -z "$pvs_url" || test "$pvs_url" = FAILED ; then + echo "failed to auto-detect PVS URL" + exit 1 + fi + echo "Auto-detected PVS URL: ${pvs_url}" + fi + + git clone --branch="$branch" . "$tgt" + + install_pvs "$tgt" "$pvs_url" + + do_recheck "$tgt" "$deps" "$environment_cc" "" +} + +do_recheck() { + local tgt="$1" ; shift + local deps="$1" ; shift + local environment_cc="$1" ; shift + local update="$1" ; shift + + if test -n "$update" ; then + ( + cd "$tgt" + local branch="$(git rev-parse --abbrev-ref HEAD)" + git checkout --detach + git fetch -f origin "${branch}:${branch}" + git checkout -f "$branch" + ) + fi + + create_compile_commands "$tgt" "$deps" "$environment_cc" + + do_analysis "$tgt" +} + +do_analysis() { + local tgt="$1" ; shift + + if test -d "$tgt/pvs-studio" ; then + local saved_pwd="$PWD" + cd "$tgt/pvs-studio" + export PATH="$PWD/bin${PATH+:}${PATH}" + cd "$saved_pwd" + fi + + run_analysis "$tgt" +} + +main() { + eval "$( + getopts_long \ + help store_const \ + pvs 'modify detect_url pvs_url' \ + patch store_const \ + only-build 'store_const --only-build' \ + recheck store_const \ + only-analyse store_const \ + pvs-install store_const \ + deps store_const \ + environment-cc store_const \ + update store_const \ + -- \ + 'modify realdir tgt "$PWD/../neovim-pvs"' \ + 'store branch master' \ + -- "$@" + )" + + if test -n "$help" ; then + help + return 0 + fi + + # set -x + + if test -n "$patch" ; then + patch_sources "$tgt" "$only_build" + elif test -n "$pvs_install" ; then + install_pvs "$tgt" "$pvs_url" + elif test -n "$recheck" ; then + do_recheck "$tgt" "$deps" "$environment_cc" "$update" + elif test -n "$only_analyse" ; then + do_analysis "$tgt" + else + do_check "$tgt" "$branch" "$pvs_url" "$deps" "$environment_cc" + fi +} + +main "$@" diff --git a/scripts/release.sh b/scripts/release.sh index 67738ccc96..fb266ad154 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,63 +1,99 @@ -#!/bin/sh +#!/usr/bin/env bash +# Usage: +# ./scripts/release.sh +# ./scripts/release.sh --use-current-commit +# ./scripts/release.sh --only-bump +# # Performs steps to tag a release. # # Steps: # Create the "release" commit: # - CMakeLists.txt: Unset NVIM_VERSION_PRERELEASE +# - CMakeLists.txt: Unset NVIM_API_PRERELEASE +# - Create test/functional/fixtures/api_level_N.mpack # - Tag the commit. # Create the "version bump" commit: # - CMakeLists.txt: Set NVIM_VERSION_PRERELEASE to "-dev" -# -# Manual steps: -# - CMakeLists.txt: Bump NVIM_VERSION_* as appropriate. -# - git push --follow-tags set -e set -u set -o pipefail +ARG1=${1:-no} + +__sed=$( [ "$(uname)" = Darwin ] && echo 'sed -E' || echo 'sed -r' ) + cd "$(git rev-parse --show-toplevel)" __LAST_TAG=$(git describe --abbrev=0) [ -z "$__LAST_TAG" ] && { echo 'ERROR: no tag found'; exit 1; } __VERSION_MAJOR=$(grep 'set(NVIM_VERSION_MAJOR' CMakeLists.txt\ - |sed -r 's/.*NVIM_VERSION_MAJOR ([[:digit:]]).*/\1/') + |$__sed 's/.*NVIM_VERSION_MAJOR ([[:digit:]]).*/\1/') __VERSION_MINOR=$(grep 'set(NVIM_VERSION_MINOR' CMakeLists.txt\ - |sed -r 's/.*NVIM_VERSION_MINOR ([[:digit:]]).*/\1/') + |$__sed 's/.*NVIM_VERSION_MINOR ([[:digit:]]).*/\1/') __VERSION_PATCH=$(grep 'set(NVIM_VERSION_PATCH' CMakeLists.txt\ - |sed -r 's/.*NVIM_VERSION_PATCH ([[:digit:]]).*/\1/') + |$__sed 's/.*NVIM_VERSION_PATCH ([[:digit:]]).*/\1/') __VERSION="${__VERSION_MAJOR}.${__VERSION_MINOR}.${__VERSION_PATCH}" +__API_LEVEL=$(grep 'set(NVIM_API_LEVEL ' CMakeLists.txt\ + |$__sed 's/.*NVIM_API_LEVEL ([[:digit:]]).*/\1/') { [ -z "$__VERSION_MAJOR" ] || [ -z "$__VERSION_MINOR" ] || [ -z "$__VERSION_PATCH" ]; } \ && { echo "ERROR: version parse failed: '${__VERSION}'"; exit 1; } __RELEASE_MSG="NVIM v${__VERSION} -Features: +FEATURES: -Fixes: +FIXES: -Changes: +CHANGES: " __BUMP_MSG="version bump" echo "Most recent tag: ${__LAST_TAG}" echo "Release version: ${__VERSION}" -sed -i -r 's/(NVIM_VERSION_PRERELEASE) "-dev"/\1 ""/' CMakeLists.txt -echo "Building changelog since ${__LAST_TAG}..." -__CHANGELOG="$(./scripts/git-log-pretty-since.sh "$__LAST_TAG" 'vim-patch:\S')" -git add CMakeLists.txt -git commit --edit -m "${__RELEASE_MSG} ${__CHANGELOG}" -git tag -a v"${__VERSION}" -m "NVIM v${__VERSION}" +_do_release_commit() { + $__sed -i.bk 's/(NVIM_VERSION_PRERELEASE) "-dev"/\1 ""/' CMakeLists.txt + if grep '(NVIM_API_PRERELEASE true)' CMakeLists.txt > /dev/null; then + $__sed -i.bk 's/(NVIM_API_PRERELEASE) true/\1 false/' CMakeLists.txt + build/bin/nvim --api-info > test/functional/fixtures/api_level_$__API_LEVEL.mpack + git add test/functional/fixtures/api_level_$__API_LEVEL.mpack + fi + + if ! test "$ARG1" = '--use-current-commit' ; then + echo "Building changelog since ${__LAST_TAG}..." + __CHANGELOG="$(./scripts/git-log-pretty-since.sh "$__LAST_TAG" 'vim-patch:[^[:space:]]')" + + git add CMakeLists.txt + git commit --edit -m "${__RELEASE_MSG} ${__CHANGELOG}" + fi + + git tag --sign -a v"${__VERSION}" -m "NVIM v${__VERSION}" +} + +_do_bump_commit() { + $__sed -i.bk 's/(NVIM_VERSION_PRERELEASE) ""/\1 "-dev"/' CMakeLists.txt + $__sed -i.bk 's/set\((NVIM_VERSION_PATCH) [[:digit:]]/set(\1 ?/' CMakeLists.txt + nvim +'/NVIM_VERSION' +10new +'exe "norm! iUpdate version numbers!!!\<CR>"' \ + +'norm! 10.' CMakeLists.txt + + git add CMakeLists.txt + git commit -m "$__BUMP_MSG" -sed -i -r 's/(NVIM_VERSION_PRERELEASE) ""/\1 "-dev"/' CMakeLists.txt -nvim -c '/NVIM_VERSION' -c 'echo "Update version numbers"' CMakeLists.txt -git add CMakeLists.txt -git commit -m "$__BUMP_MSG" + rm CMakeLists.txt.bk || true +} +if ! test "$ARG1" = '--only-bump' ; then + _do_release_commit +fi +_do_bump_commit echo " Next steps: - Double-check NVIM_VERSION_* in CMakeLists.txt - - git push --follow-tags - - update website: index.html" + - Push the tag: + git push --follow-tags + - Empty-merge (if this is a maintenance release): + git checkout upstream/master + git merge -s ours upstream/release-x.y + - Update website: index.html" diff --git a/scripts/run-api-tests.exp b/scripts/run-api-tests.exp deleted file mode 100755 index 27c9c963e5..0000000000 --- a/scripts/run-api-tests.exp +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env expect - -if {$argc < 2} { - puts "Need commands for running the tests and for starting nvim" - exit 1 -} - -set timeout 60 -set run_tests [split [lindex $argv 0] " "] -set run_nvim [split [lindex $argv 1] " "] - -# don't echo to stdout -log_user 0 -# set NVIM_LISTEN_ADDRESS, so nvim will listen on a known socket -set env(NVIM_LISTEN_ADDRESS) "/tmp/nvim-[exec date +%s%N].sock" -# start nvim -spawn {*}$run_nvim -# save the job descriptor -set nvim_id $spawn_id -# Reset function that can be invoked by test runners to put nvim in a cleaner -# state -send { -:echo "read"."y" -} -# wait until nvim is ready -expect "ready" -# run tests -spawn {*}$run_tests -set tests_id $spawn_id -set status 1 -# listen for test output in the background -expect_background { - * { - # show test output to the user - send_user -- $expect_out(buffer) - } - eof { - # collect the exit status code - set spawn_id $tests_id - catch wait result - set status [lindex $result 3] - set spawn_id $nvim_id - # quit nvim - send ":qa!\r" - } -} -# switch back nvim and wait until it exits -set spawn_id $nvim_id -expect eof -exit $status diff --git a/scripts/shadacat.py b/scripts/shadacat.py index 2f2cf19f9d..522379aad4 100755 --- a/scripts/shadacat.py +++ b/scripts/shadacat.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.4 +#!/usr/bin/env python3 import os import sys diff --git a/scripts/update_terminfo.sh b/scripts/update_terminfo.sh new file mode 100755 index 0000000000..0cfc230ca6 --- /dev/null +++ b/scripts/update_terminfo.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# +# usage: ./scripts/update_terminfo.sh +# +# This script does: +# +# 1. Download Dickey's terminfo.src +# 2. Compile temporary terminfo database from terminfo.src +# 3. Use database to generate src/nvim/tui/terminfo_defs.h +# + +set -e + +url='https://invisible-island.net/datafiles/current/terminfo.src.gz' +target='src/nvim/tui/terminfo_defs.h' + +readonly -A entries=( + [ansi]=ansi_terminfo + [interix]=interix_8colour_terminfo + [iterm2]=iterm_256colour_terminfo + [linux]=linux_16colour_terminfo + [putty-256color]=putty_256colour_terminfo + [rxvt-256color]=rxvt_256colour_terminfo + [screen-256color]=screen_256colour_terminfo + [st-256color]=st_256colour_terminfo + [tmux-256color]=tmux_256colour_terminfo + [vte-256color]=vte_256colour_terminfo + [xterm-256color]=xterm_256colour_terminfo + [cygwin]=cygwin_terminfo + [win32con]=win32con_terminfo + [conemu]=conemu_terminfo + [vtpcon]=vtpcon_terminfo +) + +db="$(mktemp -du)" + +print_bold() { + printf "\\e[1m$*\\e[0m" +} + +cd "$(git rev-parse --show-toplevel)" + +# +# Get terminfo.src +# +print_bold '[*] Get terminfo.src\n' +curl -O "$url" +gunzip -f terminfo.src.gz + +# +# Build terminfo database +# +print_bold '[*] Build terminfo database\n' +cat terminfo.src scripts/windows.ti | tic -x -o "$db" - +rm -f terminfo.src + +# +# Write src/nvim/tui/terminfo_defs.h +# +print_bold "[*] Writing $target... " +sorted_terms="$(echo "${!entries[@]}" | tr ' ' '\n' | sort | xargs)" + +cat > "$target" <<EOF +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +// +// Generated by scripts/update_terminfo.sh and $(tic -V) +// + +#ifndef NVIM_TUI_TERMINFO_DEFS_H +#define NVIM_TUI_TERMINFO_DEFS_H + +#include <stdint.h> +EOF + +for term in $sorted_terms; do + path="$(find "$db" -name "$term")" + if [ -z "$path" ]; then + >&2 echo "Not found: $term. Skipping." + continue + fi + printf '\n' + infocmp -L -1 -A "$db" "$term" | sed -e '1d' -e 's#^#// #' | tr '\t' ' ' + printf 'static const int8_t %s[] = {\n' "${entries[$term]}" + printf ' ' + od -v -t d1 < "$path" | cut -c9- | xargs | tr ' ' ',' | tr -d '\n' + printf ' // NOLINT\n};\n' +done >> "$target" + +cat >> "$target" <<EOF +#endif // NVIM_TUI_TERMINFO_DEFS_H +EOF +print_bold 'done\n' diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index a40090d4c3..11305421e5 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -2,10 +2,11 @@ set -e set -u -set -o pipefail +# Use privileged mode, which e.g. skips using CDPATH. +set -p -readonly NEOVIM_SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -readonly VIM_SOURCE_DIR_DEFAULT="${NEOVIM_SOURCE_DIR}/.vim-src" +readonly NVIM_SOURCE_DIR="${NVIM_SOURCE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +readonly VIM_SOURCE_DIR_DEFAULT="${NVIM_SOURCE_DIR}/.vim-src" readonly VIM_SOURCE_DIR="${VIM_SOURCE_DIR:-${VIM_SOURCE_DIR_DEFAULT}}" readonly BASENAME="$(basename "${0}")" readonly BRANCH_PREFIX="vim-" @@ -13,22 +14,34 @@ readonly BRANCH_PREFIX="vim-" CREATED_FILES=() usage() { - echo "Helper script for porting Vim patches. For more information, see" + echo "Port Vim patches to Neovim" echo "https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim" echo echo "Usage: ${BASENAME} [-h | -l | -p vim-revision | -r pr-number]" echo echo "Options:" echo " -h Show this message and exit." - echo " -l Show list of Vim patches missing from Neovim." - echo " -p {vim-revision} Download and apply the Vim patch vim-revision." - echo " vim-revision can be a version number of the " - echo " format '7.4.xxx' or a Git commit hash." - echo " -s Submit a vim-patch pull request to Neovim." - echo " -r {pr-number} Review a vim-patch pull request to Neovim." + echo " -l List missing Vim patches." + echo " -L List missing Vim patches (for scripts)." + echo " -M List all merged patch-numbers (at current v:version)." + echo " -p {vim-revision} Download and generate a Vim patch. vim-revision" + echo " can be a Vim version (8.0.xxx) or a Git hash." + echo " -P {vim-revision} Download, generate and apply a Vim patch." + echo " -g {vim-revision} Download a Vim patch." + echo " -s Create a vim-patch pull request." + echo " -r {pr-number} Review a vim-patch pull request." + echo ' -V Clone the Vim source code to $VIM_SOURCE_DIR.' echo - echo "Set VIM_SOURCE_DIR to change where Vim's sources are stored." - echo "The default is '${VIM_SOURCE_DIR_DEFAULT}'." + echo ' $VIM_SOURCE_DIR controls where Vim sources are found' + echo " (default: '${VIM_SOURCE_DIR_DEFAULT}')" +} + +msg_ok() { + printf '\e[32m✔\e[0m %s\n' "$@" +} + +msg_err() { + printf '\e[31m✘\e[0m %s\n' "$@" } # Checks if a program is in the user's PATH, and is executable. @@ -57,10 +70,10 @@ clean_files() { read -p "Delete these files (Y/n)? " -n 1 -r reply echo - if [[ "${reply}" =~ ^[Yy]$ ]]; then - rm -- "${CREATED_FILES[@]}" - else + if [[ "${reply}" == n ]]; then echo "You can use 'git clean' to remove these files when you're done." + else + rm -- "${CREATED_FILES[@]}" fi } @@ -68,26 +81,30 @@ get_vim_sources() { require_executable git if [[ ! -d ${VIM_SOURCE_DIR} ]]; then - echo "Cloning Vim sources into '${VIM_SOURCE_DIR}'." + echo "Cloning Vim into: ${VIM_SOURCE_DIR}" git clone https://github.com/vim/vim.git "${VIM_SOURCE_DIR}" cd "${VIM_SOURCE_DIR}" else - if [[ ! -d "${VIM_SOURCE_DIR}/.git" ]]; then - echo "✘ ${VIM_SOURCE_DIR} does not appear to be a git repository." + cd "${VIM_SOURCE_DIR}" + if ! [ -d ".git" ] \ + && ! [ "$(git rev-parse --show-toplevel)" = "${VIM_SOURCE_DIR}" ]; then + msg_err "${VIM_SOURCE_DIR} does not appear to be a git repository." echo " Please remove it and try again." exit 1 fi - cd "${VIM_SOURCE_DIR}" - echo "Updating Vim sources in '${VIM_SOURCE_DIR}'." - git pull && - echo "✔ Updated Vim sources." || - echo "✘ Could not update Vim sources; ignoring error." + echo "Updating Vim sources: ${VIM_SOURCE_DIR}" + git pull --ff && + msg_ok "Updated Vim sources." || + msg_err "Could not update Vim sources; ignoring error." fi } commit_message() { - printf 'vim-patch:%s\n\n%s\n\n%s' "${vim_version}" \ - "${vim_message}" "${vim_commit_url}" + if [[ -n "$vim_tag" ]]; then + printf '%s\n%s' "${vim_message}" "${vim_commit_url}" + else + printf 'vim-patch:%s\n\n%s\n%s' "$vim_version" "$vim_message" "$vim_commit_url" + fi } find_git_remote() { @@ -102,100 +119,162 @@ assign_commit_details() { vim_tag="v${1}" vim_commit=$(cd "${VIM_SOURCE_DIR}" \ && git log -1 --format="%H" "${vim_tag}") - local strip_commit_line=true + local munge_commit_line=true else # Interpret parameter as commit hash. - vim_version="${1:0:7}" + vim_version="${1:0:12}" + vim_tag= vim_commit=$(cd "${VIM_SOURCE_DIR}" \ && git log -1 --format="%H" "${vim_version}") - local strip_commit_line=false + local munge_commit_line=false fi vim_commit_url="https://github.com/vim/vim/commit/${vim_commit}" vim_message="$(cd "${VIM_SOURCE_DIR}" \ && git log -1 --pretty='format:%B' "${vim_commit}" \ - | sed -e 's/\(#[0-9]*\)/vim\/vim\1/g')" - if [[ ${strip_commit_line} == "true" ]]; then + | sed -e 's/\(#[0-9]\{1,\}\)/vim\/vim\1/g')" + if [[ ${munge_commit_line} == "true" ]]; then # Remove first line of commit message. - vim_message="$(echo "${vim_message}" | sed -e '1d')" + vim_message="$(echo "${vim_message}" | sed -e '1s/^patch /vim-patch:/')" fi patch_file="vim-${vim_version}.patch" } -get_vim_patch() { +# Patch surgery +preprocess_patch() { + local file="$1" + local nvim="nvim -u NORC -i NONE --headless" + + # Remove *.proto, Make*, gui_*, some if_* + local na_src='proto\|Make*\|gui_*\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%(testdir/\)\@<!\%('${na_src}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove unwanted Vim doc files. + local na_doc='channel\.txt\|netbeans\.txt\|os_\w\+\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|sponsor\.txt\|intro\.txt\|tags' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('${na_doc}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove "Last change ..." changes in doc files. + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'%s/^@@.*\n.*For Vim version.*Last change.*\n.*For Vim version.*Last change.*//' +w +q "$file" + + # Remove screen dumps, testdir/Make_*.mak files + local na_src_testdir='Make_amiga.mak\|Make_dos.mak\|Make_ming.mak\|Make_vms.mms\|dumps/.*.dump' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<\%('${na_src_testdir}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove version.c #7555 + local na_po='version.c' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\<\%('${na_po}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove some *.po files. #5622 + local na_po='sjiscorr.c\|ja.sjis.po\|ko.po\|pl.cp1250.po\|pl.po\|ru.cp1251.po\|uk.cp1251.po\|zh_CN.cp936.po\|zh_CN.po\|zh_TW.po' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/po/\<\%('${na_po}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove vimrc_example.vim + local na_vimrcexample='vimrc_example\.vim' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('${na_vimrcexample}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Rename src/ paths to src/nvim/ + LC_ALL=C sed -e 's/\( [ab]\/src\)/\1\/nvim/g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + + # Rename path to matchit plugin. + LC_ALL=C sed -e 's@\( [ab]/runtime\)/pack/dist/opt/matchit/\(plugin/matchit.vim\)@\1/\2@g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + LC_ALL=C sed -e 's@\( [ab]/runtime\)/pack/dist/opt/matchit/doc/\(matchit.txt\)@\1/doc/pi_\2@g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + + # Rename test_urls.vim to check_urls.vim + LC_ALL=C sed -e 's@\( [ab]\)/runtime/doc/test\(_urls.vim\)@\1/scripts/check\2@g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + + # Rename path to check_colors.vim + LC_ALL=C sed -e 's@\( [ab]/runtime\)/colors/\(tools/check_colors.vim\)@\1/\2@g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" +} + +get_vimpatch() { get_vim_sources assign_commit_details "${1}" git log -1 "${vim_commit}" -- >/dev/null 2>&1 || { - >&2 echo "✘ Couldn't find Vim revision '${vim_commit}'." + >&2 msg_err "Couldn't find Vim revision '${vim_commit}'." exit 3 } - echo - echo "✔ Found Vim revision '${vim_commit}'." + msg_ok "Found Vim revision '${vim_commit}'." + + local patch_content + patch_content="$(git --no-pager show --color=never -1 --pretty=medium "${vim_commit}")" + + cd "${NVIM_SOURCE_DIR}" + + printf "Creating patch...\n" + echo "$patch_content" > "${NVIM_SOURCE_DIR}/${patch_file}" - # Patch surgery: preprocess the patch. - # - transform src/ paths to src/nvim/ - local vim_full - vim_full="$(git show -1 --pretty=medium "${vim_commit}" \ - | LC_ALL=C sed -e 's/\( [ab]\/src\)/\1\/nvim/g')" - local neovim_branch="${BRANCH_PREFIX}${vim_version}" + printf "Pre-processing patch...\n" + preprocess_patch "${NVIM_SOURCE_DIR}/${patch_file}" + + msg_ok "Saved patch to '${NVIM_SOURCE_DIR}/${patch_file}'." +} + +stage_patch() { + get_vimpatch "$1" + local try_apply="${2:-}" - cd "${NEOVIM_SOURCE_DIR}" local git_remote git_remote="$(find_git_remote)" local checked_out_branch checked_out_branch="$(git rev-parse --abbrev-ref HEAD)" if [[ "${checked_out_branch}" == ${BRANCH_PREFIX}* ]]; then - echo "✔ Current branch '${checked_out_branch}' seems to be a vim-patch" + msg_ok "Current branch '${checked_out_branch}' seems to be a vim-patch" echo " branch; not creating a new branch." else - echo - echo "Fetching '${git_remote}/master'." + printf '\nFetching "%s/master".\n' "${git_remote}" output="$(git fetch "${git_remote}" master 2>&1)" && - echo "✔ ${output}" || - (echo "✘ ${output}"; false) + msg_ok "${output}" || + (msg_err "${output}"; false) + local nvim_branch="${BRANCH_PREFIX}${vim_version}" echo - echo "Creating new branch '${neovim_branch}' based on '${git_remote}/master'." - cd "${NEOVIM_SOURCE_DIR}" - output="$(git checkout -b "${neovim_branch}" "${git_remote}/master" 2>&1)" && - echo "✔ ${output}" || - (echo "✘ ${output}"; false) + echo "Creating new branch '${nvim_branch}' based on '${git_remote}/master'." + cd "${NVIM_SOURCE_DIR}" + output="$(git checkout -b "${nvim_branch}" "${git_remote}/master" 2>&1)" && + msg_ok "${output}" || + (msg_err "${output}"; false) fi - echo - echo "Creating empty commit with correct commit message." + printf "\nCreating empty commit with correct commit message.\n" output="$(commit_message | git commit --allow-empty --file 2>&1 -)" && - echo "✔ ${output}" || - (echo "✘ ${output}"; false) + msg_ok "${output}" || + (msg_err "${output}"; false) - echo - echo "Creating files." - echo "${vim_full}" > "${NEOVIM_SOURCE_DIR}/${patch_file}" - echo "✔ Saved full commit details to '${NEOVIM_SOURCE_DIR}/${patch_file}'." + if test -n "$try_apply" ; then + if ! check_executable patch; then + printf "\n" + msg_err "'patch' command not found\n" + else + printf "\nApplying patch...\n" + patch -p1 < "${patch_file}" || true + find . -name '*.orig' -type f -delete + fi + printf "\nInstructions:\n Proceed to port the patch.\n" + else + printf '\nInstructions:\n Proceed to port the patch.\n Try the "patch" command (or use "%s -P ..." next time):\n patch -p1 < %s\n' "${BASENAME}" "${patch_file}" + fi - echo - echo "Instructions:" - echo - echo " Proceed to port the patch." - echo " You might want to try 'patch -p1 < ${patch_file}' first." - echo - echo " If the patch contains a new test, consider porting it to Lua." - echo " You might want to try 'scripts/legacy2luatest.pl'." - echo - echo " Stage your changes ('git add ...') and use 'git commit --amend' to commit." - echo - echo " To port additional patches related to ${vim_version} and add them to the current" - echo " branch, call '${BASENAME} -p' again. Please use this only if it wouldn't make" - echo " sense to send in each patch individually, as it will increase the size of the" - echo " pull request and make it harder to review." - echo - echo " When you are finished, use '${BASENAME} -s' to submit a pull request." - echo - echo " See https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim" - echo " for more information." + printf ' + Stage your changes ("git add ..."), then use "git commit --amend" to commit. + + To port more patches (if any) related to %s, + run "%s" again. + * Do this only for _related_ patches (otherwise it increases the + size of the pull request, making it harder to review) + + When you are done, try "%s -s" to create the pull request. + + See the wiki for more information: + * https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim +' "${vim_version}" "${BASENAME}" "${BASENAME}" } hub_pr() { @@ -221,20 +300,21 @@ submit_pr() { exit 1 fi - cd "${NEOVIM_SOURCE_DIR}" + cd "${NVIM_SOURCE_DIR}" local checked_out_branch checked_out_branch="$(git rev-parse --abbrev-ref HEAD)" if [[ "${checked_out_branch}" != ${BRANCH_PREFIX}* ]]; then - echo "✘ Current branch '${checked_out_branch}' doesn't seem to be a vim-patch branch." + msg_err "Current branch '${checked_out_branch}' doesn't seem to be a vim-patch branch." exit 1 fi local git_remote git_remote="$(find_git_remote)" local pr_body - pr_body="$(git log --reverse --format='#### %s%n%n%b%n' "${git_remote}"/master..HEAD)" + pr_body="$(git log --grep=vim-patch --reverse --format='#### %s%n%n%b%n' "${git_remote}"/master..HEAD)" local patches - patches=("$(git log --reverse --format='%s' "${git_remote}"/master..HEAD)") + # Extract just the "vim-patch:X.Y.ZZZZ" or "vim-patch:sha" portion of each log + patches=("$(git log --grep=vim-patch --reverse --format='%s' "${git_remote}"/master..HEAD | sed 's/: .*//')") patches=(${patches[@]//vim-patch:}) # Remove 'vim-patch:' prefix for each item in array. local pr_title="${patches[*]}" # Create space-separated string from array. pr_title="${pr_title// /,}" # Replace spaces with commas. @@ -245,133 +325,173 @@ submit_pr() { if [[ $push_first -ne 0 ]]; then echo "Pushing to 'origin/${checked_out_branch}'." output="$(git push origin "${checked_out_branch}" 2>&1)" && - echo "✔ ${output}" || - (echo "✘ ${output}"; git reset --soft HEAD^1; false) + msg_ok "${output}" || + (msg_err "${output}"; false) echo fi echo "Creating pull request." output="$(${submit_fn} "${pr_message}" 2>&1)" && - echo "✔ ${output}" || - (echo "✘ ${output}"; false) + msg_ok "${output}" || + (msg_err "${output}"; false) echo echo "Cleaning up files." local patch_file for patch_file in "${patches[@]}"; do patch_file="vim-${patch_file}.patch" - if [[ ! -f "${NEOVIM_SOURCE_DIR}/${patch_file}" ]]; then + if [[ ! -f "${NVIM_SOURCE_DIR}/${patch_file}" ]]; then continue fi - rm -- "${NEOVIM_SOURCE_DIR}/${patch_file}" - echo "✔ Removed '${NEOVIM_SOURCE_DIR}/${patch_file}'." + rm -- "${NVIM_SOURCE_DIR}/${patch_file}" + msg_ok "Removed '${NVIM_SOURCE_DIR}/${patch_file}'." done } -list_vim_patches() { - get_vim_sources +# Gets all Vim commits since the "start" commit. +list_vim_commits() { ( + cd "${VIM_SOURCE_DIR}" && git log --reverse --format='%H' v8.0.0000..HEAD +) } + +# Prints all (sorted) "vim-patch:xxx" tokens found in the Nvim git log. +list_vimpatch_tokens() { + local tokens + # Find all "vim-patch:xxx" tokens in the Nvim git log. + tokens="$(cd "${NVIM_SOURCE_DIR}" && git log -E --grep='vim-patch:[^ ]+' | grep 'vim-patch')" + echo "$tokens" | grep -E 'vim-patch:[^ ,{]{7,}' \ + | sed 's/.*\(vim-patch:[.0-9a-z]\+\).*/\1/' \ + | sort \ + | uniq +} - printf "\nVim patches missing from Neovim:\n" +# Prints all patch-numbers (for the current v:version) for which there is +# a "vim-patch:xxx" token in the Nvim git log. +list_vimpatch_numbers() { + # Transform "vim-patch:X.Y.ZZZZ" to "ZZZZ". + list_vimpatch_tokens | while read vimpatch_token; do + echo "$vimpatch_token" | grep '8\.0\.' | sed 's/.*vim-patch:8\.0\.\([0-9a-z]\+\).*/\1/' + done +} - # Get commits since 7.4.602. - local vim_commits - vim_commits="$(cd "${VIM_SOURCE_DIR}" && git log --reverse --format='%H' v7.4.602..HEAD)" +# Prints a newline-delimited list of Vim commits, for use by scripts. +list_missing_vimpatches() { + local tokens vim_commit vim_commits is_missing vim_tag patch_number - local vim_commit + # Find all "vim-patch:xxx" tokens in the Nvim git log. + tokens="$(list_vimpatch_tokens)" + + # Get missing Vim commits + vim_commits="$(list_vim_commits)" for vim_commit in ${vim_commits}; do - local is_missing - local vim_tag - # This fails for untagged commits (e.g., runtime file updates) so mask the return status - vim_tag="$(cd "${VIM_SOURCE_DIR}" && git describe --tags --exact-match "${vim_commit}" 2>/dev/null)" || true - if [[ -n "${vim_tag}" ]]; then - local patch_number="${vim_tag:5}" # Remove prefix like "v7.4." - # Tagged Vim patch, check version.c: - is_missing="$(sed -n '/static int included_patches/,/}/p' "${NEOVIM_SOURCE_DIR}/src/nvim/version.c" | - grep -x -e "[[:space:]]*//[[:space:]]${patch_number} NA.*" -e "[[:space:]]*${patch_number}," >/dev/null && echo "false" || echo "true")" + # Check for vim-patch:<commit_hash> (usually runtime updates). + is_missing="$(echo "$tokens" | >/dev/null 2>&1 grep "vim\-patch:${vim_commit:0:7}" && echo false || echo true)" + + if ! [ "$is_missing" = "false" ] \ + && vim_tag="$(cd "${VIM_SOURCE_DIR}" && git describe --tags --exact-match "${vim_commit}" 2>/dev/null)" + then + # Vim version number (not commit hash). + # Check for vim-patch:<tag> (not commit hash). + patch_number="${vim_tag:1}" # "v7.4.0001" => "7.4.0001" + is_missing="$(echo "$tokens" | >/dev/null 2>&1 grep "vim\-patch:${patch_number}" && echo false || echo true)" vim_commit="${vim_tag#v}" - if (cd "${VIM_SOURCE_DIR}" && git show --name-only "v${vim_commit}" 2>/dev/null) | grep -q ^runtime; then - vim_commit="${vim_commit} (+runtime)" - fi - else - # Untagged Vim patch (e.g. runtime updates), check the Neovim git log: - is_missing="$(cd "${NEOVIM_SOURCE_DIR}" && - git log -1 --no-merges --grep="vim\-patch:${vim_commit:0:7}" --pretty=format:false)" fi - if [[ ${is_missing} != "false" ]]; then - echo " • ${vim_commit}" + if ! [ "$is_missing" = "false" ]; then + echo "${vim_commit}" fi done +} - echo - echo "Instructions:" - echo - echo " To port one of the above patches to Neovim, execute" - echo " this script with the patch revision as argument and" - echo " follow the instructions." - echo - echo " Examples: '${BASENAME} -p 7.4.487'" - echo " '${BASENAME} -p 1e8ebf870720e7b671f98f22d653009826304c4f'" - echo - echo " NOTE: Please port the _oldest_ patch if you possibly can." - echo " Out-of-order patches increase the possibility of bugs." +# Prints a human-formatted list of Vim commits, with instructional messages. +show_vimpatches() { + get_vim_sources + printf "\nVim patches missing from Neovim:\n" + + list_missing_vimpatches | while read vim_commit; do + if (cd "${VIM_SOURCE_DIR}" && git --no-pager show --color=never --name-only "v${vim_commit}" 2>/dev/null) | grep -q ^runtime; then + printf ' • %s (+runtime)\n' "${vim_commit}" + else + printf ' • %s\n' "${vim_commit}" + fi + done + + printf "Instructions: + + To port one of the above patches to Neovim, execute + this script with the patch revision as argument and + follow the instructions. + + Examples: '%s -p 7.4.487' + '%s -p 1e8ebf870720e7b671f98f22d653009826304c4f' + + NOTE: Please port the _oldest_ patch if you possibly can. + Out-of-order patches increase the possibility of bugs. +" "${BASENAME}" "${BASENAME}" } review_commit() { - local neovim_commit_url="${1}" - local neovim_patch_url="${neovim_commit_url}.patch" + local nvim_commit_url="${1}" + local nvim_patch_url="${nvim_commit_url}.patch" local git_patch_prefix='Subject: \[PATCH\] ' - local neovim_patch - neovim_patch="$(curl -Ssf "${neovim_patch_url}")" + local nvim_patch + nvim_patch="$(curl -Ssf "${nvim_patch_url}")" local vim_version - vim_version="$(head -n 4 <<< "${neovim_patch}" | sed -n "s/${git_patch_prefix}vim-patch:\([a-z0-9.]*\)$/\1/p")" + vim_version="$(head -n 4 <<< "${nvim_patch}" | sed -n 's/'"${git_patch_prefix}"'vim-patch:\([a-z0-9.]*\)\(:.*\)\{0,1\}$/\1/p')" echo if [[ -n "${vim_version}" ]]; then - echo "✔ Detected Vim patch '${vim_version}'." + msg_ok "Detected Vim patch '${vim_version}'." else - echo "✘ Could not detect the Vim patch number." + msg_err "Could not detect the Vim patch number." echo " This script assumes that the PR contains only commits" echo " with 'vim-patch:XXX' in their title." + echo + printf -- '%s\n\n' "$(head -n 4 <<< "${nvim_patch}")" + local reply + read -p "Continue reviewing (y/N)? " -n 1 -r reply + if [[ "${reply}" == y ]]; then + echo + return + fi exit 1 fi assign_commit_details "${vim_version}" - local vim_patch_url="${vim_commit_url}.patch" + echo + echo "Creating files." + echo "${nvim_patch}" > "${NVIM_SOURCE_DIR}/n${patch_file}" + msg_ok "Saved pull request diff to '${NVIM_SOURCE_DIR}/n${patch_file}'." + CREATED_FILES+=("${NVIM_SOURCE_DIR}/n${patch_file}") + + local nvim="nvim -u NORC -n -i NONE --headless" + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'1,/^$/g/^ /-1join' +w +q "${NVIM_SOURCE_DIR}/n${patch_file}" local expected_commit_message expected_commit_message="$(commit_message)" local message_length message_length="$(wc -l <<< "${expected_commit_message}")" local commit_message - commit_message="$(tail -n +4 <<< "${neovim_patch}" | head -n "${message_length}")" + commit_message="$(tail -n +4 "${NVIM_SOURCE_DIR}/n${patch_file}" | head -n "${message_length}")" if [[ "${commit_message#${git_patch_prefix}}" == "${expected_commit_message}" ]]; then - echo "✔ Found expected commit message." + msg_ok "Found expected commit message." else - echo "✘ Wrong commit message." + msg_err "Wrong commit message." echo " Expected:" echo "${expected_commit_message}" echo " Actual:" echo "${commit_message#${git_patch_prefix}}" fi - echo - echo "Creating files." - echo "${neovim_patch}" > "${NEOVIM_SOURCE_DIR}/n${patch_file}" - echo "✔ Saved pull request diff to '${NEOVIM_SOURCE_DIR}/n${patch_file}'." - CREATED_FILES+=("${NEOVIM_SOURCE_DIR}/n${patch_file}") - - curl -Ssfo "${NEOVIM_SOURCE_DIR}/${patch_file}" "${vim_patch_url}" - echo "✔ Saved Vim diff to '${NEOVIM_SOURCE_DIR}/${patch_file}'." - CREATED_FILES+=("${NEOVIM_SOURCE_DIR}/${patch_file}") + get_vimpatch "${vim_version}" + CREATED_FILES+=("${NVIM_SOURCE_DIR}/${patch_file}") echo echo "Launching nvim." - nvim -c "cd ${NEOVIM_SOURCE_DIR}" \ - -O "${NEOVIM_SOURCE_DIR}/${patch_file}" "${NEOVIM_SOURCE_DIR}/n${patch_file}" + nvim -c "cd ${NVIM_SOURCE_DIR}" \ + -O "${NVIM_SOURCE_DIR}/${patch_file}" "${NVIM_SOURCE_DIR}/n${patch_file}" } review_pr() { @@ -397,7 +517,7 @@ review_pr() { if [[ "${pr_commit_url}" != "${pr_commit_urls[-1]}" ]]; then read -p "Continue with next commit (Y/n)? " -n 1 -r reply echo - if [[ ! "${reply}" =~ ^[Yy]$ ]]; then + if [[ "${reply}" == n ]]; then break fi fi @@ -406,18 +526,34 @@ review_pr() { clean_files } -while getopts "hlp:r:s" opt; do +while getopts "hlLMVp:P:g:r:s" opt; do case ${opt} in h) usage exit 0 ;; l) - list_vim_patches + show_vimpatches + exit 0 + ;; + L) + list_missing_vimpatches + exit 0 + ;; + M) + list_vimpatch_numbers exit 0 ;; p) - get_vim_patch "${OPTARG}" + stage_patch "${OPTARG}" + exit 0 + ;; + P) + stage_patch "${OPTARG}" TRY_APPLY + exit 0 + ;; + g) + get_vimpatch "${OPTARG}" exit 0 ;; r) @@ -428,6 +564,10 @@ while getopts "hlp:r:s" opt; do submit_pr exit 0 ;; + V) + get_vim_sources + exit 0 + ;; *) exit 1 ;; diff --git a/scripts/vimpatch.lua b/scripts/vimpatch.lua new file mode 100755 index 0000000000..0924f3d718 --- /dev/null +++ b/scripts/vimpatch.lua @@ -0,0 +1,67 @@ +-- Updates version.c list of applied Vim patches. +-- +-- Usage: +-- VIM_SOURCE_DIR=~/neovim/.vim-src/ nvim -i NONE -u NONE --headless +'luafile ./scripts/vimpatch.lua' +q + +local nvim = vim.api + +local function pprint(o) + print(nvim.nvim_call_function('string', { o })) +end + +local function systemlist(...) + local rv = nvim.nvim_call_function('systemlist', ...) + local err = nvim.nvim_get_vvar('shell_error') + local args_str = nvim.nvim_call_function('string', ...) + if 0 ~= err then + error('command failed: '..args_str) + end + return rv +end + +local function vimpatch_sh_list_numbers() + return systemlist( { { 'bash', '-c', 'scripts/vim-patch.sh -M', } } ) +end + +-- Generates the lines to be inserted into the src/version.c +-- `included_patches[]` definition. +local function gen_version_c_lines() + -- Set of merged Vim 8.0.zzzz patch numbers. + local merged_patch_numbers = {} + local highest = 0 + for _, n in ipairs(vimpatch_sh_list_numbers()) do + if n then + merged_patch_numbers[tonumber(n)] = true + highest = math.max(highest, n) + end + end + + local lines = {} + for i = highest, 0, -1 do + local is_merged = (nil ~= merged_patch_numbers[i]) + if is_merged then + table.insert(lines, string.format(' %s,', i)) + else + table.insert(lines, string.format(' // %s,', i)) + end + end + + return lines +end + +local function patch_version_c() + local lines = gen_version_c_lines() + + nvim.nvim_command('silent noswapfile noautocmd edit src/nvim/version.c') + nvim.nvim_command('/static const int included_patches') + -- Delete the existing lines. + nvim.nvim_command('silent normal! j0d/};\rk') + -- Insert the lines. + nvim.nvim_call_function('append', { + nvim.nvim_eval('line(".")'), + lines, + }) + nvim.nvim_command('silent write') +end + +patch_version_c() diff --git a/scripts/windows.ti b/scripts/windows.ti new file mode 100644 index 0000000000..4f4832b786 --- /dev/null +++ b/scripts/windows.ti @@ -0,0 +1,71 @@ +libuv+basekey|vt100 base function key for libuv, + kb2=\E[G, kcub1=\E[D, kcud1=\E[B, kcuf1=\E[C, kcuu1=\E[A, + kdch1=\E[3~, kend=\E[4~, kf1=\E[[A, kf10=\E[21~, + kf11=\E[23~, kf12=\E[24~, kf13=\E[25~, kf14=\E[26~, + kf15=\E[28~, kf16=\E[29~, kf17=\E[31~, kf18=\E[32~, + kf19=\E[33~, kf2=\E[[B, kf20=\E[34~, kf3=\E[[C, kf4=\E[[D, + kf5=\E[[E, kf6=\E[17~, kf7=\E[18~, kf8=\E[19~, kf9=\E[20~, + khome=\E[1~, kich1=\E[2~, knp=\E[6~, kpp=\E[5~, + +libuv+exkey|vt100 extend function key for libuv, + kf21=\E[23$, kf22=\E[24$, kf23=\E[11\^, kf24=\E[12\^, + kf25=\E[13\^, kf26=\E[14\^, kf27=\E[15\^, kf28=\E[17\^, + kf29=\E[18\^, kf30=\E[19\^, kf31=\E[20\^, kf32=\E[21\^, + kf33=\E[23\^, kf34=\E[24\^, kf35=\E[25\^, kf36=\E[26\^, + kf41=\E[32\^, kf42=\E[33\^, kf43=\E[34\^, kf44=\E[23@, + kf37=\E[28\^, kf38=\E[29\^, kf39=\E[31\^, kf45=\E[24@, + kDC=\E[3;2~, kEND=\E[4;2~, kHOM=\E[1;2~, kIC=\E[2;2~, + kLFT=\E[1;2D, kNXT=\E[6;2~, kPRV=\E[5;2~, kRIT=\E[1;2C, + +win32con|ANSI emulation for libuv on legacy console, + hs@, fsl@, tsl@, xon@, acsc@, rmacs@, smacs@, dch@, dch1@, dl@, dl1@, + ich@, ich1@, il@, il1@, ind@, invis@, ri@, rmir@, rs1@, rmul@, smir@, + smul@, smpch@, u6@, u7@, u8@, u9@, + sgr=\E[0%?%p1%t;7%;%?%p3%t;7%;%?%p6%t;1%;m, sgr0=\E[0m, + Se=\E[0 q, Ss=\E[%p1%d q, + use=cygwin, use=libuv+exkey, + +conemu|ANIS X3.64 and Xterm 256 colors for ConEmu with libuv, + ccc@, mc5i@, xenl@, acsc@, rmacs@, smacs@, blink@, cbt@, + cvvis@, cnorm=\E[?25h, cud1=\E[B, dim@, flash@, hts@, initc@, invis@, is2@, + kf46@, kf47@, kf48@, kf49@, kf50@, kf51@, kf52@, kf53@, kf54@, + kf55@, kf56@, kf57@, kf58@, kf59@, kf60@, kf61@, kf62@, kf63@, kmous@, + mc0@, mc4@, mc5@, meml@, memu@, oc@, rmam@, rmcup=\E[?1049l, + smcup=\E[?1049h, rmir@, rmkx@, rmm@, rs1@, rs2@, + setab=\E[48;5;%p1%dm, setaf=\E[38;5;%p1%dm, + sgr=\E[0%?%p1%p3%|%t;7%;%?%p2%t;4%;%?%p6%t;1%;m, + sgr0=\E[0m, smam@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, + Cr@, Cs@, Ms@, XM@, kDC3@, kDC4@, kDC5@, kDC6@, kDC7@, + kDN@, kDN3@, kDN4@, kDN5@, kDN6@, kDN7@, + kEND3@, kEND4@, kEND5@, kEND6@, kEND7@, + kHOM3@, kHOM4@, kHOM5@, kHOM6@, kHOM7@, + kIC3@, kIC4@, kIC5@, kIC6@, kIC7@, + kLFT3@, kLFT4@, kLFT5@, kLFT6@, kLFT7@, + kNXT3@, kNXT4@, kNXT5@, kNXT6@, kNXT7@, + kPRV3@, kPRV4@, kPRV5@, kPRV6@, kPRV7@, + kRIT3@, kRIT4@, kRIT5@, kRIT6@, kRIT7@, + kUP3@, kUP4@, kUP5@, kUP6@, kUP7@, rmxx@, smxx@, xm@, + use=libuv+basekey, use=libuv+exkey, use=xterm+256color, use=xterm-new, + +vtpcon|ANIS emulation for console virtual terminal sequence with libuv, + ccc@, mc5i@, xenl@, blink@, acsc=jjkkllmmnnqqttuuvvwwxx, + cvvis@, cud1=\E[B, dim@, flash@, + initc=\E]4;%p1%d;rgb\:%p2%{255}%*%{1000}%/%2.2X/%p3%{255}%*%{1000}%/%2.2X/%p4%{255}%*%{1000}%/%2.2X\E, + invis@, is2=\E[!p\E[?3l, + kf46@, kf47@, kf48@, kf49@, kf50@, kf51@, kf52@, kf53@, kf54@, + kf55@, kf56@, kf57@, kf58@, kf59@, kf60@, kf61@, kf62@, kf63@, kmous@, + mc0@, mc4@, mc5@, meml@, memu@, oc@, rmam@, rmcup=\E[?1049l, + smcup=\E[?1049h, rmir@, rmkx@, rmm@, rs1@, rs2@, + sgr=\E[0%?%p1%p3%|%t;7%;%?%p2%t;4%;%?%p6%t;1%;m, + sgr0=\E[0m, smam@, smir@, smkx@, smm@, tbc@, u6@, u7@, u8@, u9@, + Cr@, Cs@, Ms@, XM@, kDC3@, kDC4@, kDC5@, kDC6@, kDC7@, + kDN@, kDN3@, kDN4@, kDN5@, kDN6@, kDN7@, + kEND3@, kEND4@, kEND5@, kEND6@, kEND7@, + kHOM3@, kHOM4@, kHOM5@, kHOM6@, kHOM7@, + kIC3@, kIC4@, kIC5@, kIC6@, kIC7@, + kLFT3@, kLFT4@, kLFT5@, kLFT6@, kLFT7@, + kNXT3@, kNXT4@, kNXT5@, kNXT6@, kNXT7@, + kPRV3@, kPRV4@, kPRV5@, kPRV6@, kPRV7@, + kRIT3@, kRIT4@, kRIT5@, kRIT6@, kRIT7@, + kUP3@, kUP4@, kUP5@, kUP6@, kUP7@, rmxx@, smxx@, xm@, + use=libuv+basekey, use=libuv+exkey, use=xterm+256color, use=xterm-new, |