diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/download-unicode-files.sh | 18 | ||||
| -rw-r--r-- | scripts/gen_api_vimdoc.py | 514 | ||||
| -rwxr-xr-x | scripts/gendeclarations.lua | 2 | ||||
| -rw-r--r-- | scripts/gendispatch.lua | 329 | ||||
| -rw-r--r-- | scripts/geneval.lua | 66 | ||||
| -rw-r--r-- | scripts/genoptions.lua | 8 | ||||
| -rw-r--r-- | scripts/genunicodetables.lua | 106 | ||||
| -rw-r--r-- | scripts/genvimvim.lua | 29 | ||||
| -rwxr-xr-x | scripts/git-log-pretty-since.sh | 2 | ||||
| -rw-r--r-- | scripts/msgpack-gen.lua | 310 | ||||
| -rwxr-xr-x | scripts/release.sh | 29 | ||||
| -rwxr-xr-x | scripts/vim-patch.sh | 181 |
12 files changed, 1168 insertions, 426 deletions
diff --git a/scripts/download-unicode-files.sh b/scripts/download-unicode-files.sh index cb15270cf8..54fc32550c 100755 --- a/scripts/download-unicode-files.sh +++ b/scripts/download-unicode-files.sh @@ -1,11 +1,11 @@ #!/bin/sh set -e - -files="UnicodeData.txt CaseFolding.txt EastAsianWidth.txt" +data_files="UnicodeData.txt CaseFolding.txt EastAsianWidth.txt" +emoji_files="emoji-data.txt" UNIDIR_DEFAULT=unicode -DOWNLOAD_URL_BASE_DEFAULT='http://unicode.org/Public/UNIDATA' +DOWNLOAD_URL_BASE_DEFAULT='http://unicode.org/Public' if test x$1 = 'x--help' ; then echo 'Usage:' @@ -21,8 +21,16 @@ fi UNIDIR=${1:-$UNIDIR_DEFAULT} DOWNLOAD_URL_BASE=${2:-$DOWNLOAD_URL_BASE_DEFAULT} -for filename in $files ; do - curl -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/$filename" +for filename in $data_files ; do + curl -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/$filename" + ( + cd "$UNIDIR" + git add $filename + ) +done + +for filename in $emoji_files ; do + curl -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/emoji/3.0/$filename" ( cd "$UNIDIR" git add $filename diff --git a/scripts/gen_api_vimdoc.py b/scripts/gen_api_vimdoc.py new file mode 100644 index 0000000000..d7165187f4 --- /dev/null +++ b/scripts/gen_api_vimdoc.py @@ -0,0 +1,514 @@ +#!/usr/bin/env python +"""Parses Doxygen XML output to generate Neovim's API documentation. + +This would be easier using lxml and XSLT, but: + + 1. This should avoid needing Python dependencies, especially ones that are + C modules that have library dependencies (lxml requires libxml and + libxslt). + 2. I wouldn't know how to deal with nested indentation in <para> tags using + XSLT. + +Each function documentation is formatted with the following rules: + + - Maximum width of 78 characters (`text_width`). + - Spaces for indentation. + - Function signature and helptag are on the same line. + - Helptag is right aligned. + - Signature and helptag must have a minimum of 8 spaces between them. + - If the signature is too long, it is placed on the line after the + helptag. The signature wraps at `text_width - 8` characters with + subsequent lines indented to the open parenthesis. + - Documentation body will be indented by 16 spaces. + - Subsection bodies are indented an additional 4 spaces. + - Documentation body consists of the function description, parameter details, + return description, and C declaration. + - Parameters are omitted for the `void` and `Error *` types, or if the + parameter is marked as [out]. + - Each function documentation is separated by a single line. + +The C declaration is added to the end to show actual argument types. +""" +import os +import re +import sys +import shutil +import textwrap +import subprocess + +from xml.dom import minidom + +# Text at the top of the doc file. +preamble = ''' +Note: This documentation is generated from Neovim's API source code. +''' + +doc_filename = 'api-funcs.txt' + +# Section name overrides. +section_name = { + 'vim.c': 'Global', +} + +# Section ordering. +section_order = ( + 'vim.c', + 'buffer.c', + 'window.c', + 'tabpage.c', + 'ui.c', +) + +param_exclude = ( + 'channel_id', +) + +text_width = 78 +script_path = os.path.abspath(__file__) +base_dir = os.path.dirname(os.path.dirname(script_path)) +src_dir = os.path.join(base_dir, 'src/nvim/api') +out_dir = os.path.join(base_dir, 'tmp/api_doc') +filter_cmd = '%s %s' % (sys.executable, script_path) +seen_funcs = set() + +# Tracks `xrefsect` titles. As of this writing, used only for separating +# deprecated functions. +xrefs = set() + + +# XML Parsing Utilities {{{ +def find_first(parent, name): + """Finds the first matching node within parent.""" + sub = parent.getElementsByTagName(name) + if not sub: + return None + return sub[0] + + +def get_children(parent, name): + """Yield matching child nodes within parent.""" + for child in parent.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.nodeName == name: + yield child + + +def get_child(parent, name): + """Get the first matching child node.""" + for child in get_children(parent, name): + return child + return None + + +def clean_text(text): + """Cleans text. + + Only cleans superfluous whitespace at the moment. + """ + return ' '.join(text.split()).strip() + + +def clean_lines(text): + """Removes superfluous lines. + + The beginning and end of the string is trimmed. Empty lines are collapsed. + """ + return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text)) + + +def get_text(parent): + """Combine all text in a node.""" + if parent.nodeType == parent.TEXT_NODE: + return parent.data + + out = '' + for node in parent.childNodes: + if node.nodeType == node.TEXT_NODE: + out += clean_text(node.data) + elif node.nodeType == node.ELEMENT_NODE: + out += ' ' + get_text(node) + return out + + +def doc_wrap(text, prefix='', width=70, func=False): + """Wraps text to `width`. + + The first line is prefixed with `prefix`, and subsequent lines are aligned. + If `func` is True, only wrap at commas. + """ + if not width: + return text + + indent_space = ' ' * len(prefix) + + if func: + lines = [prefix] + for part in text.split(', '): + if part[-1] not in ');': + part += ', ' + if len(lines[-1]) + len(part) > width: + lines.append(indent_space) + lines[-1] += part + return '\n'.join(x.rstrip() for x in lines).rstrip() + + return '\n'.join(textwrap.wrap(text.strip(), width=width, + initial_indent=prefix, + subsequent_indent=indent_space)) + + +def parse_params(parent, width=62): + """Parse Doxygen `parameterlist`.""" + name_length = 0 + items = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + continue + + name_node = find_first(child, 'parametername') + if name_node.getAttribute('direction') == 'out': + continue + + name = get_text(name_node) + if name in param_exclude: + continue + + name = '{%s}' % name + name_length = max(name_length, len(name) + 2) + + desc = '' + desc_node = get_child(child, 'parameterdescription') + if desc_node: + desc = parse_parblock(desc_node, width=None) + items.append((name.strip(), desc.strip())) + + out = 'Parameters:~\n' + for name, desc in items: + name = ' %s' % name.ljust(name_length) + out += doc_wrap(desc, prefix=name, width=width) + '\n' + return out.strip() + + +def parse_para(parent, width=62): + """Parse doxygen `para` tag. + + I assume <para> is a paragraph block or "a block of text". It can contain + text nodes, or other tags. + """ + line = '' + lines = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + line += child.data + elif child.nodeName == 'computeroutput': + line += '`%s`' % get_text(child) + else: + if line: + lines.append(doc_wrap(line, width=width)) + line = '' + + if child.nodeName == 'parameterlist': + lines.append(parse_params(child, width=width)) + elif child.nodeName == 'xrefsect': + title = get_text(get_child(child, 'xreftitle')) + xrefs.add(title) + xrefdesc = parse_para(get_child(child, 'xrefdescription')) + lines.append(doc_wrap(xrefdesc, prefix='%s: ' % title, + width=width) + '\n') + elif child.nodeName == 'simplesect': + kind = child.getAttribute('kind') + if kind == 'return': + lines.append('%s:~' % kind.title()) + lines.append(doc_wrap(parse_para(child), + prefix=' ', + width=width)) + else: + lines.append(get_text(child)) + + if line: + lines.append(doc_wrap(line, width=width)) + return clean_lines('\n'.join(lines).strip()) + + +def parse_parblock(parent, width=62): + """Parses a nested block of `para` tags. + + Named after the \parblock command, but not directly related. + """ + paragraphs = [] + for child in parent.childNodes: + if child.nodeType == child.TEXT_NODE: + paragraphs.append(doc_wrap(child.data, width=width)) + elif child.nodeName == 'para': + paragraphs.append(parse_para(child, width=width)) + else: + paragraphs.append(doc_wrap(get_text(child), width=width)) + paragraphs.append('') + return clean_lines('\n'.join(paragraphs).strip()) +# }}} + + +def parse_source_xml(filename): + """Collects API functions. + + This returns two strings: + 1. The API functions + 2. The deprecated API functions + + The caller decides what to do with the deprecated documentation. + """ + global xrefs + xrefs = set() + functions = [] + deprecated_functions = [] + + dom = minidom.parse(filename) + for member in dom.getElementsByTagName('memberdef'): + if member.getAttribute('static') == 'yes' or \ + member.getAttribute('kind') != 'function': + continue + + loc = find_first(member, 'location') + if 'private' in loc.getAttribute('file'): + continue + + return_type = get_text(get_child(member, 'type')) + if return_type == '': + continue + + if return_type.startswith(('ArrayOf', 'DictionaryOf')): + parts = return_type.strip('_').split('_') + return_type = '%s(%s)' % (parts[0], ', '.join(parts[1:])) + + name = get_text(get_child(member, 'name')) + + vimtag = '*%s()*' % name + args = [] + type_length = 0 + + for param in get_children(member, 'param'): + arg_type = get_text(get_child(param, 'type')).strip() + arg_name = '' + declname = get_child(param, 'declname') + if declname: + arg_name = get_text(declname).strip() + + if arg_name in param_exclude: + continue + + if arg_type.endswith('*'): + arg_type = arg_type.strip('* ') + arg_name = '*' + arg_name + type_length = max(type_length, len(arg_type)) + args.append((arg_type, arg_name)) + + c_args = [] + for arg_type, arg_name in args: + c_args.append(' ' + ( + '%s %s' % (arg_type.ljust(type_length), arg_name)).strip()) + + c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name, + ',\n'.join(c_args)), + ' ') + + prefix = '%s(' % name + suffix = '%s)' % ', '.join('{%s}' % a[1] for a in args + if a[0] not in ('void', 'Error')) + + # Minimum 8 chars between signature and vimtag + lhs = (text_width - 8) - len(prefix) + + if len(prefix) + len(suffix) > lhs: + signature = vimtag.rjust(text_width) + '\n' + signature += doc_wrap(suffix, width=text_width-8, prefix=prefix, + func=True) + else: + signature = prefix + suffix + signature += vimtag.rjust(text_width - len(signature)) + + doc = '' + desc = find_first(member, 'detaileddescription') + if desc: + doc = parse_parblock(desc) + if 'DEBUG' in os.environ: + print(textwrap.indent( + re.sub(r'\n\s*\n+', '\n', + desc.toprettyxml(indent=' ', newl='\n')), ' ' * 16)) + + if not doc: + doc = 'TODO: Documentation' + + if 'INCLUDE_C_DECL' in os.environ: + doc += '\n\nC Declaration:~\n>\n' + doc += c_decl + doc += '\n<' + + func_doc = signature + '\n' + func_doc += textwrap.indent(clean_lines(doc), ' ' * 16) + func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M) + + if 'Deprecated' in xrefs: + deprecated_functions.append(func_doc) + else: + functions.append(func_doc) + + xrefs.clear() + + return '\n\n'.join(functions), '\n\n'.join(deprecated_functions) + + +def gen_docs(config): + """Generate documentation. + + Doxygen is called and configured through stdin. + """ + p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE) + p.communicate(config.format(input=src_dir, output=out_dir, + filter=filter_cmd).encode('utf8')) + if p.returncode: + sys.exit(p.returncode) + + title_length = 0 + sections = {} + sep = '=' * text_width + + base = os.path.join(out_dir, 'xml') + dom = minidom.parse(os.path.join(base, 'index.xml')) + for compound in dom.getElementsByTagName('compound'): + if compound.getAttribute('kind') != 'file': + continue + + filename = get_text(find_first(compound, 'name')) + if filename.endswith('.c'): + functions, deprecated = parse_source_xml( + os.path.join(base, '%s.xml' % compound.getAttribute('refid'))) + + if not functions and not deprecated: + continue + + if functions or deprecated: + name = os.path.splitext(os.path.basename(filename))[0] + if name == 'ui': + name = name.upper() + else: + name = name.title() + + doc = '' + if functions: + doc += '\n\n' + functions + + if 'INCLUDE_DEPRECATED' in os.environ and deprecated: + doc += '\n\n\nDeprecated %s Functions:~\n\n' % name + doc += deprecated + + if doc: + filename = os.path.basename(filename) + name = section_name.get(filename, name) + title = '%s Functions' % name + helptag = '*api-%s*' % name.lower() + title_length = max(title_length, len(title)) + sections[filename] = (title, helptag, doc) + + if not sections: + return + + title_left = '*%s*' % doc_filename + title_center = 'Neovim API Function Reference' + title_right = '{Nvim}' + margin = max(len(title_left), len(title_right)) + head = (title_left.ljust(margin) + + title_center.center(text_width - margin * 2) + + title_right.rjust(margin)) + '\n' + + head += '\n%s\n\n' % doc_wrap(preamble, width=text_width) + head += 'Contents:\n\n' + + docs = '' + + title_length += len(str(len(section_order))) + 2 + i = 0 + for filename in section_order: + if filename not in sections: + continue + title, helptag, section_doc = sections.pop(filename) + + i += 1 + docs += sep + title = '%d. %s' % (i, title) + head += (title.ljust(title_length) + ' ' + + helptag.replace('*', '|') + '\n') + docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) + docs += section_doc + docs += '\n\n\n' + + if sections: + # In case new API sources are added without updating the order dict. + for title, helptag, section_doc in sections.values(): + i += 1 + docs += sep + title = '%d. %s' % (i, title) + head += (title.ljust(title_length) + ' ' + + helptag.replace('*', '|') + '\n') + docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title))) + docs += section_doc + docs += '\n\n\n' + + docs = '%s\n%s' % (head, docs) + docs = docs.rstrip() + '\n\n' + docs += ' vim:tw=78:ts=8:ft=help:norl:' + + doc_file = os.path.join(base_dir, 'runtime/doc', doc_filename) + with open(doc_file, 'wb') as fp: + fp.write(docs.encode('utf8')) + + shutil.rmtree(out_dir) + + +def filter_source(filename): + """Filters the source to fix macros that confuse Doxygen.""" + with open(filename, 'rt') as fp: + print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))', + lambda m: m.group(1)+'_'.join( + re.split(r'[^\w]+', m.group(2))), + fp.read(), flags=re.M)) + + +# Doxygen Config {{{ +Doxyfile = ''' +OUTPUT_DIRECTORY = {output} +INPUT = {input} +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = *.h *.c +RECURSIVE = YES +INPUT_FILTER = "{filter}" +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = */private/* +EXCLUDE_SYMBOLS = + +GENERATE_HTML = NO +GENERATE_DOCSET = NO +GENERATE_HTMLHELP = NO +GENERATE_QHP = NO +GENERATE_TREEVIEW = NO +GENERATE_LATEX = NO +GENERATE_RTF = NO +GENERATE_MAN = NO +GENERATE_DOCBOOK = NO +GENERATE_AUTOGEN_DEF = NO + +GENERATE_XML = YES +XML_OUTPUT = xml +XML_PROGRAMLISTING = NO + +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = YES +EXPAND_ONLY_PREDEF = NO +''' +# }}} + +if __name__ == "__main__": + if len(sys.argv) > 1: + filter_source(sys.argv[1]) + else: + gen_docs(Doxyfile) + +# vim: set ft=python ts=4 sw=4 tw=79 et fdm=marker : diff --git a/scripts/gendeclarations.lua b/scripts/gendeclarations.lua index 4e74e4e301..ff69b18ae4 100755 --- a/scripts/gendeclarations.lua +++ b/scripts/gendeclarations.lua @@ -143,7 +143,7 @@ local pattern = concat( lit(')'), any_amount(concat( -- optional attributes spaces, - lit('FUNC_ATTR_'), + lit('FUNC_'), any_amount(aw), one_or_no(concat( -- attribute argument spaces, diff --git a/scripts/gendispatch.lua b/scripts/gendispatch.lua new file mode 100644 index 0000000000..397ccc9aaf --- /dev/null +++ b/scripts/gendispatch.lua @@ -0,0 +1,329 @@ +lpeg = require('lpeg') +mpack = require('mpack') + +-- lpeg grammar for building api metadata from a set of header files. It +-- ignores comments and preprocessor commands and parses a very small subset +-- of C prototypes with a limited set of types +P, R, S = lpeg.P, lpeg.R, lpeg.S +C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg + +any = P(1) -- (consume one character) +letter = R('az', 'AZ') + S('_$') +alpha = letter + R('09') +nl = P('\r\n') + P('\n') +not_nl = any - nl +ws = S(' \t') + nl +fill = ws ^ 0 +c_comment = P('//') * (not_nl ^ 0) +c_preproc = P('#') * (not_nl ^ 0) +typed_container = + (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')') +c_id = ( + typed_container + + (letter * (alpha ^ 0)) +) +c_void = P('void') +c_param_type = ( + ((P('Error') * fill * P('*') * fill) * Cc('error')) + + (C(c_id) * (ws ^ 1)) + ) +c_type = (C(c_void) * (ws ^ 1)) + c_param_type +c_param = Ct(c_param_type * C(c_id)) +c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0) +c_params = Ct(c_void + c_param_list) +c_proto = Ct( + Cg(c_type, 'return_type') * Cg(c_id, 'name') * + fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') * + Cg(Cc(false), 'async') * + (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) * + (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) * + (fill * Cg((P('FUNC_API_NOEVAL') * Cc(true)), 'noeval') ^ -1) * + fill * P(';') + ) +grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1) + +-- we need at least 4 arguments since the last two are output files +assert(#arg >= 3) +functions = {} + +local nvimsrcdir = arg[1] +package.path = nvimsrcdir .. '/?.lua;' .. package.path + +-- names of all headers relative to the source root (for inclusion in the +-- generated file) +headers = {} +-- output h file with generated dispatch functions +dispatch_outputf = arg[#arg-2] +-- output h file with packed metadata +funcs_metadata_outputf = arg[#arg-1] +-- output metadata mpack file, for use by other build scripts +mpack_outputf = arg[#arg] + +-- set of function names, used to detect duplicates +function_names = {} + +-- read each input file, parse and append to the api metadata +for i = 2, #arg - 3 do + local full_path = arg[i] + local parts = {} + for part in string.gmatch(full_path, '[^/]+') do + parts[#parts + 1] = part + end + headers[#headers + 1] = parts[#parts - 1]..'/'..parts[#parts] + + local input = io.open(full_path, 'rb') + local tmp = grammar:match(input:read('*all')) + for i = 1, #tmp do + local fn = tmp[i] + if not fn.noexport then + functions[#functions + 1] = tmp[i] + function_names[fn.name] = true + if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then + -- this function should receive the channel id + fn.receives_channel_id = true + -- remove the parameter since it won't be passed by the api client + table.remove(fn.parameters, 1) + end + if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then + -- function can fail if the last parameter type is 'Error' + fn.can_fail = true + -- remove the error parameter, msgpack has it's own special field + -- for specifying errors + fn.parameters[#fn.parameters] = nil + end + end + end + input:close() +end + +local function shallowcopy(orig) + local copy = {} + for orig_key, orig_value in pairs(orig) do + copy[orig_key] = orig_value + end + return copy +end + +local function startswith(String,Start) + return string.sub(String,1,string.len(Start))==Start +end + +-- Export functions under older deprecated names. +-- These will be removed eventually. +local deprecated_aliases = require("api.dispatch_deprecated") +for i,f in ipairs(shallowcopy(functions)) do + local ismethod = false + if startswith(f.name, "nvim_") then + -- TODO(bfredl) after 0.1.6 allow method definitions + -- to specify the since and deprecated_since field + f.since = 1 + if startswith(f.name, "nvim_buf_") then + ismethod = true + elseif startswith(f.name, "nvim_win_") then + ismethod = true + elseif startswith(f.name, "nvim_tabpage_") then + ismethod = true + end + else + f.noeval = true + f.since = 0 + f.deprecated_since = 1 + end + f.method = ismethod + local newname = deprecated_aliases[f.name] + if newname ~= nil then + if function_names[newname] then + -- duplicate + print("Function "..f.name.." has deprecated alias\n" + ..newname.." which has a separate implementation.\n".. + "Please remove it from src/nvim/api/dispatch_deprecated.lua") + os.exit(1) + end + local newf = shallowcopy(f) + newf.name = newname + if newname == "ui_try_resize" then + -- The return type was incorrectly set to Object in 0.1.5. + -- Keep it that way for clients that rely on this. + newf.return_type = "Object" + end + newf.impl_name = f.name + newf.noeval = true + newf.since = 0 + newf.deprecated_since = 1 + functions[#functions+1] = newf + end +end + +-- don't expose internal attributes like "impl_name" in public metadata +exported_attributes = {'name', 'parameters', 'return_type', 'method', + 'since', 'deprecated_since'} +exported_functions = {} +for _,f in ipairs(functions) do + local f_exported = {} + for _,attr in ipairs(exported_attributes) do + f_exported[attr] = f[attr] + end + exported_functions[#exported_functions+1] = f_exported +end + + +funcs_metadata_output = io.open(funcs_metadata_outputf, 'wb') +funcs_metadata_output:write([[ +static const uint8_t funcs_metadata[] = { +]]) + +-- serialize the API metadata using msgpack and embed into the resulting +-- binary for easy querying by clients +packed_exported_functions = mpack.pack(exported_functions) +for i = 1, #packed_exported_functions do + funcs_metadata_output:write(string.byte(packed_exported_functions, i)..', ') + if i % 10 == 0 then + funcs_metadata_output:write('\n ') + end +end +funcs_metadata_output:write([[ +}; +]]) +funcs_metadata_output:close() + +-- start building the dispatch wrapper output +output = io.open(dispatch_outputf, 'wb') + +local function real_type(type) + local rv = type + if typed_container:match(rv) then + if rv:match('Array') then + rv = 'Array' + else + rv = 'Dictionary' + end + end + return rv +end + +-- start the handler functions. Visit each function metadata to build the +-- handler function with code generated for validating arguments and calling to +-- the real API. +for i = 1, #functions do + local fn = functions[i] + if fn.impl_name == nil then + local args = {} + + output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)') + output:write('\n{') + output:write('\n Object ret = NIL;') + -- Declare/initialize variables that will hold converted arguments + for j = 1, #fn.parameters do + local param = fn.parameters[j] + local converted = 'arg_'..j + output:write('\n '..param[1]..' '..converted..';') + end + output:write('\n') + output:write('\n if (args.size != '..#fn.parameters..') {') + output:write('\n snprintf(error->msg, sizeof(error->msg), "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);') + output:write('\n error->set = true;') + output:write('\n goto cleanup;') + output:write('\n }\n') + + -- Validation/conversion for each argument + for j = 1, #fn.parameters do + local converted, convert_arg, param, arg + param = fn.parameters[j] + converted = 'arg_'..j + local rt = real_type(param[1]) + if rt ~= 'Object' then + if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then + -- Buffer, Window, and Tabpage have a specific type, but are stored in integer + output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..' && args.items['..(j - 1)..'].data.integer >= 0) {') + output:write('\n '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;') + else + output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {') + output:write('\n '..converted..' = args.items['..(j - 1)..'].data.'..rt:lower()..';') + end + if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then + -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages + output:write('\n } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer >= 0) {') + output:write('\n '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;') + end + output:write('\n } else {') + output:write('\n snprintf(error->msg, sizeof(error->msg), "Wrong type for argument '..j..', expecting '..param[1]..'");') + output:write('\n error->set = true;') + output:write('\n goto cleanup;') + output:write('\n }\n') + else + output:write('\n '..converted..' = args.items['..(j - 1)..'];\n') + end + + args[#args + 1] = converted + end + + -- function call + local call_args = table.concat(args, ', ') + output:write('\n ') + if fn.return_type ~= 'void' then + -- has a return value, prefix the call with a declaration + output:write(fn.return_type..' rv = ') + end + + -- write the function name and the opening parenthesis + output:write(fn.name..'(') + + if fn.receives_channel_id then + -- if the function receives the channel id, pass it as first argument + if #args > 0 or fn.can_fail then + output:write('channel_id, '..call_args) + else + output:write('channel_id') + end + else + output:write(call_args) + end + + if fn.can_fail then + -- if the function can fail, also pass a pointer to the local error object + if #args > 0 then + output:write(', error);\n') + else + output:write('error);\n') + end + -- and check for the error + output:write('\n if (error->set) {') + output:write('\n goto cleanup;') + output:write('\n }\n') + else + output:write(');\n') + end + + if fn.return_type ~= 'void' then + output:write('\n ret = '..string.upper(real_type(fn.return_type))..'_OBJ(rv);') + end + output:write('\n\ncleanup:'); + + output:write('\n return ret;\n}\n\n'); + end +end + +-- Generate a function that initializes method names with handler functions +output:write([[ +void msgpack_rpc_init_method_table(void) +{ + methods = map_new(String, MsgpackRpcRequestHandler)(); + +]]) + +for i = 1, #functions do + local fn = functions[i] + output:write(' msgpack_rpc_add_method_handler('.. + '(String) {.data = "'..fn.name..'", '.. + '.size = sizeof("'..fn.name..'") - 1}, '.. + '(MsgpackRpcRequestHandler) {.fn = handle_'.. (fn.impl_name or fn.name).. + ', .async = '..tostring(fn.async)..'});\n') + +end + +output:write('\n}\n\n') +output:close() + +mpack_output = io.open(mpack_outputf, 'wb') +mpack_output:write(mpack.pack(functions)) +mpack_output:close() diff --git a/scripts/geneval.lua b/scripts/geneval.lua new file mode 100644 index 0000000000..b1ba76296c --- /dev/null +++ b/scripts/geneval.lua @@ -0,0 +1,66 @@ +mpack = require('mpack') + +local nvimsrcdir = arg[1] +local autodir = arg[2] +local metadata_file = arg[3] +local funcs_file = arg[4] + +if nvimsrcdir == '--help' then + print([[ +Usage: + lua geneval.lua src/nvim build/src/nvim/auto + +Will generate build/src/nvim/auto/funcs.generated.h with definition of functions +static const array. +]]) + os.exit(0) +end + +package.path = nvimsrcdir .. '/?.lua;' .. package.path + +local funcsfname = autodir .. '/funcs.generated.h' + +local gperfpipe = io.open(funcsfname .. '.gperf', 'wb') + +local funcs = require('eval').funcs +local metadata = mpack.unpack(io.open(arg[3], 'rb'):read("*all")) +for i,fun in ipairs(metadata) do + if not fun.noeval then + funcs[fun.name] = { + args=#fun.parameters, + func='api_wrapper', + data='&handle_'..fun.name, + } + end +end + +local funcsdata = io.open(funcs_file, 'w') +funcsdata:write(mpack.pack(funcs)) +funcsdata:close() + +gperfpipe:write([[ +%language=ANSI-C +%global-table +%define initializer-suffix ,0,0,NULL,NULL +%define word-array-name functions +%define hash-function-name hash_internal_func_gperf +%define lookup-function-name find_internal_func_gperf +%omit-struct-type +%struct-type +VimLFuncDef; +%% +]]) + +for name, def in pairs(funcs) do + args = def.args or 0 + if type(args) == 'number' then + args = {args, args} + elseif #args == 1 then + args[2] = 'MAX_FUNC_ARGS' + end + func = def.func or ('f_' .. name) + data = def.data or "NULL" + gperfpipe:write(('%s, %s, %s, &%s, (FunPtr)%s\n') + :format(name, args[1], args[2], func, data)) +end +gperfpipe:close() diff --git a/scripts/genoptions.lua b/scripts/genoptions.lua index da53d010bd..9d7f235a3b 100644 --- a/scripts/genoptions.lua +++ b/scripts/genoptions.lua @@ -31,6 +31,7 @@ local type_flags={ local redraw_flags={ statuslines='P_RSTAT', current_window='P_RWIN', + current_window_only='P_RWINONLY', current_buffer='P_RBUF', all_windows='P_RALL', everything='P_RCLR', @@ -57,9 +58,14 @@ local get_flags = function(o) add_flag(redraw_flags[r_flag]) end end + if o.expand then + add_flag('P_EXPAND') + if o.expand == 'nodefault' then + add_flag('P_NO_DEF_EXP') + end + end for _, flag_desc in ipairs({ {'alloced'}, - {'expand'}, {'nodefault'}, {'no_mkrc'}, {'vi_def'}, diff --git a/scripts/genunicodetables.lua b/scripts/genunicodetables.lua index 36339e2fc6..66430ba26e 100644 --- a/scripts/genunicodetables.lua +++ b/scripts/genunicodetables.lua @@ -12,19 +12,27 @@ -- 2 then interval applies only to first, third, fifth, … character in range. -- Fourth value is number that should be added to the codepoint to yield -- folded/lower/upper codepoint. +-- 4. emoji_width and emoji_all tables: sorted lists of non-overlapping closed +-- intervals of Emoji characters. emoji_width contains all the characters +-- which don't have ambiguous or double width, and emoji_all has all Emojis. if arg[1] == '--help' then print('Usage:') - print(' genunicodetables.lua UnicodeData.txt CaseFolding.txt ' .. - 'EastAsianWidth.txt') - print(' unicode_tables.generated.h') + print(' genunicodetables.lua unicode/ unicode_tables.generated.h') os.exit(0) end -local unicodedata_fname = arg[1] -local casefolding_fname = arg[2] -local eastasianwidth_fname = arg[3] +local basedir = arg[1] +local pathsep = package.config:sub(1, 1) +local get_path = function(fname) + return basedir .. pathsep .. fname +end + +local unicodedata_fname = get_path('UnicodeData.txt') +local casefolding_fname = get_path('CaseFolding.txt') +local eastasianwidth_fname = get_path('EastAsianWidth.txt') +local emoji_fname = get_path('emoji-data.txt') -local utf_tables_fname = arg[4] +local utf_tables_fname = arg[2] local split_on_semicolons = function(s) local ret = {} @@ -79,6 +87,10 @@ local parse_width_props = function(eaw_fp) return fp_lines_to_lists(eaw_fp, 2, true) end +local parse_emoji_props = function(emoji_fp) + return fp_lines_to_lists(emoji_fp, 2, true) +end + local make_range = function(start, end_, step, add) if step and add then return (' {0x%x, 0x%x, %d, %d},\n'):format( @@ -168,6 +180,7 @@ local build_width_table = function(ut_fp, dataprops, widthprops, widths, local start = -1 local end_ = -1 local dataidx = 1 + local ret = {} for _, p in ipairs(widthprops) do if widths[p[2]:sub(1, 1)] then local rng_start, rng_end = p[1]:find('%.%.') @@ -200,6 +213,7 @@ local build_width_table = function(ut_fp, dataprops, widthprops, widths, else if start >= 0 then ut_fp:write(make_range(start, end_)) + table.insert(ret, {start, end_}) end start = n end @@ -209,6 +223,72 @@ local build_width_table = function(ut_fp, dataprops, widthprops, widths, end if start >= 0 then ut_fp:write(make_range(start, end_)) + table.insert(ret, {start, end_}) + end + ut_fp:write('};\n') + return ret +end + +local build_emoji_table = function(ut_fp, emojiprops, doublewidth, ambiwidth) + local emojiwidth = {} + local emoji = {} + for _, p in ipairs(emojiprops) do + if p[2]:match('Emoji%s+#') then + local rng_start, rng_end = p[1]:find('%.%.') + if rng_start then + n = tonumber(p[1]:sub(1, rng_start - 1), 16) + n_last = tonumber(p[1]:sub(rng_end + 1), 16) + else + n = tonumber(p[1], 16) + n_last = n + end + if #emoji > 0 and n - 1 == emoji[#emoji][2] then + emoji[#emoji][2] = n_last + else + table.insert(emoji, { n, n_last }) + end + + -- Characters below 1F000 may be considered single width traditionally, + -- making them double width causes problems. + if n >= 0x1f000 then + -- exclude characters that are in the ambiguous/doublewidth table + for _, ambi in ipairs(ambiwidth) do + if n >= ambi[1] and n <= ambi[2] then + n = ambi[2] + 1 + end + if n_last >= ambi[1] and n_last <= ambi[2] then + n_last = ambi[1] - 1 + end + end + for _, double in ipairs(doublewidth) do + if n >= double[1] and n <= double[2] then + n = double[2] + 1 + end + if n_last >= double[1] and n_last <= double[2] then + n_last = double[1] - 1 + end + end + + if n <= n_last then + if #emojiwidth > 0 and n - 1 == emojiwidth[#emojiwidth][2] then + emojiwidth[#emojiwidth][2] = n_last + else + table.insert(emojiwidth, { n, n_last }) + end + end + end + end + end + + ut_fp:write('static const struct interval emoji_all[] = {\n') + for _, p in ipairs(emoji) do + ut_fp:write(make_range(p[1], p[2])) + end + ut_fp:write('};\n') + + ut_fp:write('static const struct interval emoji_width[] = {\n') + for _, p in ipairs(emojiwidth) do + ut_fp:write(make_range(p[1], p[2])) end ut_fp:write('};\n') end @@ -233,7 +313,15 @@ local eaw_fp = io.open(eastasianwidth_fname, 'r') local widthprops = parse_width_props(eaw_fp) eaw_fp:close() -build_width_table(ut_fp, dataprops, widthprops, {W=true, F=true}, 'doublewidth') -build_width_table(ut_fp, dataprops, widthprops, {A=true}, 'ambiguous') +local doublewidth = build_width_table(ut_fp, dataprops, widthprops, + {W=true, F=true}, 'doublewidth') +local ambiwidth = build_width_table(ut_fp, dataprops, widthprops, + {A=true}, 'ambiguous') + +local emoji_fp = io.open(emoji_fname, 'r') +local emojiprops = parse_emoji_props(emoji_fp) +emoji_fp:close() + +build_emoji_table(ut_fp, emojiprops, doublewidth, ambiwidth) ut_fp:close() diff --git a/scripts/genvimvim.lua b/scripts/genvimvim.lua index 9135c8e3ab..24c147b811 100644 --- a/scripts/genvimvim.lua +++ b/scripts/genvimvim.lua @@ -1,10 +1,13 @@ +mpack = require('mpack') + if arg[1] == '--help' then - print('Usage: genoptions.lua src/nvim runtime/syntax/vim/generated.vim') + print('Usage: lua genvimvim.lua src/nvim runtime/syntax/vim/generated.vim') os.exit(0) end local nvimsrcdir = arg[1] local syntax_file = arg[2] +local funcs_file = arg[3] package.path = nvimsrcdir .. '/?.lua;' .. package.path @@ -111,28 +114,16 @@ end w('\n\nsyn case match') local vimfun_start = 'syn keyword vimFuncName contained ' w('\n\n' .. vimfun_start) -eval_fd = io.open(nvimsrcdir .. '/eval.c', 'r') +funcs = mpack.unpack(io.open(funcs_file):read("*all")) local started = 0 -for line in eval_fd:lines() do - if line == '} functions[] =' then - started = 1 - elseif started == 1 then - assert (line == '{') - started = 2 - elseif started == 2 then - if line == '};' then - break - end - local func_name = line:match('^ {"(%w+)",') - if func_name then - if lld.line_length > 850 then - w('\n' .. vimfun_start) - end - w(' ' .. func_name) +for name, def in pairs(funcs) do + if name then + if lld.line_length > 850 then + w('\n' .. vimfun_start) end + w(' ' .. name) end end -eval_fd:close() w('\n') syn_fd:close() diff --git a/scripts/git-log-pretty-since.sh b/scripts/git-log-pretty-since.sh index d8e3282fb3..6281127542 100755 --- a/scripts/git-log-pretty-since.sh +++ b/scripts/git-log-pretty-since.sh @@ -19,7 +19,7 @@ is_merge_commit() { git log $1^2 >/dev/null 2>&1 && return 0 || return 1 } -for commit in $(git log --format='%H' --first-parent --since $__SINCE); do +for commit in $(git log --format='%H' --first-parent "$__SINCE"..HEAD); do if is_merge_commit ${commit} ; then if [ -z "$__INVMATCH" ] || ! git log --oneline ${commit}^1..${commit}^2 \ | grep -E "$__INVMATCH" >/dev/null 2>&1 ; then diff --git a/scripts/msgpack-gen.lua b/scripts/msgpack-gen.lua deleted file mode 100644 index c726db3920..0000000000 --- a/scripts/msgpack-gen.lua +++ /dev/null @@ -1,310 +0,0 @@ -lpeg = require('lpeg') -mpack = require('mpack') - --- lpeg grammar for building api metadata from a set of header files. It --- ignores comments and preprocessor commands and parses a very small subset --- of C prototypes with a limited set of types -P, R, S = lpeg.P, lpeg.R, lpeg.S -C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg - -any = P(1) -- (consume one character) -letter = R('az', 'AZ') + S('_$') -alpha = letter + R('09') -nl = P('\r\n') + P('\n') -not_nl = any - nl -ws = S(' \t') + nl -fill = ws ^ 0 -c_comment = P('//') * (not_nl ^ 0) -c_preproc = P('#') * (not_nl ^ 0) -typed_container = - (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')') -c_id = ( - typed_container + - (letter * (alpha ^ 0)) -) -c_void = P('void') -c_param_type = ( - ((P('Error') * fill * P('*') * fill) * Cc('error')) + - (C(c_id) * (ws ^ 1)) - ) -c_type = (C(c_void) * (ws ^ 1)) + c_param_type -c_param = Ct(c_param_type * C(c_id)) -c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0) -c_params = Ct(c_void + c_param_list) -c_proto = Ct( - Cg(c_type, 'return_type') * Cg(c_id, 'name') * - fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') * - Cg(Cc(false), 'async') * - (fill * Cg((P('FUNC_ATTR_ASYNC') * Cc(true)), 'async') ^ -1) * - fill * P(';') - ) -grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1) - --- we need at least 2 arguments since the last one is the output file -assert(#arg >= 1) -functions = {} - --- names of all headers relative to the source root(for inclusion in the --- generated file) -headers = {} --- output file(dispatch function + metadata serialized with msgpack) -outputf = arg[#arg] - --- read each input file, parse and append to the api metadata -for i = 1, #arg - 1 do - local full_path = arg[i] - local parts = {} - for part in string.gmatch(full_path, '[^/]+') do - parts[#parts + 1] = part - end - headers[#headers + 1] = parts[#parts - 1]..'/'..parts[#parts] - - local input = io.open(full_path, 'rb') - local tmp = grammar:match(input:read('*all')) - for i = 1, #tmp do - functions[#functions + 1] = tmp[i] - local fn = tmp[i] - if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then - -- this function should receive the channel id - fn.receives_channel_id = true - -- remove the parameter since it won't be passed by the api client - table.remove(fn.parameters, 1) - end - if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then - -- function can fail if the last parameter type is 'Error' - fn.can_fail = true - -- remove the error parameter, msgpack has it's own special field - -- for specifying errors - fn.parameters[#fn.parameters] = nil - end - end - input:close() -end - - --- start building the output -output = io.open(outputf, 'wb') - -output:write([[ -#include <inttypes.h> -#include <stdbool.h> -#include <stdint.h> -#include <assert.h> -#include <msgpack.h> - -#include "nvim/map.h" -#include "nvim/log.h" -#include "nvim/vim.h" -#include "nvim/msgpack_rpc/helpers.h" -#include "nvim/msgpack_rpc/defs.h" -#include "nvim/api/private/helpers.h" -#include "nvim/api/private/defs.h" -]]) - -for i = 1, #headers do - if headers[i]:sub(-12) ~= '.generated.h' then - output:write('\n#include "nvim/'..headers[i]..'"') - end -end - -output:write([[ - - -static const uint8_t msgpack_metadata[] = { - -]]) --- serialize the API metadata using msgpack and embed into the resulting --- binary for easy querying by clients -packed = mpack.pack(functions) -for i = 1, #packed do - output:write(string.byte(packed, i)..', ') - if i % 10 == 0 then - output:write('\n ') - end -end -output:write([[ -}; - -void msgpack_rpc_init_function_metadata(Dictionary *metadata) -{ - msgpack_unpacked unpacked; - msgpack_unpacked_init(&unpacked); - if (msgpack_unpack_next(&unpacked, - (const char *)msgpack_metadata, - sizeof(msgpack_metadata), - NULL) != MSGPACK_UNPACK_SUCCESS) { - abort(); - } - Object functions; - msgpack_rpc_to_object(&unpacked.data, &functions); - msgpack_unpacked_destroy(&unpacked); - PUT(*metadata, "functions", functions); -} - -]]) - -local function real_type(type) - local rv = type - if typed_container:match(rv) then - if rv:match('Array') then - rv = 'Array' - else - rv = 'Dictionary' - end - end - return rv -end - --- start the handler functions. Visit each function metadata to build the --- handler function with code generated for validating arguments and calling to --- the real API. -for i = 1, #functions do - local fn = functions[i] - local args = {} - - output:write('static Object handle_'..fn.name..'(uint64_t channel_id, uint64_t request_id, Array args, Error *error)') - output:write('\n{') - output:write('\n Object ret = NIL;') - -- Declare/initialize variables that will hold converted arguments - for j = 1, #fn.parameters do - local param = fn.parameters[j] - local converted = 'arg_'..j - output:write('\n '..param[1]..' '..converted..' api_init_'..string.lower(real_type(param[1]))..';') - end - output:write('\n') - output:write('\n if (args.size != '..#fn.parameters..') {') - output:write('\n snprintf(error->msg, sizeof(error->msg), "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);') - output:write('\n error->set = true;') - output:write('\n goto cleanup;') - output:write('\n }\n') - - -- Validation/conversion for each argument - for j = 1, #fn.parameters do - local converted, convert_arg, param, arg - param = fn.parameters[j] - converted = 'arg_'..j - local rt = real_type(param[1]) - if rt ~= 'Object' then - output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {') - output:write('\n '..converted..' = args.items['..(j - 1)..'].data.'..rt:lower()..';') - if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then - -- accept positive integers for Buffers, Windows and Tabpages - output:write('\n } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer > 0) {') - output:write('\n '..converted..' = (unsigned)args.items['..(j - 1)..'].data.integer;') - end - output:write('\n } else {') - output:write('\n snprintf(error->msg, sizeof(error->msg), "Wrong type for argument '..j..', expecting '..param[1]..'");') - output:write('\n error->set = true;') - output:write('\n goto cleanup;') - output:write('\n }\n') - else - output:write('\n '..converted..' = args.items['..(j - 1)..'];\n') - end - - args[#args + 1] = converted - end - - -- function call - local call_args = table.concat(args, ', ') - output:write('\n ') - if fn.return_type ~= 'void' then - -- has a return value, prefix the call with a declaration - output:write(fn.return_type..' rv = ') - end - - -- write the function name and the opening parenthesis - output:write(fn.name..'(') - - if fn.receives_channel_id then - -- if the function receives the channel id, pass it as first argument - if #args > 0 then - output:write('channel_id, '..call_args) - else - output:write('channel_id') - end - else - output:write(call_args) - end - - if fn.can_fail then - -- if the function can fail, also pass a pointer to the local error object - if #args > 0 then - output:write(', error);\n') - else - output:write('error);\n') - end - -- and check for the error - output:write('\n if (error->set) {') - output:write('\n goto cleanup;') - output:write('\n }\n') - else - output:write(');\n') - end - - if fn.return_type ~= 'void' then - output:write('\n ret = '..string.upper(real_type(fn.return_type))..'_OBJ(rv);') - end - -- Now generate the cleanup label for freeing memory allocated for the - -- arguments - output:write('\n\ncleanup:'); - - for j = 1, #fn.parameters do - local param = fn.parameters[j] - output:write('\n api_free_'..string.lower(real_type(param[1]))..'(arg_'..j..');') - end - output:write('\n return ret;\n}\n\n'); -end - --- Generate a function that initializes method names with handler functions -output:write([[ -static Map(String, MsgpackRpcRequestHandler) *methods = NULL; - -void msgpack_rpc_add_method_handler(String method, MsgpackRpcRequestHandler handler) -{ - map_put(String, MsgpackRpcRequestHandler)(methods, method, handler); -} - -void msgpack_rpc_init_method_table(void) -{ - methods = map_new(String, MsgpackRpcRequestHandler)(); - -]]) - --- Keep track of the maximum method name length in order to avoid walking --- strings longer than that when searching for a method handler -local max_fname_len = 0 -for i = 1, #functions do - local fn = functions[i] - output:write(' msgpack_rpc_add_method_handler('.. - '(String) {.data = "'..fn.name..'", '.. - '.size = sizeof("'..fn.name..'") - 1}, '.. - '(MsgpackRpcRequestHandler) {.fn = handle_'.. fn.name.. - ', .async = '..tostring(fn.async)..'});\n') - - if #fn.name > max_fname_len then - max_fname_len = #fn.name - end -end - -output:write('\n}\n\n') - -output:write([[ -MsgpackRpcRequestHandler msgpack_rpc_get_handler_for(const char *name, - size_t name_len) -{ - String m = { - .data=(char *)name, - .size=MIN(name_len, ]]..max_fname_len..[[) - }; - MsgpackRpcRequestHandler rv = - map_get(String, MsgpackRpcRequestHandler)(methods, m); - - if (!rv.fn) { - rv.fn = msgpack_rpc_handle_missing_method; - } - - return rv; -} -]]) - -output:close() diff --git a/scripts/release.sh b/scripts/release.sh index 67738ccc96..93f9fa3d35 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,10 +1,11 @@ -#!/bin/sh +#!/usr/bin/env bash # Performs steps to tag a release. # # Steps: # Create the "release" commit: # - CMakeLists.txt: Unset NVIM_VERSION_PRERELEASE +# - CMakeLists.txt: Unset NVIM_API_PRERELEASE # - Tag the commit. # Create the "version bump" commit: # - CMakeLists.txt: Set NVIM_VERSION_PRERELEASE to "-dev" @@ -17,42 +18,48 @@ set -e set -u set -o pipefail +__sed=$( [ "$(uname)" = Darwin ] && echo 'sed -E' || echo 'sed -r' ) + cd "$(git rev-parse --show-toplevel)" __LAST_TAG=$(git describe --abbrev=0) [ -z "$__LAST_TAG" ] && { echo 'ERROR: no tag found'; exit 1; } __VERSION_MAJOR=$(grep 'set(NVIM_VERSION_MAJOR' CMakeLists.txt\ - |sed -r 's/.*NVIM_VERSION_MAJOR ([[:digit:]]).*/\1/') + |$__sed 's/.*NVIM_VERSION_MAJOR ([[:digit:]]).*/\1/') __VERSION_MINOR=$(grep 'set(NVIM_VERSION_MINOR' CMakeLists.txt\ - |sed -r 's/.*NVIM_VERSION_MINOR ([[:digit:]]).*/\1/') + |$__sed 's/.*NVIM_VERSION_MINOR ([[:digit:]]).*/\1/') __VERSION_PATCH=$(grep 'set(NVIM_VERSION_PATCH' CMakeLists.txt\ - |sed -r 's/.*NVIM_VERSION_PATCH ([[:digit:]]).*/\1/') + |$__sed 's/.*NVIM_VERSION_PATCH ([[:digit:]]).*/\1/') __VERSION="${__VERSION_MAJOR}.${__VERSION_MINOR}.${__VERSION_PATCH}" { [ -z "$__VERSION_MAJOR" ] || [ -z "$__VERSION_MINOR" ] || [ -z "$__VERSION_PATCH" ]; } \ && { echo "ERROR: version parse failed: '${__VERSION}'"; exit 1; } __RELEASE_MSG="NVIM v${__VERSION} -Features: +FEATURES: -Fixes: +FIXES: -Changes: +CHANGES: " __BUMP_MSG="version bump" echo "Most recent tag: ${__LAST_TAG}" echo "Release version: ${__VERSION}" -sed -i -r 's/(NVIM_VERSION_PRERELEASE) "-dev"/\1 ""/' CMakeLists.txt +$__sed -i.bk 's/(NVIM_VERSION_PRERELEASE) "-dev"/\1 ""/' CMakeLists.txt +$__sed -i.bk 's/(NVIM_API_PRERELEASE) true/\1 false/' CMakeLists.txt echo "Building changelog since ${__LAST_TAG}..." __CHANGELOG="$(./scripts/git-log-pretty-since.sh "$__LAST_TAG" 'vim-patch:\S')" git add CMakeLists.txt git commit --edit -m "${__RELEASE_MSG} ${__CHANGELOG}" -git tag -a v"${__VERSION}" -m "NVIM v${__VERSION}" +git tag --sign -a v"${__VERSION}" -m "NVIM v${__VERSION}" + +$__sed -i.bk 's/(NVIM_VERSION_PRERELEASE) ""/\1 "-dev"/' CMakeLists.txt +$__sed -i.bk 's/set\((NVIM_VERSION_PATCH) [[:digit:]]/set(\1 ?/' CMakeLists.txt +nvim +'/NVIM_VERSION' +10new +'exe "norm! iUpdate version numbers!!!\<CR>"' \ + +'norm! 10.' CMakeLists.txt -sed -i -r 's/(NVIM_VERSION_PRERELEASE) ""/\1 "-dev"/' CMakeLists.txt -nvim -c '/NVIM_VERSION' -c 'echo "Update version numbers"' CMakeLists.txt git add CMakeLists.txt git commit -m "$__BUMP_MSG" diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index 7a0001769a..88fb3cae04 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -2,10 +2,11 @@ set -e set -u -set -o pipefail +# Use privileged mode, which e.g. skips using CDPATH. +set -p -readonly NEOVIM_SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -readonly VIM_SOURCE_DIR_DEFAULT="${NEOVIM_SOURCE_DIR}/.vim-src" +readonly NVIM_SOURCE_DIR="${NVIM_SOURCE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +readonly VIM_SOURCE_DIR_DEFAULT="${NVIM_SOURCE_DIR}/.vim-src" readonly VIM_SOURCE_DIR="${VIM_SOURCE_DIR:-${VIM_SOURCE_DIR_DEFAULT}}" readonly BASENAME="$(basename "${0}")" readonly BRANCH_PREFIX="vim-" @@ -24,6 +25,9 @@ usage() { echo " -p {vim-revision} Download and apply the Vim patch vim-revision." echo " vim-revision can be a version number of the " echo " format '7.4.xxx' or a Git commit hash." + echo " -g {vim-revision} Download the Vim patch vim-revision." + echo " vim-revision can be a version number of the " + echo " format '7.4.xxx' or a Git commit hash." echo " -s Submit a vim-patch pull request to Neovim." echo " -r {pr-number} Review a vim-patch pull request to Neovim." echo @@ -57,10 +61,10 @@ clean_files() { read -p "Delete these files (Y/n)? " -n 1 -r reply echo - if [[ "${reply}" =~ ^[Yy]$ ]]; then - rm -- "${CREATED_FILES[@]}" - else + if [[ "${reply}" == n ]]; then echo "You can use 'git clean' to remove these files when you're done." + else + rm -- "${CREATED_FILES[@]}" fi } @@ -92,7 +96,7 @@ commit_message() { find_git_remote() { git remote -v \ - | awk '$2 ~ /github.com[:/]neovim\/neovim/ && $3 == "(fetch)" {print $1}' + | awk '$2 ~ /github.com[:\/]neovim\/neovim/ && $3 == "(fetch)" {print $1; exit}' } assign_commit_details() { @@ -122,6 +126,36 @@ assign_commit_details() { patch_file="vim-${vim_version}.patch" } +# Patch surgery +preprocess_patch() { + local file="$1" + local nvim="nvim -u NORC -i NONE --headless" + + # Remove *.proto, Make*, gui_*, some if_* + local na_src='proto\|Make*\|gui_*\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%('${na_src}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove channel.txt, netbeans.txt, os_*.txt, todo.txt, version*.txt, tags + local na_doc='channel\.txt\|netbeans\.txt\|os_\w\+\.txt\|todo\.txt\|version\d\.txt\|tags' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\%('${na_doc}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove some testdir/Make_*.mak files + local na_src_testdir='Make_amiga.mak\|Make_dos.mak\|Make_ming.mak\|Make_vms.mms' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\%('${na_src_testdir}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove some *.po files. #5622 + local na_po='sjiscorr.c\|ja.sjis.po\|ko.po\|pl.cp1250.po\|pl.po\|ru.cp1251.po\|uk.cp1251.po\|zh_CN.cp936.po\|zh_CN.po\|zh_TW.po' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/po/\%('${na_po}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Rename src/ paths to src/nvim/ + LC_ALL=C sed -e 's/\( [ab]\/src\)/\1\/nvim/g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" + + # Rename path to matchit plugin. + LC_ALL=C sed -e 's@\( [ab]/runtime\)/pack/dist/opt/matchit/\(plugin/matchit.vim\)@\1/\2@g' \ + "$file" > "$file".tmp && mv "$file".tmp "$file" +} + get_vim_patch() { get_vim_sources @@ -131,17 +165,25 @@ get_vim_patch() { >&2 echo "✘ Couldn't find Vim revision '${vim_commit}'." exit 3 } - echo echo "✔ Found Vim revision '${vim_commit}'." - # Patch surgery: preprocess the patch. - # - transform src/ paths to src/nvim/ - local vim_full - vim_full="$(git show -1 --pretty=medium "${vim_commit}" \ - | LC_ALL=C sed -e 's/\( [ab]\/src\)/\1\/nvim/g')" - local neovim_branch="${BRANCH_PREFIX}${vim_version}" + local patch_content + patch_content="$(git --no-pager show --color=never -1 --pretty=medium "${vim_commit}")" + + cd "${NVIM_SOURCE_DIR}" + + printf "Creating patch...\n" + echo "$patch_content" > "${NVIM_SOURCE_DIR}/${patch_file}" + + printf "Pre-processing patch...\n" + preprocess_patch "${NVIM_SOURCE_DIR}/${patch_file}" + + printf "✔ Saved patch to '${NVIM_SOURCE_DIR}/${patch_file}'.\n" +} + +stage_patch() { + get_vim_patch "$1" - cd "${NEOVIM_SOURCE_DIR}" local git_remote git_remote="$(find_git_remote)" local checked_out_branch @@ -151,51 +193,40 @@ get_vim_patch() { echo "✔ Current branch '${checked_out_branch}' seems to be a vim-patch" echo " branch; not creating a new branch." else - echo - echo "Fetching '${git_remote}/master'." + printf "\nFetching '${git_remote}/master'.\n" output="$(git fetch "${git_remote}" master 2>&1)" && echo "✔ ${output}" || (echo "✘ ${output}"; false) + local nvim_branch="${BRANCH_PREFIX}${vim_version}" echo - echo "Creating new branch '${neovim_branch}' based on '${git_remote}/master'." - cd "${NEOVIM_SOURCE_DIR}" - output="$(git checkout -b "${neovim_branch}" "${git_remote}/master" 2>&1)" && + echo "Creating new branch '${nvim_branch}' based on '${git_remote}/master'." + cd "${NVIM_SOURCE_DIR}" + output="$(git checkout -b "${nvim_branch}" "${git_remote}/master" 2>&1)" && echo "✔ ${output}" || (echo "✘ ${output}"; false) fi - echo - echo "Creating empty commit with correct commit message." + printf "\nCreating empty commit with correct commit message.\n" output="$(commit_message | git commit --allow-empty --file 2>&1 -)" && echo "✔ ${output}" || (echo "✘ ${output}"; false) - echo - echo "Creating files." - echo "${vim_full}" > "${NEOVIM_SOURCE_DIR}/${patch_file}" - echo "✔ Saved full commit details to '${NEOVIM_SOURCE_DIR}/${patch_file}'." + printf "\nInstructions: + Proceed to port the patch. This may help: + patch -p1 < ${patch_file} - echo - echo "Instructions:" - echo - echo " Proceed to port the patch." - echo " You might want to try 'patch -p1 < ${patch_file}' first." - echo - echo " If the patch contains a new test, consider porting it to Lua." - echo " You might want to try 'scripts/legacy2luatest.pl'." - echo - echo " Stage your changes ('git add ...') and use 'git commit --amend' to commit." - echo - echo " To port additional patches related to ${vim_version} and add them to the current" - echo " branch, call '${BASENAME} -p' again. Please use this only if it wouldn't make" - echo " sense to send in each patch individually, as it will increase the size of the" - echo " pull request and make it harder to review." - echo - echo " When you are finished, use '${BASENAME} -s' to submit a pull request." - echo - echo " See https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim" - echo " for more information." + Stage your changes ('git add ...') and use 'git commit --amend' to commit. + + To port additional patches related to ${vim_version} and add them to the + current branch, call '${BASENAME} -p' again. + * Do this only for _related_ patches (otherwise it increases the + size of the pull request, making it harder to review) + + When you're done, try '${BASENAME} -s' to create the pull request. + + See the wiki for more information: + * https://github.com/neovim/neovim/wiki/Merging-patches-from-upstream-vim\n" } hub_pr() { @@ -221,7 +252,7 @@ submit_pr() { exit 1 fi - cd "${NEOVIM_SOURCE_DIR}" + cd "${NVIM_SOURCE_DIR}" local checked_out_branch checked_out_branch="$(git rev-parse --abbrev-ref HEAD)" if [[ "${checked_out_branch}" != ${BRANCH_PREFIX}* ]]; then @@ -261,11 +292,11 @@ submit_pr() { local patch_file for patch_file in "${patches[@]}"; do patch_file="vim-${patch_file}.patch" - if [[ ! -f "${NEOVIM_SOURCE_DIR}/${patch_file}" ]]; then + if [[ ! -f "${NVIM_SOURCE_DIR}/${patch_file}" ]]; then continue fi - rm -- "${NEOVIM_SOURCE_DIR}/${patch_file}" - echo "✔ Removed '${NEOVIM_SOURCE_DIR}/${patch_file}'." + rm -- "${NVIM_SOURCE_DIR}/${patch_file}" + echo "✔ Removed '${NVIM_SOURCE_DIR}/${patch_file}'." done } @@ -287,15 +318,15 @@ list_vim_patches() { if [[ -n "${vim_tag}" ]]; then local patch_number="${vim_tag:5}" # Remove prefix like "v7.4." # Tagged Vim patch, check version.c: - is_missing="$(sed -n '/static int included_patches/,/}/p' "${NEOVIM_SOURCE_DIR}/src/nvim/version.c" | + is_missing="$(sed -n '/static int included_patches/,/}/p' "${NVIM_SOURCE_DIR}/src/nvim/version.c" | grep -x -e "[[:space:]]*//[[:space:]]${patch_number} NA.*" -e "[[:space:]]*${patch_number}," >/dev/null && echo "false" || echo "true")" vim_commit="${vim_tag#v}" - if (cd "${VIM_SOURCE_DIR}" && git show --name-only "v${vim_commit}" 2>/dev/null) | grep -q ^runtime; then + if (cd "${VIM_SOURCE_DIR}" && git --no-pager show --color=never --name-only "v${vim_commit}" 2>/dev/null) | grep -q ^runtime; then vim_commit="${vim_commit} (+runtime)" fi else # Untagged Vim patch (e.g. runtime updates), check the Neovim git log: - is_missing="$(cd "${NEOVIM_SOURCE_DIR}" && + is_missing="$(cd "${NVIM_SOURCE_DIR}" && git log -1 --no-merges --grep="vim\-patch:${vim_commit:0:7}" --pretty=format:false)" fi @@ -319,14 +350,14 @@ list_vim_patches() { } review_commit() { - local neovim_commit_url="${1}" - local neovim_patch_url="${neovim_commit_url}.patch" + local nvim_commit_url="${1}" + local nvim_patch_url="${nvim_commit_url}.patch" local git_patch_prefix='Subject: \[PATCH\] ' - local neovim_patch - neovim_patch="$(curl -Ssf "${neovim_patch_url}")" + local nvim_patch + nvim_patch="$(curl -Ssf "${nvim_patch_url}")" local vim_version - vim_version="$(head -n 4 <<< "${neovim_patch}" | sed -n "s/${git_patch_prefix}vim-patch:\([a-z0-9.]*\)$/\1/p")" + vim_version="$(head -n 4 <<< "${nvim_patch}" | sed -n "s/${git_patch_prefix}vim-patch:\([a-z0-9.]*\)$/\1/p")" echo if [[ -n "${vim_version}" ]]; then @@ -335,6 +366,14 @@ review_commit() { echo "✘ Could not detect the Vim patch number." echo " This script assumes that the PR contains only commits" echo " with 'vim-patch:XXX' in their title." + echo + printf -- "$(head -n 4 <<< "${nvim_patch}")\n\n" + local reply + read -p "Continue reviewing (y/N)? " -n 1 -r reply + if [[ "${reply}" == y ]]; then + echo + return + fi exit 1 fi @@ -347,7 +386,7 @@ review_commit() { local message_length message_length="$(wc -l <<< "${expected_commit_message}")" local commit_message - commit_message="$(tail -n +4 <<< "${neovim_patch}" | head -n "${message_length}")" + commit_message="$(tail -n +4 <<< "${nvim_patch}" | head -n "${message_length}")" if [[ "${commit_message#${git_patch_prefix}}" == "${expected_commit_message}" ]]; then echo "✔ Found expected commit message." else @@ -360,18 +399,18 @@ review_commit() { echo echo "Creating files." - echo "${neovim_patch}" > "${NEOVIM_SOURCE_DIR}/n${patch_file}" - echo "✔ Saved pull request diff to '${NEOVIM_SOURCE_DIR}/n${patch_file}'." - CREATED_FILES+=("${NEOVIM_SOURCE_DIR}/n${patch_file}") + echo "${nvim_patch}" > "${NVIM_SOURCE_DIR}/n${patch_file}" + echo "✔ Saved pull request diff to '${NVIM_SOURCE_DIR}/n${patch_file}'." + CREATED_FILES+=("${NVIM_SOURCE_DIR}/n${patch_file}") - curl -Ssfo "${NEOVIM_SOURCE_DIR}/${patch_file}" "${vim_patch_url}" - echo "✔ Saved Vim diff to '${NEOVIM_SOURCE_DIR}/${patch_file}'." - CREATED_FILES+=("${NEOVIM_SOURCE_DIR}/${patch_file}") + curl -Ssfo "${NVIM_SOURCE_DIR}/${patch_file}" "${vim_patch_url}" + echo "✔ Saved Vim diff to '${NVIM_SOURCE_DIR}/${patch_file}'." + CREATED_FILES+=("${NVIM_SOURCE_DIR}/${patch_file}") echo echo "Launching nvim." - nvim -c "cd ${NEOVIM_SOURCE_DIR}" \ - -O "${NEOVIM_SOURCE_DIR}/${patch_file}" "${NEOVIM_SOURCE_DIR}/n${patch_file}" + nvim -c "cd ${NVIM_SOURCE_DIR}" \ + -O "${NVIM_SOURCE_DIR}/${patch_file}" "${NVIM_SOURCE_DIR}/n${patch_file}" } review_pr() { @@ -397,7 +436,7 @@ review_pr() { if [[ "${pr_commit_url}" != "${pr_commit_urls[-1]}" ]]; then read -p "Continue with next commit (Y/n)? " -n 1 -r reply echo - if [[ ! "${reply}" =~ ^[Yy]$ ]]; then + if [[ "${reply}" == n ]]; then break fi fi @@ -406,7 +445,7 @@ review_pr() { clean_files } -while getopts "hlp:r:s" opt; do +while getopts "hlp:g:r:s" opt; do case ${opt} in h) usage @@ -417,6 +456,10 @@ while getopts "hlp:r:s" opt; do exit 0 ;; p) + stage_patch "${OPTARG}" + exit 0 + ;; + g) get_vim_patch "${OPTARG}" exit 0 ;; |