diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/check-includes.py | 66 | ||||
| -rwxr-xr-x | scripts/download-unicode-files.sh | 4 | ||||
| -rw-r--r-- | scripts/gen_api_vimdoc.py | 22 | ||||
| -rw-r--r-- | scripts/gen_events.lua | 65 | ||||
| -rwxr-xr-x | scripts/genappimage.sh | 83 | ||||
| -rwxr-xr-x | scripts/gendeclarations.lua | 261 | ||||
| -rw-r--r-- | scripts/gendispatch.lua | 334 | ||||
| -rw-r--r-- | scripts/geneval.lua | 66 | ||||
| -rw-r--r-- | scripts/genex_cmds.lua | 88 | ||||
| -rw-r--r-- | scripts/genoptions.lua | 190 | ||||
| -rw-r--r-- | scripts/genunicodetables.lua | 327 | ||||
| -rwxr-xr-x | scripts/pvscheck.sh | 454 | ||||
| -rwxr-xr-x | scripts/release.sh | 17 | ||||
| -rwxr-xr-x | scripts/vim-patch.sh | 65 |
14 files changed, 679 insertions, 1363 deletions
diff --git a/scripts/check-includes.py b/scripts/check-includes.py new file mode 100755 index 0000000000..21308a21aa --- /dev/null +++ b/scripts/check-includes.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +import sys +import re +import os + +from subprocess import Popen, PIPE +from argparse import ArgumentParser + + +GENERATED_INCLUDE_RE = re.compile( + r'^\s*#\s*include\s*"([/a-z_0-9.]+\.generated\.h)"(\s+//.*)?$') + + +def main(argv): + argparser = ArgumentParser() + argparser.add_argument('--generated-includes-dir', action='append', + help='Directory where generated includes are located.') + argparser.add_argument('--file', type=open, help='File to check.') + argparser.add_argument('iwyu_args', nargs='*', + help='IWYU arguments, must go after --.') + args = argparser.parse_args(argv) + + with args.file: + include_dirs = [] + + iwyu = Popen(['include-what-you-use', '-xc'] + args.iwyu_args + ['/dev/stdin'], + stdin=PIPE, stdout=PIPE, stderr=PIPE) + + for line in args.file: + match = GENERATED_INCLUDE_RE.match(line) + if match: + for d in args.generated_includes_dir: + try: + f = open(os.path.join(d, match.group(1))) + except IOError: + continue + else: + with f: + for generated_line in f: + iwyu.stdin.write(generated_line) + break + else: + raise IOError('Failed to find {0}'.format(match.group(1))) + else: + iwyu.stdin.write(line) + + iwyu.stdin.close() + + out = iwyu.stdout.read() + err = iwyu.stderr.read() + + ret = iwyu.wait() + + if ret != 2: + print('IWYU failed with exit code {0}:'.format(ret)) + print('{0} stdout {0}'.format('=' * ((80 - len(' stdout ')) // 2))) + print(out) + print('{0} stderr {0}'.format('=' * ((80 - len(' stderr ')) // 2))) + print(err) + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/download-unicode-files.sh b/scripts/download-unicode-files.sh index 54fc32550c..5f38d0589a 100755 --- a/scripts/download-unicode-files.sh +++ b/scripts/download-unicode-files.sh @@ -22,7 +22,7 @@ UNIDIR=${1:-$UNIDIR_DEFAULT} DOWNLOAD_URL_BASE=${2:-$DOWNLOAD_URL_BASE_DEFAULT} for filename in $data_files ; do - curl -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/$filename" + curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/UNIDATA/$filename" ( cd "$UNIDIR" git add $filename @@ -30,7 +30,7 @@ for filename in $data_files ; do done for filename in $emoji_files ; do - curl -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/emoji/3.0/$filename" + curl -L -o "$UNIDIR/$filename" "$DOWNLOAD_URL_BASE/emoji/latest/$filename" ( cd "$UNIDIR" git add $filename diff --git a/scripts/gen_api_vimdoc.py b/scripts/gen_api_vimdoc.py index 4dcb42d685..51e585a007 100644 --- a/scripts/gen_api_vimdoc.py +++ b/scripts/gen_api_vimdoc.py @@ -38,6 +38,10 @@ import subprocess from xml.dom import minidom +if sys.version_info[0] < 3: + print("use Python 3") + sys.exit(1) + doc_filename = 'api.txt' # String used to find the start of the generated part of the doc. section_start_token = '*api-global*' @@ -69,7 +73,7 @@ text_width = 78 script_path = os.path.abspath(__file__) base_dir = os.path.dirname(os.path.dirname(script_path)) src_dir = os.path.join(base_dir, 'src/nvim/api') -out_dir = os.path.join(base_dir, 'tmp/api_doc') +out_dir = os.path.join(base_dir, 'tmp-api-doc') filter_cmd = '%s %s' % (sys.executable, script_path) seen_funcs = set() @@ -217,7 +221,12 @@ def parse_para(parent, width=62): width=width) + '\n') elif child.nodeName == 'simplesect': kind = child.getAttribute('kind') - if kind == 'return': + if kind == 'note': + lines.append('Note:') + lines.append(doc_wrap(parse_para(child), + prefix=' ', + width=width)) + elif kind == 'return': lines.append('%s:~' % kind.title()) lines.append(doc_wrap(parse_para(child), prefix=' ', @@ -280,14 +289,19 @@ def parse_source_xml(filename): parts = return_type.strip('_').split('_') return_type = '%s(%s)' % (parts[0], ', '.join(parts[1:])) + name = get_text(get_child(member, 'name')) + annotations = get_text(get_child(member, 'argsstring')) if annotations and ')' in annotations: annotations = annotations.rsplit(')', 1)[-1].strip() + # XXX: (doxygen 1.8.11) 'argsstring' only includes FUNC_ATTR_* + # attributes if the function signature is non-void. + # Force attributes here for such functions. + if name == 'nvim_get_mode' and len(annotations) == 0: + annotations += 'FUNC_API_ASYNC' annotations = filter(None, map(lambda x: annotation_map.get(x), annotations.split())) - name = get_text(get_child(member, 'name')) - vimtag = '*%s()*' % name args = [] type_length = 0 diff --git a/scripts/gen_events.lua b/scripts/gen_events.lua deleted file mode 100644 index 75e0b3da3a..0000000000 --- a/scripts/gen_events.lua +++ /dev/null @@ -1,65 +0,0 @@ -if arg[1] == '--help' then - print('Usage: gen_events.lua src/nvim enum_file event_names_file') - os.exit(0) -end - -local nvimsrcdir = arg[1] -local fileio_enum_file = arg[2] -local names_file = arg[3] - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local auevents = require('auevents') -local events = auevents.events -local aliases = auevents.aliases - -enum_tgt = io.open(fileio_enum_file, 'w') -names_tgt = io.open(names_file, 'w') - -enum_tgt:write('typedef enum auto_event {') -names_tgt:write([[ -static const struct event_name { - size_t len; - char *name; - event_T event; -} event_names[] = {]]) - -for i, event in ipairs(events) do - if i > 1 then - comma = ',\n' - else - comma = '\n' - end - enum_tgt:write(('%s EVENT_%s = %u'):format(comma, event:upper(), i - 1)) - names_tgt:write(('%s {%u, "%s", EVENT_%s}'):format(comma, #event, event, event:upper())) -end - -for alias, event in pairs(aliases) do - names_tgt:write((',\n {%u, "%s", EVENT_%s}'):format(#alias, alias, event:upper())) -end - -names_tgt:write(',\n {0, NULL, (event_T)0}') - -enum_tgt:write('\n} event_T;\n') -names_tgt:write('\n};\n') - -enum_tgt:write(('\n#define NUM_EVENTS %u\n'):format(#events)) -names_tgt:write('\nstatic AutoPat *first_autopat[NUM_EVENTS] = {\n ') -line_len = 1 -for i = 1,((#events) - 1) do - line_len = line_len + #(' NULL,') - if line_len > 80 then - names_tgt:write('\n ') - line_len = 1 + #(' NULL,') - end - names_tgt:write(' NULL,') -end -if line_len + #(' NULL') > 80 then - names_tgt:write('\n NULL') -else - names_tgt:write(' NULL') -end -names_tgt:write('\n};\n') - -enum_tgt:close() -names_tgt:close() diff --git a/scripts/genappimage.sh b/scripts/genappimage.sh new file mode 100755 index 0000000000..a73ccd86c7 --- /dev/null +++ b/scripts/genappimage.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +######################################################################## +# Package the binaries built as an AppImage +# By Simon Peter 2016 +# For more information, see http://appimage.org/ +######################################################################## + +# App arch, used by generate_appimage. +if [ -z "$ARCH" ]; then + export ARCH="$(arch)" +fi + +# App name, used by generate_appimage. +APP=nvim + +ROOT_DIR="$(git rev-parse --show-toplevel)" +APP_BUILD_DIR="$ROOT_DIR/build" +APP_DIR="$APP.AppDir" + +######################################################################## +# Compile nvim and install it into AppDir +######################################################################## + +# Build and install nvim into the AppImage +make CMAKE_BUILD_TYPE=RelWithDebInfo CMAKE_EXTRA_FLAGS="-DCMAKE_INSTALL_PREFIX=${APP_DIR}/usr -DCMAKE_INSTALL_MANDIR=man" +make install + +######################################################################## +# Get helper functions and move to AppDir +######################################################################## + +# App version, used by generate_appimage. +VERSION=$("$ROOT_DIR"/build/bin/nvim --version | head -n 1 | grep -o 'v.*') + +cd "$APP_BUILD_DIR" + +curl -Lo "$APP_BUILD_DIR"/appimage_functions.sh https://github.com/probonopd/AppImages/raw/master/functions.sh +. ./appimage_functions.sh + +# Copy desktop and icon file to AppDir for AppRun to pick them up. +# get_apprun +# get_desktop +cp "$ROOT_DIR/runtime/nvim.desktop" "$APP_DIR/" +cp "$ROOT_DIR/runtime/nvim.png" "$APP_DIR/" + +cd "$APP_DIR" + +# copy dependencies +copy_deps +# Move the libraries to usr/bin +move_lib + +# Delete stuff that should not go into the AppImage. +# Delete dangerous libraries; see +# https://github.com/probonopd/AppImages/blob/master/excludelist +delete_blacklisted + +######################################################################## +# AppDir complete. Now package it as an AppImage. +######################################################################## + +# No need for a fancy script. AppRun can just be a symlink to nvim. +ln -s usr/bin/nvim AppRun + +cd "$APP_BUILD_DIR" # Get out of AppImage directory. + +# Generate AppImage. +# - Expects: $ARCH, $APP, $VERSION env vars +# - Expects: ./$APP.AppDir/ directory +# - Produces: ../out/$APP-$VERSION.glibc$GLIBC_NEEDED-$ARCH.AppImage +generate_appimage + +# NOTE: There is currently a bug in the `generate_appimage` function (see +# https://github.com/probonopd/AppImages/issues/228) that causes repeated builds +# that result in the same name to fail. +# Moving the final executable to a different folder gets around this issue. + +mv "$ROOT_DIR"/out/*.AppImage "$ROOT_DIR"/build/bin +# Remove the (now empty) folder the AppImage was built in +rmdir "$ROOT_DIR"/out + +echo 'genappimage.sh: finished' diff --git a/scripts/gendeclarations.lua b/scripts/gendeclarations.lua deleted file mode 100755 index ff69b18ae4..0000000000 --- a/scripts/gendeclarations.lua +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/lua - -local fname = arg[1] -local static_fname = arg[2] -local non_static_fname = arg[3] -local preproc_fname = arg[4] - - -local lpeg = require('lpeg') - -local fold = function (func, ...) - local result = nil - for i, v in ipairs({...}) do - if result == nil then - result = v - else - result = func(result, v) - end - end - return result -end - -local folder = function (func) - return function (...) - return fold(func, ...) - end -end - -local lit = lpeg.P -local set = function(...) - return lpeg.S(fold(function (a, b) return a .. b end, ...)) -end -local any_character = lpeg.P(1) -local rng = function(s, e) return lpeg.R(s .. e) end -local concat = folder(function (a, b) return a * b end) -local branch = folder(function (a, b) return a + b end) -local one_or_more = function(v) return v ^ 1 end -local two_or_more = function(v) return v ^ 2 end -local any_amount = function(v) return v ^ 0 end -local one_or_no = function(v) return v ^ -1 end -local look_behind = lpeg.B -local look_ahead = function(v) return #v end -local neg_look_ahead = function(v) return -v end -local neg_look_behind = function(v) return -look_behind(v) end - -local w = branch( - rng('a', 'z'), - rng('A', 'Z'), - lit('_') -) -local aw = branch( - w, - rng('0', '9') -) -local s = set(' ', '\n', '\t') -local raw_word = concat(w, any_amount(aw)) -local right_word = concat( - raw_word, - neg_look_ahead(aw) -) -local word = branch( - concat( - branch(lit('ArrayOf('), lit('DictionaryOf(')), -- typed container macro - one_or_more(any_character - lit(')')), - lit(')') - ), - concat( - neg_look_behind(aw), - right_word - ) -) -local spaces = any_amount(branch( - s, - -- Comments are really handled by preprocessor, so the following is not needed - concat( - lit('/*'), - any_amount(concat( - neg_look_ahead(lit('*/')), - any_character - )), - lit('*/') - ), - concat( - lit('//'), - any_amount(concat( - neg_look_ahead(lit('\n')), - any_character - )), - lit('\n') - ), - -- Linemarker inserted by preprocessor - concat( - lit('# '), - any_amount(concat( - neg_look_ahead(lit('\n')), - any_character - )), - lit('\n') - ) -)) -local typ_part = concat( - word, - any_amount(concat( - spaces, - lit('*') - )), - spaces -) -local typ = one_or_more(typ_part) -local typ_id = two_or_more(typ_part) -local arg = typ_id -- argument name is swallowed by typ -local pattern = concat( - typ_id, -- return type with function name - spaces, - lit('('), - spaces, - one_or_no(branch( -- function arguments - concat( - arg, -- first argument, does not require comma - any_amount(concat( -- following arguments, start with a comma - spaces, - lit(','), - spaces, - arg, - any_amount(concat( - lit('['), - spaces, - any_amount(aw), - spaces, - lit(']') - )) - )), - one_or_no(concat( - spaces, - lit(','), - spaces, - lit('...') - )) - ), - lit('void') -- also accepts just void - )), - spaces, - lit(')'), - any_amount(concat( -- optional attributes - spaces, - lit('FUNC_'), - any_amount(aw), - one_or_no(concat( -- attribute argument - spaces, - lit('('), - any_amount(concat( - neg_look_ahead(lit(')')), - any_character - )), - lit(')') - )) - )), - look_ahead(concat( -- definition must be followed by "{" - spaces, - lit('{') - )) -) - -if fname == '--help' then - print'Usage:' - print() - print' gendeclarations.lua definitions.c static.h non-static.h preprocessor.i' - os.exit() -end - -local preproc_f = io.open(preproc_fname) -local text = preproc_f:read("*all") -preproc_f:close() - - -local header = [[ -#ifndef DEFINE_FUNC_ATTRIBUTES -# define DEFINE_FUNC_ATTRIBUTES -#endif -#include "nvim/func_attr.h" -#undef DEFINE_FUNC_ATTRIBUTES -]] - -local footer = [[ -#include "nvim/func_attr.h" -]] - -local non_static = header -local static = header - -local filepattern = '^#%a* %d+ "[^"]-/?([^"/]+)"' -local curfile - -init = 0 -curfile = nil -neededfile = fname:match('[^/]+$') -while init ~= nil do - init = text:find('\n', init) - if init == nil then - break - end - init = init + 1 - if text:sub(init, init) == '#' then - file = text:match(filepattern, init) - if file ~= nil then - curfile = file - end - elseif curfile == neededfile then - s = init - e = pattern:match(text, init) - if e ~= nil then - local declaration = text:sub(s, e - 1) - -- Comments are really handled by preprocessor, so the following is not - -- needed - declaration = declaration:gsub('/%*.-%*/', '') - declaration = declaration:gsub('//.-\n', '\n') - - declaration = declaration:gsub('# .-\n', '') - - declaration = declaration:gsub('\n', ' ') - declaration = declaration:gsub('%s+', ' ') - declaration = declaration:gsub(' ?%( ?', '(') - -- declaration = declaration:gsub(' ?%) ?', ')') - declaration = declaration:gsub(' ?, ?', ', ') - declaration = declaration:gsub(' ?(%*+) ?', ' %1') - declaration = declaration:gsub(' ?(FUNC_ATTR_)', ' %1') - declaration = declaration:gsub(' $', '') - declaration = declaration .. ';\n' - if text:sub(s, s + 5) == 'static' then - static = static .. declaration - else - non_static = non_static .. declaration - end - init = e - end - end -end - -non_static = non_static .. footer -static = static .. footer - -local F -F = io.open(static_fname, 'w') -F:write(static) -F:close() - --- Before generating the non-static headers, check if the current file(if --- exists) is different from the new one. If they are the same, we won't touch --- the current version to avoid triggering an unnecessary rebuilds of modules --- that depend on this one -F = io.open(non_static_fname, 'r') -if F ~= nil then - if F:read('*a') == non_static then - os.exit(0) - end - io.close(F) -end - -F = io.open(non_static_fname, 'w') -F:write(non_static) -F:close() diff --git a/scripts/gendispatch.lua b/scripts/gendispatch.lua deleted file mode 100644 index 45a4f4a4de..0000000000 --- a/scripts/gendispatch.lua +++ /dev/null @@ -1,334 +0,0 @@ -lpeg = require('lpeg') -mpack = require('mpack') - --- lpeg grammar for building api metadata from a set of header files. It --- ignores comments and preprocessor commands and parses a very small subset --- of C prototypes with a limited set of types -P, R, S = lpeg.P, lpeg.R, lpeg.S -C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg - -any = P(1) -- (consume one character) -letter = R('az', 'AZ') + S('_$') -num = R('09') -alpha = letter + num -nl = P('\r\n') + P('\n') -not_nl = any - nl -ws = S(' \t') + nl -fill = ws ^ 0 -c_comment = P('//') * (not_nl ^ 0) -c_preproc = P('#') * (not_nl ^ 0) -typed_container = - (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')') -c_id = ( - typed_container + - (letter * (alpha ^ 0)) -) -c_void = P('void') -c_param_type = ( - ((P('Error') * fill * P('*') * fill) * Cc('error')) + - (C(c_id) * (ws ^ 1)) - ) -c_type = (C(c_void) * (ws ^ 1)) + c_param_type -c_param = Ct(c_param_type * C(c_id)) -c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0) -c_params = Ct(c_void + c_param_list) -c_proto = Ct( - Cg(c_type, 'return_type') * Cg(c_id, 'name') * - fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') * - Cg(Cc(false), 'async') * - (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) * - (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) * - (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) * - (fill * Cg((P('FUNC_API_NOEVAL') * Cc(true)), 'noeval') ^ -1) * - fill * P(';') - ) -grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1) - --- we need at least 4 arguments since the last two are output files -assert(#arg >= 3) -functions = {} - -local nvimsrcdir = arg[1] -package.path = nvimsrcdir .. '/?.lua;' .. package.path - --- names of all headers relative to the source root (for inclusion in the --- generated file) -headers = {} - --- output h file with generated dispatch functions -dispatch_outputf = arg[#arg-2] --- output h file with packed metadata -funcs_metadata_outputf = arg[#arg-1] --- output metadata mpack file, for use by other build scripts -mpack_outputf = arg[#arg] - --- set of function names, used to detect duplicates -function_names = {} - --- read each input file, parse and append to the api metadata -for i = 2, #arg - 3 do - local full_path = arg[i] - local parts = {} - for part in string.gmatch(full_path, '[^/]+') do - parts[#parts + 1] = part - end - headers[#headers + 1] = parts[#parts - 1]..'/'..parts[#parts] - - local input = io.open(full_path, 'rb') - local tmp = grammar:match(input:read('*all')) - for i = 1, #tmp do - local fn = tmp[i] - if not fn.noexport then - functions[#functions + 1] = tmp[i] - function_names[fn.name] = true - if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then - -- this function should receive the channel id - fn.receives_channel_id = true - -- remove the parameter since it won't be passed by the api client - table.remove(fn.parameters, 1) - end - if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then - -- function can fail if the last parameter type is 'Error' - fn.can_fail = true - -- remove the error parameter, msgpack has it's own special field - -- for specifying errors - fn.parameters[#fn.parameters] = nil - end - end - end - input:close() -end - -local function shallowcopy(orig) - local copy = {} - for orig_key, orig_value in pairs(orig) do - copy[orig_key] = orig_value - end - return copy -end - -local function startswith(String,Start) - return string.sub(String,1,string.len(Start))==Start -end - --- Export functions under older deprecated names. --- These will be removed eventually. -local deprecated_aliases = require("api.dispatch_deprecated") -for i,f in ipairs(shallowcopy(functions)) do - local ismethod = false - if startswith(f.name, "nvim_") then - if f.since == nil then - print("Function "..f.name.." lacks since field.\n") - os.exit(1) - end - f.since = tonumber(f.since) - if startswith(f.name, "nvim_buf_") then - ismethod = true - elseif startswith(f.name, "nvim_win_") then - ismethod = true - elseif startswith(f.name, "nvim_tabpage_") then - ismethod = true - end - else - f.noeval = true - f.since = 0 - f.deprecated_since = 1 - end - f.method = ismethod - local newname = deprecated_aliases[f.name] - if newname ~= nil then - if function_names[newname] then - -- duplicate - print("Function "..f.name.." has deprecated alias\n" - ..newname.." which has a separate implementation.\n".. - "Please remove it from src/nvim/api/dispatch_deprecated.lua") - os.exit(1) - end - local newf = shallowcopy(f) - newf.name = newname - if newname == "ui_try_resize" then - -- The return type was incorrectly set to Object in 0.1.5. - -- Keep it that way for clients that rely on this. - newf.return_type = "Object" - end - newf.impl_name = f.name - newf.noeval = true - newf.since = 0 - newf.deprecated_since = 1 - functions[#functions+1] = newf - end -end - --- don't expose internal attributes like "impl_name" in public metadata -exported_attributes = {'name', 'parameters', 'return_type', 'method', - 'since', 'deprecated_since'} -exported_functions = {} -for _,f in ipairs(functions) do - local f_exported = {} - for _,attr in ipairs(exported_attributes) do - f_exported[attr] = f[attr] - end - exported_functions[#exported_functions+1] = f_exported -end - - -funcs_metadata_output = io.open(funcs_metadata_outputf, 'wb') -funcs_metadata_output:write([[ -static const uint8_t funcs_metadata[] = { -]]) - --- serialize the API metadata using msgpack and embed into the resulting --- binary for easy querying by clients -packed_exported_functions = mpack.pack(exported_functions) -for i = 1, #packed_exported_functions do - funcs_metadata_output:write(string.byte(packed_exported_functions, i)..', ') - if i % 10 == 0 then - funcs_metadata_output:write('\n ') - end -end -funcs_metadata_output:write([[ -}; -]]) -funcs_metadata_output:close() - --- start building the dispatch wrapper output -output = io.open(dispatch_outputf, 'wb') - -local function real_type(type) - local rv = type - if typed_container:match(rv) then - if rv:match('Array') then - rv = 'Array' - else - rv = 'Dictionary' - end - end - return rv -end - --- start the handler functions. Visit each function metadata to build the --- handler function with code generated for validating arguments and calling to --- the real API. -for i = 1, #functions do - local fn = functions[i] - if fn.impl_name == nil then - local args = {} - - output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)') - output:write('\n{') - output:write('\n Object ret = NIL;') - -- Declare/initialize variables that will hold converted arguments - for j = 1, #fn.parameters do - local param = fn.parameters[j] - local converted = 'arg_'..j - output:write('\n '..param[1]..' '..converted..';') - end - output:write('\n') - output:write('\n if (args.size != '..#fn.parameters..') {') - output:write('\n snprintf(error->msg, sizeof(error->msg), "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);') - output:write('\n error->set = true;') - output:write('\n goto cleanup;') - output:write('\n }\n') - - -- Validation/conversion for each argument - for j = 1, #fn.parameters do - local converted, convert_arg, param, arg - param = fn.parameters[j] - converted = 'arg_'..j - local rt = real_type(param[1]) - if rt ~= 'Object' then - if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then - -- Buffer, Window, and Tabpage have a specific type, but are stored in integer - output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..' && args.items['..(j - 1)..'].data.integer >= 0) {') - output:write('\n '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;') - else - output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {') - output:write('\n '..converted..' = args.items['..(j - 1)..'].data.'..rt:lower()..';') - end - if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then - -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages - output:write('\n } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer >= 0) {') - output:write('\n '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;') - end - output:write('\n } else {') - output:write('\n snprintf(error->msg, sizeof(error->msg), "Wrong type for argument '..j..', expecting '..param[1]..'");') - output:write('\n error->set = true;') - output:write('\n goto cleanup;') - output:write('\n }\n') - else - output:write('\n '..converted..' = args.items['..(j - 1)..'];\n') - end - - args[#args + 1] = converted - end - - -- function call - local call_args = table.concat(args, ', ') - output:write('\n ') - if fn.return_type ~= 'void' then - -- has a return value, prefix the call with a declaration - output:write(fn.return_type..' rv = ') - end - - -- write the function name and the opening parenthesis - output:write(fn.name..'(') - - if fn.receives_channel_id then - -- if the function receives the channel id, pass it as first argument - if #args > 0 or fn.can_fail then - output:write('channel_id, '..call_args) - else - output:write('channel_id') - end - else - output:write(call_args) - end - - if fn.can_fail then - -- if the function can fail, also pass a pointer to the local error object - if #args > 0 then - output:write(', error);\n') - else - output:write('error);\n') - end - -- and check for the error - output:write('\n if (error->set) {') - output:write('\n goto cleanup;') - output:write('\n }\n') - else - output:write(');\n') - end - - if fn.return_type ~= 'void' then - output:write('\n ret = '..string.upper(real_type(fn.return_type))..'_OBJ(rv);') - end - output:write('\n\ncleanup:'); - - output:write('\n return ret;\n}\n\n'); - end -end - --- Generate a function that initializes method names with handler functions -output:write([[ -void msgpack_rpc_init_method_table(void) -{ - methods = map_new(String, MsgpackRpcRequestHandler)(); - -]]) - -for i = 1, #functions do - local fn = functions[i] - output:write(' msgpack_rpc_add_method_handler('.. - '(String) {.data = "'..fn.name..'", '.. - '.size = sizeof("'..fn.name..'") - 1}, '.. - '(MsgpackRpcRequestHandler) {.fn = handle_'.. (fn.impl_name or fn.name).. - ', .async = '..tostring(fn.async)..'});\n') - -end - -output:write('\n}\n\n') -output:close() - -mpack_output = io.open(mpack_outputf, 'wb') -mpack_output:write(mpack.pack(functions)) -mpack_output:close() diff --git a/scripts/geneval.lua b/scripts/geneval.lua deleted file mode 100644 index b1ba76296c..0000000000 --- a/scripts/geneval.lua +++ /dev/null @@ -1,66 +0,0 @@ -mpack = require('mpack') - -local nvimsrcdir = arg[1] -local autodir = arg[2] -local metadata_file = arg[3] -local funcs_file = arg[4] - -if nvimsrcdir == '--help' then - print([[ -Usage: - lua geneval.lua src/nvim build/src/nvim/auto - -Will generate build/src/nvim/auto/funcs.generated.h with definition of functions -static const array. -]]) - os.exit(0) -end - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local funcsfname = autodir .. '/funcs.generated.h' - -local gperfpipe = io.open(funcsfname .. '.gperf', 'wb') - -local funcs = require('eval').funcs -local metadata = mpack.unpack(io.open(arg[3], 'rb'):read("*all")) -for i,fun in ipairs(metadata) do - if not fun.noeval then - funcs[fun.name] = { - args=#fun.parameters, - func='api_wrapper', - data='&handle_'..fun.name, - } - end -end - -local funcsdata = io.open(funcs_file, 'w') -funcsdata:write(mpack.pack(funcs)) -funcsdata:close() - -gperfpipe:write([[ -%language=ANSI-C -%global-table -%define initializer-suffix ,0,0,NULL,NULL -%define word-array-name functions -%define hash-function-name hash_internal_func_gperf -%define lookup-function-name find_internal_func_gperf -%omit-struct-type -%struct-type -VimLFuncDef; -%% -]]) - -for name, def in pairs(funcs) do - args = def.args or 0 - if type(args) == 'number' then - args = {args, args} - elseif #args == 1 then - args[2] = 'MAX_FUNC_ARGS' - end - func = def.func or ('f_' .. name) - data = def.data or "NULL" - gperfpipe:write(('%s, %s, %s, &%s, (FunPtr)%s\n') - :format(name, args[1], args[2], func, data)) -end -gperfpipe:close() diff --git a/scripts/genex_cmds.lua b/scripts/genex_cmds.lua deleted file mode 100644 index cb566d46ca..0000000000 --- a/scripts/genex_cmds.lua +++ /dev/null @@ -1,88 +0,0 @@ -local nvimsrcdir = arg[1] -local includedir = arg[2] -local autodir = arg[3] - -if nvimsrcdir == '--help' then - print ([[ -Usage: - lua genex_cmds.lua src/nvim build/include build/src/nvim/auto - -Will generate files build/include/ex_cmds_enum.generated.h with cmdidx_T -enum and build/src/nvim/auto/ex_cmds_defs.generated.h with main Ex commands -definitions. -]]) - os.exit(0) -end - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local enumfname = includedir .. '/ex_cmds_enum.generated.h' -local defsfname = autodir .. '/ex_cmds_defs.generated.h' - -local enumfile = io.open(enumfname, 'w') -local defsfile = io.open(defsfname, 'w') - -local defs = require('ex_cmds') -local lastchar = nil - -local i -local cmd -local first = true -local prevfirstchar = nil - -local byte_a = string.byte('a') -local byte_z = string.byte('z') - -local cmdidxs = string.format([[ -static const cmdidx_T cmdidxs[%u] = { -]], byte_z - byte_a + 2) - -enumfile:write([[ -typedef enum CMD_index { -]]) -defsfile:write(string.format([[ -static CommandDefinition cmdnames[%u] = { -]], #defs)) -for i, cmd in ipairs(defs) do - local enumname = cmd.enum or ('CMD_' .. cmd.command) - firstchar = string.byte(cmd.command) - if firstchar ~= prevfirstchar then - if (not prevfirstchar - or (byte_a <= firstchar and firstchar <= byte_z) - or (byte_a <= prevfirstchar and prevfirstchar <= byte_z)) then - if not first then - cmdidxs = cmdidxs .. ',\n' - end - cmdidxs = cmdidxs .. ' ' .. enumname - end - prevfirstchar = firstchar - end - if first then - first = false - else - defsfile:write(',\n') - end - enumfile:write(' ' .. enumname .. ',\n') - defsfile:write(string.format([[ - [%s] = { - .cmd_name = (char_u *) "%s", - .cmd_func = (ex_func_T)&%s, - .cmd_argt = %uL, - .cmd_addr_type = %i - }]], enumname, cmd.command, cmd.func, cmd.flags, cmd.addr_type)) -end -defsfile:write([[ - -}; -]]) -enumfile:write([[ - CMD_SIZE, - CMD_USER = -1, - CMD_USER_BUF = -2 -} cmdidx_T; -]]) -cmdidxs = cmdidxs .. [[ - -}; -]] -defsfile:write(cmdidxs) diff --git a/scripts/genoptions.lua b/scripts/genoptions.lua deleted file mode 100644 index 9d7f235a3b..0000000000 --- a/scripts/genoptions.lua +++ /dev/null @@ -1,190 +0,0 @@ -if arg[1] == '--help' then - print('Usage: genoptions.lua src/nvim options_file') - os.exit(0) -end - -local nvimsrcdir = arg[1] -local options_file = arg[2] - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local opt_fd = io.open(options_file, 'w') - -local w = function(s) - if s:match('^ %.') then - opt_fd:write(s .. ',\n') - else - opt_fd:write(s .. '\n') - end -end - -local options = require('options') - -cstr = options.cstr - -local type_flags={ - bool='P_BOOL', - number='P_NUM', - string='P_STRING', -} - -local redraw_flags={ - statuslines='P_RSTAT', - current_window='P_RWIN', - current_window_only='P_RWINONLY', - current_buffer='P_RBUF', - all_windows='P_RALL', - everything='P_RCLR', - curswant='P_CURSWANT', -} - -local list_flags={ - comma='P_COMMA', - onecomma='P_ONECOMMA', - flags='P_FLAGLIST', - flagscomma='P_COMMA|P_FLAGLIST', -} - -local get_flags = function(o) - local ret = {type_flags[o.type]} - local add_flag = function(f) - ret[1] = ret[1] .. '|' .. f - end - if o.list then - add_flag(list_flags[o.list]) - end - if o.redraw then - for _, r_flag in ipairs(o.redraw) do - add_flag(redraw_flags[r_flag]) - end - end - if o.expand then - add_flag('P_EXPAND') - if o.expand == 'nodefault' then - add_flag('P_NO_DEF_EXP') - end - end - for _, flag_desc in ipairs({ - {'alloced'}, - {'nodefault'}, - {'no_mkrc'}, - {'vi_def'}, - {'vim'}, - {'secure'}, - {'gettext'}, - {'noglob'}, - {'normal_fname_chars', 'P_NFNAME'}, - {'pri_mkrc'}, - {'deny_in_modelines', 'P_NO_ML'}, - {'deny_duplicates', 'P_NODUP'}, - }) do - local key_name = flag_desc[1] - local def_name = flag_desc[2] or ('P_' .. key_name:upper()) - if o[key_name] then - add_flag(def_name) - end - end - return ret[1] -end - -local get_cond -get_cond = function(c, base_string) - local cond_string = base_string or '#if ' - if type(c) == 'table' then - cond_string = cond_string .. get_cond(c[1], '') - for i, subc in ipairs(c) do - if i > 1 then - cond_string = cond_string .. ' && ' .. get_cond(subc, '') - end - end - elseif c:sub(1, 1) == '!' then - cond_string = cond_string .. '!defined(' .. c:sub(2) .. ')' - else - cond_string = cond_string .. 'defined(' .. c .. ')' - end - return cond_string -end - -value_dumpers = { - ['function']=function(v) return v() end, - string=cstr, - boolean=function(v) return v and 'true' or 'false' end, - number=function(v) return ('%iL'):format(v) end, - ['nil']=function(v) return '0L' end, -} - -local get_value = function(v) - return '(char_u *) ' .. value_dumpers[type(v)](v) -end - -local get_defaults = function(d) - return '{' .. get_value(d.vi) .. ', ' .. get_value(d.vim) .. '}' -end - -local defines = {} - -local dump_option = function(i, o) - w(' [' .. ('%u'):format(i - 1) .. ']={') - w(' .fullname=' .. cstr(o.full_name)) - if o.abbreviation then - w(' .shortname=' .. cstr(o.abbreviation)) - end - w(' .flags=' .. get_flags(o)) - if o.enable_if then - w(get_cond(o.enable_if)) - end - if o.varname then - w(' .var=(char_u *)&' .. o.varname) - elseif #o.scope == 1 and o.scope[1] == 'window' then - w(' .var=VAR_WIN') - end - if o.enable_if then - w('#endif') - end - if #o.scope == 1 and o.scope[1] == 'global' then - w(' .indir=PV_NONE') - else - assert (#o.scope == 1 or #o.scope == 2) - assert (#o.scope == 1 or o.scope[1] == 'global') - local min_scope = o.scope[#o.scope] - local varname = o.pv_name or o.varname or ( - 'p_' .. (o.abbreviation or o.full_name)) - local pv_name = ( - 'OPT_' .. min_scope:sub(1, 3):upper() .. '(' .. ( - min_scope:sub(1, 1):upper() .. 'V_' .. varname:sub(3):upper() - ) .. ')' - ) - if #o.scope == 2 then - pv_name = 'OPT_BOTH(' .. pv_name .. ')' - end - defines['PV_' .. varname:sub(3):upper()] = pv_name - w(' .indir=' .. pv_name) - end - if o.defaults then - if o.defaults.condition then - w(get_cond(o.defaults.condition)) - end - w(' .def_val=' .. get_defaults(o.defaults.if_true)) - if o.defaults.condition then - if o.defaults.if_false then - w('#else') - w(' .def_val=' .. get_defaults(o.defaults.if_false)) - end - w('#endif') - end - end - w(' },') -end - -w('static vimoption_T options[] = {') -for i, o in ipairs(options.options) do - dump_option(i, o) -end -w(' [' .. ('%u'):format(#options.options) .. ']={.fullname=NULL}') -w('};') -w('') - -for k, v in pairs(defines) do - w('#define ' .. k .. ' ' .. v) -end -opt_fd:close() diff --git a/scripts/genunicodetables.lua b/scripts/genunicodetables.lua deleted file mode 100644 index 66430ba26e..0000000000 --- a/scripts/genunicodetables.lua +++ /dev/null @@ -1,327 +0,0 @@ --- Script creates the following tables in unicode_tables.generated.h: --- --- 1. doublewidth and ambiguous tables: sorted list of non-overlapping closed --- intervals. Codepoints in these intervals have double (W or F) or ambiguous --- (A) east asian width respectively. --- 2. combining table: same as the above, but characters inside are combining --- characters (i.e. have general categories equal to Mn, Mc or Me). --- 3. foldCase, toLower and toUpper tables used to convert characters to --- folded/lower/upper variants. In these tables first two values are --- character ranges: like in previous tables they are sorted and must be --- non-overlapping. Third value means step inside the range: e.g. if it is --- 2 then interval applies only to first, third, fifth, … character in range. --- Fourth value is number that should be added to the codepoint to yield --- folded/lower/upper codepoint. --- 4. emoji_width and emoji_all tables: sorted lists of non-overlapping closed --- intervals of Emoji characters. emoji_width contains all the characters --- which don't have ambiguous or double width, and emoji_all has all Emojis. -if arg[1] == '--help' then - print('Usage:') - print(' genunicodetables.lua unicode/ unicode_tables.generated.h') - os.exit(0) -end - -local basedir = arg[1] -local pathsep = package.config:sub(1, 1) -local get_path = function(fname) - return basedir .. pathsep .. fname -end - -local unicodedata_fname = get_path('UnicodeData.txt') -local casefolding_fname = get_path('CaseFolding.txt') -local eastasianwidth_fname = get_path('EastAsianWidth.txt') -local emoji_fname = get_path('emoji-data.txt') - -local utf_tables_fname = arg[2] - -local split_on_semicolons = function(s) - local ret = {} - local idx = 1 - while idx <= #s + 1 do - item = s:match('^[^;]*', idx) - idx = idx + #item + 1 - if idx <= #s + 1 then - assert(s:sub(idx - 1, idx - 1) == ';') - end - item = item:gsub('^%s*', '') - item = item:gsub('%s*$', '') - table.insert(ret, item) - end - return ret -end - -local fp_lines_to_lists = function(fp, n, has_comments) - local ret = {} - local line - local i = 0 - while true do - i = i + 1 - line = fp:read('*l') - if not line then - break - end - if (not has_comments - or (line:sub(1, 1) ~= '#' and not line:match('^%s*$'))) then - local l = split_on_semicolons(line) - if #l ~= n then - io.stderr:write(('Found %s items in line %u, expected %u\n'):format( - #l, i, n)) - io.stderr:write('Line: ' .. line .. '\n') - return nil - end - table.insert(ret, l) - end - end - return ret -end - -local parse_data_to_props = function(ud_fp) - return fp_lines_to_lists(ud_fp, 15, false) -end - -local parse_fold_props = function(cf_fp) - return fp_lines_to_lists(cf_fp, 4, true) -end - -local parse_width_props = function(eaw_fp) - return fp_lines_to_lists(eaw_fp, 2, true) -end - -local parse_emoji_props = function(emoji_fp) - return fp_lines_to_lists(emoji_fp, 2, true) -end - -local make_range = function(start, end_, step, add) - if step and add then - return (' {0x%x, 0x%x, %d, %d},\n'):format( - start, end_, step == 0 and -1 or step, add) - else - return (' {0x%04x, 0x%04x},\n'):format(start, end_) - end -end - -local build_convert_table = function(ut_fp, props, cond_func, nl_index, - table_name) - ut_fp:write('static const convertStruct ' .. table_name .. '[] = {\n') - local start = -1 - local end_ = -1 - local step = 0 - local add = -1 - for _, p in ipairs(props) do - if cond_func(p) then - local n = tonumber(p[1], 16) - local nl = tonumber(p[nl_index], 16) - if start >= 0 and add == (nl - n) and (step == 0 or n - end_ == step) then - -- Continue with the same range. - step = n - end_ - end_ = n - else - if start >= 0 then - -- Produce previous range. - ut_fp:write(make_range(start, end_, step, add)) - end - start = n - end_ = n - step = 0 - add = nl - n - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_, step, add)) - end - ut_fp:write('};\n') -end - -local build_case_table = function(ut_fp, dataprops, table_name, index) - local cond_func = function(p) - return p[index] ~= '' - end - return build_convert_table(ut_fp, dataprops, cond_func, index, - 'to' .. table_name) -end - -local build_fold_table = function(ut_fp, foldprops) - local cond_func = function(p) - return (p[2] == 'C' or p[2] == 'S') - end - return build_convert_table(ut_fp, foldprops, cond_func, 3, 'foldCase') -end - -local build_combining_table = function(ut_fp, dataprops) - ut_fp:write('static const struct interval combining[] = {\n') - local start = -1 - local end_ = -1 - for _, p in ipairs(dataprops) do - if (({Mn=true, Mc=true, Me=true})[p[3]]) then - local n = tonumber(p[1], 16) - if start >= 0 and end_ + 1 == n then - -- Continue with the same range. - end_ = n - else - if start >= 0 then - -- Produce previous range. - ut_fp:write(make_range(start, end_)) - end - start = n - end_ = n - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_)) - end - ut_fp:write('};\n') -end - -local build_width_table = function(ut_fp, dataprops, widthprops, widths, - table_name) - ut_fp:write('static const struct interval ' .. table_name .. '[] = {\n') - local start = -1 - local end_ = -1 - local dataidx = 1 - local ret = {} - for _, p in ipairs(widthprops) do - if widths[p[2]:sub(1, 1)] then - local rng_start, rng_end = p[1]:find('%.%.') - local n, n_last - if rng_start then - -- It is a range. We don’t check for composing char then. - n = tonumber(p[1]:sub(1, rng_start - 1), 16) - n_last = tonumber(p[1]:sub(rng_end + 1), 16) - else - n = tonumber(p[1], 16) - n_last = n - end - local dn - while true do - dn = tonumber(dataprops[dataidx][1], 16) - if dn >= n then - break - end - dataidx = dataidx + 1 - end - if dn ~= n and n_last == n then - io.stderr:write('Cannot find character ' .. n .. ' in data table.\n') - end - -- Only use the char when it’s not a composing char. - -- But use all chars from a range. - local dp = dataprops[dataidx] - if (n_last > n) or (not (({Mn=true, Mc=true, Me=true})[dp[3]])) then - if start >= 0 and end_ + 1 == n then - -- Continue with the same range. - else - if start >= 0 then - ut_fp:write(make_range(start, end_)) - table.insert(ret, {start, end_}) - end - start = n - end - end_ = n_last - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_)) - table.insert(ret, {start, end_}) - end - ut_fp:write('};\n') - return ret -end - -local build_emoji_table = function(ut_fp, emojiprops, doublewidth, ambiwidth) - local emojiwidth = {} - local emoji = {} - for _, p in ipairs(emojiprops) do - if p[2]:match('Emoji%s+#') then - local rng_start, rng_end = p[1]:find('%.%.') - if rng_start then - n = tonumber(p[1]:sub(1, rng_start - 1), 16) - n_last = tonumber(p[1]:sub(rng_end + 1), 16) - else - n = tonumber(p[1], 16) - n_last = n - end - if #emoji > 0 and n - 1 == emoji[#emoji][2] then - emoji[#emoji][2] = n_last - else - table.insert(emoji, { n, n_last }) - end - - -- Characters below 1F000 may be considered single width traditionally, - -- making them double width causes problems. - if n >= 0x1f000 then - -- exclude characters that are in the ambiguous/doublewidth table - for _, ambi in ipairs(ambiwidth) do - if n >= ambi[1] and n <= ambi[2] then - n = ambi[2] + 1 - end - if n_last >= ambi[1] and n_last <= ambi[2] then - n_last = ambi[1] - 1 - end - end - for _, double in ipairs(doublewidth) do - if n >= double[1] and n <= double[2] then - n = double[2] + 1 - end - if n_last >= double[1] and n_last <= double[2] then - n_last = double[1] - 1 - end - end - - if n <= n_last then - if #emojiwidth > 0 and n - 1 == emojiwidth[#emojiwidth][2] then - emojiwidth[#emojiwidth][2] = n_last - else - table.insert(emojiwidth, { n, n_last }) - end - end - end - end - end - - ut_fp:write('static const struct interval emoji_all[] = {\n') - for _, p in ipairs(emoji) do - ut_fp:write(make_range(p[1], p[2])) - end - ut_fp:write('};\n') - - ut_fp:write('static const struct interval emoji_width[] = {\n') - for _, p in ipairs(emojiwidth) do - ut_fp:write(make_range(p[1], p[2])) - end - ut_fp:write('};\n') -end - -local ud_fp = io.open(unicodedata_fname, 'r') -local dataprops = parse_data_to_props(ud_fp) -ud_fp:close() - -local ut_fp = io.open(utf_tables_fname, 'w') - -build_case_table(ut_fp, dataprops, 'Lower', 14) -build_case_table(ut_fp, dataprops, 'Upper', 13) -build_combining_table(ut_fp, dataprops) - -local cf_fp = io.open(casefolding_fname, 'r') -local foldprops = parse_fold_props(cf_fp) -cf_fp:close() - -build_fold_table(ut_fp, foldprops) - -local eaw_fp = io.open(eastasianwidth_fname, 'r') -local widthprops = parse_width_props(eaw_fp) -eaw_fp:close() - -local doublewidth = build_width_table(ut_fp, dataprops, widthprops, - {W=true, F=true}, 'doublewidth') -local ambiwidth = build_width_table(ut_fp, dataprops, widthprops, - {A=true}, 'ambiguous') - -local emoji_fp = io.open(emoji_fname, 'r') -local emojiprops = parse_emoji_props(emoji_fp) -emoji_fp:close() - -build_emoji_table(ut_fp, emojiprops, doublewidth, ambiwidth) - -ut_fp:close() diff --git a/scripts/pvscheck.sh b/scripts/pvscheck.sh new file mode 100755 index 0000000000..30c4d296d7 --- /dev/null +++ b/scripts/pvscheck.sh @@ -0,0 +1,454 @@ +#!/bin/sh + +# Assume that "local" is available. +# shellcheck disable=SC2039 + +set -e +# Note: -u causes problems with posh, it barks at “undefined” $@ when no +# arguments provided. +test -z "$POSH_VERSION" && set -u + +get_jobs_num() { + if [ -n "${TRAVIS:-}" ] ; then + # HACK: /proc/cpuinfo on Travis CI is misleading, so hardcode 1. + echo 1 + else + echo $(( $(grep -c "^processor" /proc/cpuinfo) + 1 )) + fi +} + +help() { + echo 'Usage:' + echo ' pvscheck.sh [--pvs URL] [--deps] [--environment-cc]' + echo ' [target-directory [branch]]' + echo ' pvscheck.sh [--pvs URL] [--recheck] [--environment-cc]' + echo ' [target-directory]' + echo ' pvscheck.sh [--pvs URL] --only-analyse [target-directory]' + echo ' pvscheck.sh [--pvs URL] --pvs-install {target-directory}' + echo ' pvscheck.sh --patch [--only-build]' + echo + echo ' --pvs: Fetch pvs-studio from URL.' + echo ' --pvs detect: Auto-detect latest version (by scraping viva64.com).' + echo + echo ' --deps: (for regular run) Use top-level Makefile and build deps.' + echo ' Without this it assumes all dependencies are already' + echo ' installed.' + echo + echo ' --environment-cc: (for regular run and --recheck) Do not export' + echo ' CC=clang. Build is still run with CFLAGS=-O0.' + echo + echo ' --only-build: (for --patch) Only patch files in ./build directory.' + echo + echo ' --pvs-install: Only install PVS-studio to the specified location.' + echo + echo ' --patch: patch sources in the current directory.' + echo ' Does not patch already patched files.' + echo ' Does not run analysis.' + echo + echo ' --recheck: run analysis on a prepared target directory.' + echo + echo ' --only-analyse: run analysis on a prepared target directory ' + echo ' without building Neovim.' + echo + echo ' target-directory: Directory where build should occur.' + echo ' Default: ../neovim-pvs' + echo + echo ' branch: Branch to check.' + echo ' Default: master.' +} + +getopts_error() { + local msg="$1" ; shift + local do_help= + if test "$msg" = "--help" ; then + msg="$1" ; shift + do_help=1 + fi + printf '%s\n' "$msg" >&2 + if test -n "$do_help" ; then + printf '\n' >&2 + help >&2 + fi + echo 'return 1' + return 1 +} + +# Usage `eval "$(getopts_long long_defs -- positionals_defs -- "$@")"` +# +# long_defs: list of pairs of arguments like `longopt action`. +# positionals_defs: list of arguments like `action`. +# +# `action` is a space-separated commands: +# +# store_const [const] [varname] [default] +# Store constant [const] (default 1) (note: eval’ed) if argument is present +# (long options only). Assumes long option accepts no arguments. +# store [varname] [default] +# Store value. Assumes long option needs an argument. +# run {func} [varname] [default] +# Run function {func} and store its output to the [varname]. Assumes no +# arguments accepted (long options only). +# modify {func} [varname] [default] +# Like run, but assumes a single argument, passed to function {func} as $1. +# +# All actions stores empty value if neither [varname] nor [default] are +# present. [default] is evaled by top-level `eval`, so be careful. Also note +# that no arguments may contain spaces, including [default] and [const]. +getopts_long() { + local positional= + local opt_bases="" + while test $# -gt 0 ; do + local arg="$1" ; shift + local opt_base= + local act= + local opt_name= + if test -z "$positional" ; then + if test "$arg" = "--" ; then + positional=0 + continue + fi + act="$1" ; shift + opt_name="$(echo "$arg" | tr '-' '_')" + opt_base="longopt_$opt_name" + else + if test "$arg" = "--" ; then + break + fi + : $(( positional+=1 )) + act="$arg" + opt_name="arg_$positional" + opt_base="positional_$positional" + fi + opt_bases="$opt_bases $opt_base" + eval "local varname_$opt_base=$opt_name" + local i=0 + for act_subarg in $act ; do + eval "local act_$(( i+=1 ))_$opt_base=\"\$act_subarg\"" + done + done + # Process options + local positional=0 + local force_positional= + while test $# -gt 0 ; do + local argument="$1" ; shift + local opt_base= + local has_equal= + local equal_arg= + local is_positional= + if test "$argument" = "--" ; then + force_positional=1 + continue + elif test -z "$force_positional" && test "${argument#--}" != "$argument" + then + local opt_name="${argument#--}" + local opt_name_striparg="${opt_name%%=*}" + if test "$opt_name" = "$opt_name_striparg" ; then + has_equal=0 + else + has_equal=1 + equal_arg="${argument#*=}" + opt_name="$opt_name_striparg" + fi + # Use trailing x to prevent stripping newlines + opt_name="$(printf '%sx' "$opt_name" | tr '-' '_')" + opt_name="${opt_name%x}" + if test -n "$(printf '%sx' "$opt_name" | tr -d 'a-z_')" ; then + getopts_error "Option contains invalid characters: $opt_name" + fi + opt_base="longopt_$opt_name" + else + : $(( positional+=1 )) + opt_base="positional_$positional" + is_positional=1 + fi + if test -n "$opt_base" ; then + eval "local occurred_$opt_base=1" + + eval "local act_1=\"\${act_1_$opt_base:-}\"" + eval "local varname=\"\${varname_$opt_base:-}\"" + local need_val= + local func= + case "$act_1" in + (store_const) + eval "local const=\"\${act_2_${opt_base}:-1}\"" + eval "local varname=\"\${act_3_${opt_base}:-$varname}\"" + printf 'local %s=%s\n' "$varname" "$const" + ;; + (store) + eval "varname=\"\${act_2_${opt_base}:-$varname}\"" + need_val=1 + ;; + (run) + eval "func=\"\${act_2_${opt_base}}\"" + eval "varname=\"\${act_3_${opt_base}:-$varname}\"" + printf 'local %s="$(%s)"\n' "$varname" "$func" + ;; + (modify) + eval "func=\"\${act_2_${opt_base}}\"" + eval "varname=\"\${act_3_${opt_base}:-$varname}\"" + need_val=1 + ;; + ("") + getopts_error --help "Wrong argument: $argument" + ;; + esac + if test -n "$need_val" ; then + local val= + if test -z "$is_positional" ; then + if test $has_equal = 1 ; then + val="$equal_arg" + else + if test $# -eq 0 ; then + getopts_error "Missing argument for $opt_name" + fi + val="$1" ; shift + fi + else + val="$argument" + fi + local escaped_val="'$(printf "%s" "$val" | sed "s/'/'\\\\''/g")'" + case "$act_1" in + (store) + printf 'local %s=%s\n' "$varname" "$escaped_val" + ;; + (modify) + printf 'local %s="$(%s %s)"\n' "$varname" "$func" "$escaped_val" + ;; + esac + fi + fi + done + # Print default values when no values were provided + local opt_base= + for opt_base in $opt_bases ; do + eval "local occurred=\"\${occurred_$opt_base:-}\"" + if test -n "$occurred" ; then + continue + fi + eval "local act_1=\"\$act_1_$opt_base\"" + eval "local varname=\"\$varname_$opt_base\"" + case "$act_1" in + (store) + eval "local varname=\"\${act_2_${opt_base}:-$varname}\"" + eval "local default=\"\${act_3_${opt_base}:-}\"" + printf 'local %s=%s\n' "$varname" "$default" + ;; + (store_const|run|modify) + eval "local varname=\"\${act_3_${opt_base}:-$varname}\"" + eval "local default=\"\${act_4_${opt_base}:-}\"" + printf 'local %s=%s\n' "$varname" "$default" + ;; + esac + done +} + +get_pvs_comment() { + local tgt="$1" ; shift + + cat > "$tgt/pvs-comment" << EOF +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +EOF +} + +install_pvs() {( + local tgt="$1" ; shift + local pvs_url="$1" ; shift + + cd "$tgt" + + mkdir pvs-studio + cd pvs-studio + + curl -L -o pvs-studio.tar.gz "$pvs_url" + tar xzf pvs-studio.tar.gz + rm pvs-studio.tar.gz + local pvsdir="$(find . -maxdepth 1 -mindepth 1)" + find "$pvsdir" -maxdepth 1 -mindepth 1 -exec mv '{}' . \; + rmdir "$pvsdir" +)} + +create_compile_commands() {( + local tgt="$1" ; shift + local deps="$1" ; shift + local environment_cc="$1" ; shift + + if test -z "$environment_cc" ; then + export CC=clang + fi + export CFLAGS=' -O0 ' + + if test -z "$deps" ; then + mkdir -p "$tgt/build" + ( + cd "$tgt/build" + + cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$PWD/root" + make -j"$(get_jobs_num)" + ) + else + ( + cd "$tgt" + + make -j"$(get_jobs_num)" CMAKE_EXTRA_FLAGS=" -DCMAKE_INSTALL_PREFIX=$PWD/root -DCMAKE_BUILD_TYPE=Debug " + ) + fi + find "$tgt/build/src/nvim/auto" -name '*.test-include.c' -delete +)} + +# Warning: realdir below only cares about directories unlike realpath. +# +# realpath is not available in Ubuntu trusty yet. +realdir() {( + local dir="$1" + cd "$dir" + printf '%s\n' "$PWD" +)} + +patch_sources() {( + local tgt="$1" ; shift + local only_bulid="${1}" ; shift + + get_pvs_comment "$tgt" + + local sh_script=' + pvs_comment="$(cat pvs-comment ; echo -n EOS)" + filehead="$(head -c $(( ${#pvs_comment} - 3 )) "$1" ; echo -n EOS)" + if test "x$filehead" != "x$pvs_comment" ; then + cat pvs-comment "$1" > "$1.tmp" + mv "$1.tmp" "$1" + fi + ' + + cd "$tgt" + + if test "$only_build" != "--only-build" ; then + find \ + src/nvim test/functional/fixtures test/unit/fixtures \ + -name '*.c' \ + -exec /bin/sh -c "$sh_script" - '{}' \; + fi + + find \ + build/src/nvim/auto build/config \ + -name '*.c' -not -name '*.test-include.c' \ + -exec /bin/sh -c "$sh_script" - '{}' \; + + rm pvs-comment +)} + +run_analysis() {( + local tgt="$1" ; shift + + cd "$tgt" + + pvs-studio-analyzer \ + analyze \ + --threads "$(get_jobs_num)" \ + --output-file PVS-studio.log \ + --verbose \ + --file build/compile_commands.json \ + --sourcetree-root . + + plog-converter -t xml -o PVS-studio.xml PVS-studio.log + plog-converter -t errorfile -o PVS-studio.err PVS-studio.log + plog-converter -t tasklist -o PVS-studio.tsk PVS-studio.log +)} + +detect_url() { + local url="${1:-detect}" + if test "$url" = detect ; then + curl --silent -L 'https://www.viva64.com/en/pvs-studio-download-linux/' \ + | grep -o 'https\{0,1\}://[^"<>]\{1,\}/pvs-studio[^/"<>]*\.tgz' \ + || echo FAILED + else + printf '%s' "$url" + fi +} + +do_check() { + local tgt="$1" ; shift + local branch="$1" ; shift + local pvs_url="$1" ; shift + local deps="$1" ; shift + local environment_cc="$1" ; shift + + if test -z "$pvs_url" || test "$pvs_url" = FAILED ; then + pvs_url="$(detect_url detect)" + if test -z "$pvs_url" || test "$pvs_url" = FAILED ; then + echo "failed to auto-detect PVS URL" + exit 1 + fi + echo "Auto-detected PVS URL: ${pvs_url}" + fi + + git clone --branch="$branch" . "$tgt" + + install_pvs "$tgt" "$pvs_url" + + do_recheck "$tgt" "$deps" "$environment_cc" +} + +do_recheck() { + local tgt="$1" ; shift + local deps="$1" ; shift + local environment_cc="$1" ; shift + + create_compile_commands "$tgt" "$deps" "$environment_cc" + + do_analysis "$tgt" +} + +do_analysis() { + local tgt="$1" ; shift + + if test -d "$tgt/pvs-studio" ; then + local saved_pwd="$PWD" + cd "$tgt/pvs-studio" + export PATH="$PWD/bin${PATH+:}${PATH}" + cd "$saved_pwd" + fi + + run_analysis "$tgt" +} + +main() { + eval "$( + getopts_long \ + help store_const \ + pvs 'modify detect_url pvs_url' \ + patch store_const \ + only-build 'store_const --only-build' \ + recheck store_const \ + only-analyse store_const \ + pvs-install store_const \ + deps store_const \ + environment-cc store_const \ + -- \ + 'modify realdir tgt "$PWD/../neovim-pvs"' \ + 'store branch master' \ + -- "$@" + )" + + if test -n "$help" ; then + help + return 0 + fi + + # set -x + + if test -n "$patch" ; then + patch_sources "$tgt" "$only_build" + elif test -n "$pvs_install" ; then + install_pvs "$tgt" "$pvs_url" + elif test -n "$recheck" ; then + do_recheck "$tgt" "$deps" "$environment_cc" + elif test -n "$only_analyse" ; then + do_analysis "$tgt" + else + do_check "$tgt" "$branch" "$pvs_url" "$deps" "$environment_cc" + fi +} + +main "$@" diff --git a/scripts/release.sh b/scripts/release.sh index dac5e9b177..f1dbf99473 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -18,6 +18,8 @@ set -e set -u set -o pipefail +USE_CURRENT_COMMIT=${1:-no} + __sed=$( [ "$(uname)" = Darwin ] && echo 'sed -E' || echo 'sed -r' ) cd "$(git rev-parse --show-toplevel)" @@ -51,15 +53,18 @@ echo "Release version: ${__VERSION}" $__sed -i.bk 's/(NVIM_VERSION_PRERELEASE) "-dev"/\1 ""/' CMakeLists.txt if grep '(NVIM_API_PRERELEASE true)' CMakeLists.txt > /dev/null; then $__sed -i.bk 's/(NVIM_API_PRERELEASE) true/\1 false/' CMakeLists.txt - cp build/funcs_data.mpack test/functional/fixtures/api_level_$__API_LEVEL.mpack + build/bin/nvim --api-info > test/functional/fixtures/api_level_$__API_LEVEL.mpack git add test/functional/fixtures/api_level_$__API_LEVEL.mpack fi -echo "Building changelog since ${__LAST_TAG}..." -__CHANGELOG="$(./scripts/git-log-pretty-since.sh "$__LAST_TAG" 'vim-patch:\S')" +if ! test "$USE_CURRENT_COMMIT" = 'use-current-commit' ; then + echo "Building changelog since ${__LAST_TAG}..." + __CHANGELOG="$(./scripts/git-log-pretty-since.sh "$__LAST_TAG" 'vim-patch:\S')" + + git add CMakeLists.txt + git commit --edit -m "${__RELEASE_MSG} ${__CHANGELOG}" +fi -git add CMakeLists.txt -git commit --edit -m "${__RELEASE_MSG} ${__CHANGELOG}" git tag --sign -a v"${__VERSION}" -m "NVIM v${__VERSION}" $__sed -i.bk 's/(NVIM_VERSION_PRERELEASE) ""/\1 "-dev"/' CMakeLists.txt @@ -70,6 +75,8 @@ nvim +'/NVIM_VERSION' +10new +'exe "norm! iUpdate version numbers!!!\<CR>"' \ git add CMakeLists.txt git commit -m "$__BUMP_MSG" +rm CMakeLists.txt.bk || true + echo " Next steps: - Double-check NVIM_VERSION_* in CMakeLists.txt diff --git a/scripts/vim-patch.sh b/scripts/vim-patch.sh index bf4b7d9080..2875e7d95a 100755 --- a/scripts/vim-patch.sh +++ b/scripts/vim-patch.sh @@ -22,9 +22,10 @@ usage() { echo "Options:" echo " -h Show this message and exit." echo " -l Show list of Vim patches missing from Neovim." - echo " -p {vim-revision} Download and apply the Vim patch vim-revision." - echo " vim-revision can be a version number of the " - echo " format '7.4.xxx' or a Git commit hash." + echo " -p {vim-revision} Download and generate the specified Vim patch." + echo " vim-revision can be a version number '8.0.xxx'" + echo " or a valid Git ref (hash, tag, etc.)." + echo " -P {vim-revision} Download, generate and apply the Vim patch." echo " -g {vim-revision} Download the Vim patch vim-revision." echo " vim-revision can be a version number of the " echo " format '7.4.xxx' or a Git commit hash." @@ -32,7 +33,7 @@ usage() { echo " -r {pr-number} Review a vim-patch pull request to Neovim." echo echo "Set VIM_SOURCE_DIR to change where Vim's sources are stored." - echo "The default is '${VIM_SOURCE_DIR_DEFAULT}'." + echo "Default is '${VIM_SOURCE_DIR_DEFAULT}'." } # Checks if a program is in the user's PATH, and is executable. @@ -109,7 +110,7 @@ assign_commit_details() { local strip_commit_line=true else # Interpret parameter as commit hash. - vim_version="${1:0:7}" + vim_version="${1:0:12}" vim_commit=$(cd "${VIM_SOURCE_DIR}" \ && git log -1 --format="%H" "${vim_version}") local strip_commit_line=false @@ -133,19 +134,26 @@ preprocess_patch() { # Remove *.proto, Make*, gui_*, some if_* local na_src='proto\|Make*\|gui_*\|if_lua\|if_mzsch\|if_olepp\|if_ole\|if_perl\|if_py\|if_ruby\|if_tcl\|if_xcmdsrv' - 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%('${na_src}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/\S*\<\%(testdir/\)\@<!\%('${na_src}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove channel.txt, netbeans.txt, os_*.txt, term.txt, todo.txt, version*.txt, tags + local na_doc='channel\.txt\|netbeans\.txt\|os_\w\+\.txt\|term\.txt\|todo\.txt\|version\d\.txt\|tags' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\<\%('${na_doc}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" - # Remove channel.txt, netbeans.txt, os_*.txt, todo.txt, version*.txt, tags - local na_doc='channel\.txt\|netbeans\.txt\|os_\w\+\.txt\|todo\.txt\|version\d\.txt\|tags$' - 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/doc/\%('${na_doc}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + # Remove "Last change ..." changes in doc files. + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'%s/^@@.*\n.*For Vim version.*Last change.*\n.*For Vim version.*Last change.*//' +w +q "$file" # Remove some testdir/Make_*.mak files local na_src_testdir='Make_amiga.mak\|Make_dos.mak\|Make_ming.mak\|Make_vms.mms' - 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\%('${na_src_testdir}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/testdir/\<\%('${na_src_testdir}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" # Remove some *.po files. #5622 local na_po='sjiscorr.c\|ja.sjis.po\|ko.po\|pl.cp1250.po\|pl.po\|ru.cp1251.po\|uk.cp1251.po\|zh_CN.cp936.po\|zh_CN.po\|zh_TW.po' - 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/po/\%('${na_po}'\)@norm! d/\v(^diff)|%$
' +w +q "$file" + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/src/po/\<\%('${na_po}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" + + # Remove vimrc_example.vim + local na_vimrcexample='vimrc_example\.vim' + 2>/dev/null $nvim --cmd 'set dir=/tmp' +'g@^diff --git a/runtime/\<\%('${na_vimrcexample}'\)\>@norm! d/\v(^diff)|%$
' +w +q "$file" # Rename src/ paths to src/nvim/ LC_ALL=C sed -e 's/\( [ab]\/src\)/\1\/nvim/g' \ @@ -183,6 +191,7 @@ get_vim_patch() { stage_patch() { get_vim_patch "$1" + local try_apply="${2:-}" local git_remote git_remote="$(find_git_remote)" @@ -212,14 +221,23 @@ stage_patch() { echo "✔ ${output}" || (echo "✘ ${output}"; false) - printf "\nInstructions: - Proceed to port the patch. This may help: - patch -p1 < ${patch_file} + if test -n "$try_apply" ; then + if ! check_executable patch; then + printf "\n✘ 'patch' command not found\n" + else + printf "\nApplying patch...\n" + patch -p1 --posix < "${patch_file}" + fi + printf "\nInstructions:\n Proceed to port the patch.\n" + else + printf "\nInstructions:\n Proceed to port the patch.\n Try the 'patch' command (or use '${BASENAME} -P ...' next time):\n patch -p1 < ${patch_file}\n" + fi - Stage your changes ('git add ...') and use 'git commit --amend' to commit. + printf " + Stage your changes ('git add ...'), then use 'git commit --amend' to commit. - To port additional patches related to ${vim_version} and add them to the - current branch, call '${BASENAME} -p' again. + To port more patches (if any) related to ${vim_version}, + run '${BASENAME}' again. * Do this only for _related_ patches (otherwise it increases the size of the pull request, making it harder to review) @@ -277,7 +295,7 @@ submit_pr() { echo "Pushing to 'origin/${checked_out_branch}'." output="$(git push origin "${checked_out_branch}" 2>&1)" && echo "✔ ${output}" || - (echo "✘ ${output}"; git reset --soft HEAD^1; false) + (echo "✘ ${output}"; false) echo fi @@ -307,7 +325,7 @@ list_vim_patches() { # Get missing Vim commits local vim_commits - vim_commits="$(cd "${VIM_SOURCE_DIR}" && git log --reverse --format='%H' v7.4.1979..HEAD)" + vim_commits="$(cd "${VIM_SOURCE_DIR}" && git log --reverse --format='%H' v8.0.0000..HEAD)" local vim_commit for vim_commit in ${vim_commits}; do @@ -317,8 +335,9 @@ list_vim_patches() { vim_tag="$(cd "${VIM_SOURCE_DIR}" && git describe --tags --exact-match "${vim_commit}" 2>/dev/null)" || true if [[ -n "${vim_tag}" ]]; then local patch_number="${vim_tag:5}" # Remove prefix like "v7.4." + patch_number="$(echo ${patch_number} | sed 's/^0*//g')" # Remove prefix "0" # Tagged Vim patch, check version.c: - is_missing="$(sed -n '/static int included_patches/,/}/p' "${NVIM_SOURCE_DIR}/src/nvim/version.c" | + is_missing="$(sed -n '/static const int included_patches/,/}/p' "${NVIM_SOURCE_DIR}/src/nvim/version.c" | grep -x -e "[[:space:]]*//[[:space:]]${patch_number} NA.*" -e "[[:space:]]*${patch_number}," >/dev/null && echo "false" || echo "true")" vim_commit="${vim_tag#v}" if (cd "${VIM_SOURCE_DIR}" && git --no-pager show --color=never --name-only "v${vim_commit}" 2>/dev/null) | grep -q ^runtime; then @@ -442,7 +461,7 @@ review_pr() { clean_files } -while getopts "hlp:g:r:s" opt; do +while getopts "hlp:P:g:r:s" opt; do case ${opt} in h) usage @@ -456,6 +475,10 @@ while getopts "hlp:g:r:s" opt; do stage_patch "${OPTARG}" exit 0 ;; + P) + stage_patch "${OPTARG}" TRY_APPLY + exit 0 + ;; g) get_vim_patch "${OPTARG}" exit 0 |