From c778311505fe89e3f32ff3027f0a41f0a2b2fd95 Mon Sep 17 00:00:00 2001 From: Björn Linse Date: Sun, 30 Apr 2017 13:08:39 +0200 Subject: generators: separate source generators from scripts --- scripts/gen_events.lua | 65 ---- scripts/gen_ui_events.lua | 183 ---------- scripts/gencharblob.lua | 48 --- scripts/gendeclarations.lua | 287 ---------------- scripts/geneval.lua | 67 ---- scripts/genex_cmds.lua | 88 ----- scripts/genmsgpack.lua | 513 ----------------------------- scripts/genoptions.lua | 190 ----------- scripts/genunicodetables.lua | 327 ------------------ src/nvim/CMakeLists.txt | 32 +- src/nvim/generators/c_grammar.lua | 51 +++ src/nvim/generators/dump_bin_array.lua | 17 + src/nvim/generators/gen_api_dispatch.lua | 458 +++++++++++++++++++++++++ src/nvim/generators/gen_api_ui_events.lua | 136 ++++++++ src/nvim/generators/gen_char_blob.lua | 48 +++ src/nvim/generators/gen_declarations.lua | 287 ++++++++++++++++ src/nvim/generators/gen_eval.lua | 67 ++++ src/nvim/generators/gen_events.lua | 65 ++++ src/nvim/generators/gen_ex_cmds.lua | 88 +++++ src/nvim/generators/gen_options.lua | 190 +++++++++++ src/nvim/generators/gen_unicode_tables.lua | 327 ++++++++++++++++++ 21 files changed, 1751 insertions(+), 1783 deletions(-) delete mode 100644 scripts/gen_events.lua delete mode 100644 scripts/gen_ui_events.lua delete mode 100644 scripts/gencharblob.lua delete mode 100755 scripts/gendeclarations.lua delete mode 100644 scripts/geneval.lua delete mode 100644 scripts/genex_cmds.lua delete mode 100644 scripts/genmsgpack.lua delete mode 100644 scripts/genoptions.lua delete mode 100644 scripts/genunicodetables.lua create mode 100644 src/nvim/generators/c_grammar.lua create mode 100644 src/nvim/generators/dump_bin_array.lua create mode 100644 src/nvim/generators/gen_api_dispatch.lua create mode 100644 src/nvim/generators/gen_api_ui_events.lua create mode 100644 src/nvim/generators/gen_char_blob.lua create mode 100755 src/nvim/generators/gen_declarations.lua create mode 100644 src/nvim/generators/gen_eval.lua create mode 100644 src/nvim/generators/gen_events.lua create mode 100644 src/nvim/generators/gen_ex_cmds.lua create mode 100644 src/nvim/generators/gen_options.lua create mode 100644 src/nvim/generators/gen_unicode_tables.lua diff --git a/scripts/gen_events.lua b/scripts/gen_events.lua deleted file mode 100644 index 75e0b3da3a..0000000000 --- a/scripts/gen_events.lua +++ /dev/null @@ -1,65 +0,0 @@ -if arg[1] == '--help' then - print('Usage: gen_events.lua src/nvim enum_file event_names_file') - os.exit(0) -end - -local nvimsrcdir = arg[1] -local fileio_enum_file = arg[2] -local names_file = arg[3] - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local auevents = require('auevents') -local events = auevents.events -local aliases = auevents.aliases - -enum_tgt = io.open(fileio_enum_file, 'w') -names_tgt = io.open(names_file, 'w') - -enum_tgt:write('typedef enum auto_event {') -names_tgt:write([[ -static const struct event_name { - size_t len; - char *name; - event_T event; -} event_names[] = {]]) - -for i, event in ipairs(events) do - if i > 1 then - comma = ',\n' - else - comma = '\n' - end - enum_tgt:write(('%s EVENT_%s = %u'):format(comma, event:upper(), i - 1)) - names_tgt:write(('%s {%u, "%s", EVENT_%s}'):format(comma, #event, event, event:upper())) -end - -for alias, event in pairs(aliases) do - names_tgt:write((',\n {%u, "%s", EVENT_%s}'):format(#alias, alias, event:upper())) -end - -names_tgt:write(',\n {0, NULL, (event_T)0}') - -enum_tgt:write('\n} event_T;\n') -names_tgt:write('\n};\n') - -enum_tgt:write(('\n#define NUM_EVENTS %u\n'):format(#events)) -names_tgt:write('\nstatic AutoPat *first_autopat[NUM_EVENTS] = {\n ') -line_len = 1 -for i = 1,((#events) - 1) do - line_len = line_len + #(' NULL,') - if line_len > 80 then - names_tgt:write('\n ') - line_len = 1 + #(' NULL,') - end - names_tgt:write(' NULL,') -end -if line_len + #(' NULL') > 80 then - names_tgt:write('\n NULL') -else - names_tgt:write(' NULL') -end -names_tgt:write('\n};\n') - -enum_tgt:close() -names_tgt:close() diff --git a/scripts/gen_ui_events.lua b/scripts/gen_ui_events.lua deleted file mode 100644 index ab5a8655fe..0000000000 --- a/scripts/gen_ui_events.lua +++ /dev/null @@ -1,183 +0,0 @@ -lpeg = require('lpeg') -mpack = require('mpack') - --- TODO: reduce copying --- lpeg grammar for building api metadata from a set of header files. It --- ignores comments and preprocessor commands and parses a very small subset --- of C prototypes with a limited set of types -P, R, S = lpeg.P, lpeg.R, lpeg.S -C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg - -any = P(1) -- (consume one character) -letter = R('az', 'AZ') + S('_$') -num = R('09') -alpha = letter + num -nl = P('\r\n') + P('\n') -not_nl = any - nl -ws = S(' \t') + nl -fill = ws ^ 0 -c_comment = P('//') * (not_nl ^ 0) -c_preproc = P('#') * (not_nl ^ 0) -typed_container = - (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')') -c_id = ( - typed_container + - (letter * (alpha ^ 0)) -) -c_void = P('void') -c_param_type = ( - ((P('Error') * fill * P('*') * fill) * Cc('error')) + - (C(c_id) * (ws ^ 1)) - ) -c_type = (C(c_void) * (ws ^ 1)) + c_param_type -c_param = Ct(c_param_type * C(c_id)) -c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0) -c_params = Ct(c_void + c_param_list) -c_proto = Ct( - Cg(c_type, 'return_type') * Cg(c_id, 'name') * - fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') * - Cg(Cc(false), 'async') * - (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) * - (fill * Cg((P('REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) * - (fill * Cg((P('REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) * - (fill * Cg((P('BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) * - fill * P(';') - ) -grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1) - - --- we need at least 4 arguments since the last two are output files -assert(#arg == 6) -input = io.open(arg[2], 'rb') -proto_output = io.open(arg[3], 'wb') -call_output = io.open(arg[4], 'wb') -remote_output = io.open(arg[5], 'wb') -bridge_output = io.open(arg[6], 'wb') - -functions = {} - -local events = grammar:match(input:read('*all')) - -function write_signature(output, ev, prefix, notype) - output:write('('..prefix) - if prefix == "" and #ev.parameters == 0 then - output:write('void') - end - for j = 1, #ev.parameters do - if j > 1 or prefix ~= '' then - output:write(', ') - end - local param = ev.parameters[j] - if not notype then - output:write(param[1]..' ') - end - output:write(param[2]) - end - output:write(')') -end - -function write_arglist(output, ev, need_copy) - output:write(' Array args = ARRAY_DICT_INIT;\n') - for j = 1, #ev.parameters do - local param = ev.parameters[j] - local kind = string.upper(param[1]) - local do_copy = need_copy and (kind == "ARRAY" or kind == "DICTIONARY" or kind == "STRING") - output:write(' ADD(args, ') - if do_copy then - output:write('copy_object(') - end - output:write(kind..'_OBJ('..param[2]..')') - if do_copy then - output:write(')') - end - output:write(');\n') - end -end - -for i = 1, #events do - ev = events[i] - assert(ev.return_type == 'void') - - if not ev.remote_only then - proto_output:write(' void (*'..ev.name..')') - write_signature(proto_output, ev, 'UI *ui') - proto_output:write(';\n') - - if not ev.remote_impl then - remote_output:write('static void remote_ui_'..ev.name) - write_signature(remote_output, ev, 'UI *ui') - remote_output:write('\n{\n') - write_arglist(remote_output, ev, true) - remote_output:write(' push_call(ui, "'..ev.name..'", args);\n') - remote_output:write('}\n\n') - end - - if not ev.bridge_impl then - - send, argv, recv, recv_argv, recv_cleanup = '', '', '', '', '' - argc = 1 - for j = 1, #ev.parameters do - local param = ev.parameters[j] - copy = 'copy_'..param[2] - if param[1] == 'String' then - send = send..' String copy_'..param[2]..' = copy_string('..param[2]..');\n' - argv = argv..', '..copy..'.data, INT2PTR('..copy..'.size)' - recv = (recv..' String '..param[2].. - ' = (String){.data = argv['..argc..'],'.. - '.size = (size_t)argv['..(argc+1)..']};\n') - recv_argv = recv_argv..', '..param[2] - recv_cleanup = recv_cleanup..' api_free_string('..param[2]..');\n' - argc = argc+2 - elseif param[1] == 'Array' then - send = send..' Array copy_'..param[2]..' = copy_array('..param[2]..');\n' - argv = argv..', '..copy..'.items, INT2PTR('..copy..'.size)' - recv = (recv..' Array '..param[2].. - ' = (Array){.items = argv['..argc..'],'.. - '.size = (size_t)argv['..(argc+1)..']};\n') - recv_argv = recv_argv..', '..param[2] - recv_cleanup = recv_cleanup..' api_free_array('..param[2]..');\n' - argc = argc+2 - elseif param[1] == 'Integer' or param[1] == 'Boolean' then - argv = argv..', INT2PTR('..param[2]..')' - recv_argv = recv_argv..', PTR2INT(argv['..argc..'])' - argc = argc+1 - else - assert(false) - end - end - bridge_output:write('static void ui_bridge_'..ev.name.. - '_event(void **argv)\n{\n') - bridge_output:write(' UI *ui = UI(argv[0]);\n') - bridge_output:write(recv) - bridge_output:write(' ui->'..ev.name..'(ui'..recv_argv..');\n') - bridge_output:write(recv_cleanup) - bridge_output:write('}\n\n') - - bridge_output:write('static void ui_bridge_'..ev.name) - write_signature(bridge_output, ev, 'UI *ui') - bridge_output:write('\n{\n') - bridge_output:write(send) - bridge_output:write(' UI_CALL(ui, '..ev.name..', '..argc..', ui'..argv..');\n}\n') - end - end - - call_output:write('void ui_call_'..ev.name) - write_signature(call_output, ev, '') - call_output:write('\n{\n') - if ev.remote_only then - write_arglist(call_output, ev, false) - call_output:write(' ui_event("'..ev.name..'", args);\n') - else - call_output:write(' UI_CALL') - write_signature(call_output, ev, ev.name, true) - call_output:write(";\n") - end - call_output:write("}\n\n") - -end - - - -proto_output:close() -call_output:close() -remote_output:close() diff --git a/scripts/gencharblob.lua b/scripts/gencharblob.lua deleted file mode 100644 index d860375e26..0000000000 --- a/scripts/gencharblob.lua +++ /dev/null @@ -1,48 +0,0 @@ -if arg[1] == '--help' then - print('Usage:') - print(' gencharblob.lua source target varname') - print('') - print('Generates C file with big uint8_t blob.') - print('Blob will be stored in a static const array named varname.') - os.exit() -end - -assert(#arg == 3) - -local source_file = arg[1] -local target_file = arg[2] -local varname = arg[3] - -source = io.open(source_file, 'r') -target = io.open(target_file, 'w') - -target:write('#include \n\n') -target:write(('static const uint8_t %s[] = {\n'):format(varname)) - -num_bytes = 0 -MAX_NUM_BYTES = 15 -- 78 / 5: maximum number of bytes on one line -target:write(' ') - -increase_num_bytes = function() - num_bytes = num_bytes + 1 - if num_bytes == MAX_NUM_BYTES then - num_bytes = 0 - target:write('\n ') - end -end - -for line in source:lines() do - for i = 1,string.len(line) do - byte = string.byte(line, i) - assert(byte ~= 0) - target:write(string.format(' %3u,', byte)) - increase_num_bytes() - end - target:write(string.format(' %3u,', string.byte('\n', 1))) - increase_num_bytes() -end - -target:write(' 0};\n') - -source:close() -target:close() diff --git a/scripts/gendeclarations.lua b/scripts/gendeclarations.lua deleted file mode 100755 index e999e53e4a..0000000000 --- a/scripts/gendeclarations.lua +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/lua - -local fname = arg[1] -local static_fname = arg[2] -local non_static_fname = arg[3] -local preproc_fname = arg[4] - - -local lpeg = require('lpeg') - -local fold = function (func, ...) - local result = nil - for i, v in ipairs({...}) do - if result == nil then - result = v - else - result = func(result, v) - end - end - return result -end - -local folder = function (func) - return function (...) - return fold(func, ...) - end -end - -local lit = lpeg.P -local set = function(...) - return lpeg.S(fold(function (a, b) return a .. b end, ...)) -end -local any_character = lpeg.P(1) -local rng = function(s, e) return lpeg.R(s .. e) end -local concat = folder(function (a, b) return a * b end) -local branch = folder(function (a, b) return a + b end) -local one_or_more = function(v) return v ^ 1 end -local two_or_more = function(v) return v ^ 2 end -local any_amount = function(v) return v ^ 0 end -local one_or_no = function(v) return v ^ -1 end -local look_behind = lpeg.B -local look_ahead = function(v) return #v end -local neg_look_ahead = function(v) return -v end -local neg_look_behind = function(v) return -look_behind(v) end - -local w = branch( - rng('a', 'z'), - rng('A', 'Z'), - lit('_') -) -local aw = branch( - w, - rng('0', '9') -) -local s = set(' ', '\n', '\t') -local raw_word = concat(w, any_amount(aw)) -local right_word = concat( - raw_word, - neg_look_ahead(aw) -) -local word = branch( - concat( - branch(lit('ArrayOf('), lit('DictionaryOf(')), -- typed container macro - one_or_more(any_character - lit(')')), - lit(')') - ), - concat( - neg_look_behind(aw), - right_word - ) -) -local inline_comment = concat( - lit('/*'), - any_amount(concat( - neg_look_ahead(lit('*/')), - any_character - )), - lit('*/') -) -local spaces = any_amount(branch( - s, - -- Comments are really handled by preprocessor, so the following is not needed - inline_comment, - concat( - lit('//'), - any_amount(concat( - neg_look_ahead(lit('\n')), - any_character - )), - lit('\n') - ), - -- Linemarker inserted by preprocessor - concat( - lit('# '), - any_amount(concat( - neg_look_ahead(lit('\n')), - any_character - )), - lit('\n') - ) -)) -local typ_part = concat( - word, - any_amount(concat( - spaces, - lit('*') - )), - spaces -) -local typ = one_or_more(typ_part) -local typ_id = two_or_more(typ_part) -local arg = typ_id -- argument name is swallowed by typ -local pattern = concat( - any_amount(branch(set(' ', '\t'), inline_comment)), - typ_id, -- return type with function name - spaces, - lit('('), - spaces, - one_or_no(branch( -- function arguments - concat( - arg, -- first argument, does not require comma - any_amount(concat( -- following arguments, start with a comma - spaces, - lit(','), - spaces, - arg, - any_amount(concat( - lit('['), - spaces, - any_amount(aw), - spaces, - lit(']') - )) - )), - one_or_no(concat( - spaces, - lit(','), - spaces, - lit('...') - )) - ), - lit('void') -- also accepts just void - )), - spaces, - lit(')'), - any_amount(concat( -- optional attributes - spaces, - lit('FUNC_'), - any_amount(aw), - one_or_no(concat( -- attribute argument - spaces, - lit('('), - any_amount(concat( - neg_look_ahead(lit(')')), - any_character - )), - lit(')') - )) - )), - look_ahead(concat( -- definition must be followed by "{" - spaces, - lit('{') - )) -) - -if fname == '--help' then - print'Usage:' - print() - print' gendeclarations.lua definitions.c static.h non-static.h preprocessor.i' - os.exit() -end - -local preproc_f = io.open(preproc_fname) -local text = preproc_f:read("*all") -preproc_f:close() - - -local header = [[ -#ifndef DEFINE_FUNC_ATTRIBUTES -# define DEFINE_FUNC_ATTRIBUTES -#endif -#include "nvim/func_attr.h" -#undef DEFINE_FUNC_ATTRIBUTES -]] - -local footer = [[ -#include "nvim/func_attr.h" -]] - -local non_static = header -local static = header - -local filepattern = '^#%a* (%d+) "([^"]-)/?([^"/]+)"' -local curfile - -local init = 0 -local curfile = nil -local neededfile = fname:match('[^/]+$') -local declline = 0 -local declendpos = 0 -local curdir = nil -local is_needed_file = false -while init ~= nil do - init = text:find('[\n;}]', init) - if init == nil then - break - end - local init_is_nl = text:sub(init, init) == '\n' - init = init + 1 - if init_is_nl and is_needed_file then - declline = declline + 1 - end - if init_is_nl and text:sub(init, init) == '#' then - local line, dir, file = text:match(filepattern, init) - if file ~= nil then - curfile = file - is_needed_file = (curfile == neededfile) - declline = tonumber(line) - 1 - local curdir_start = dir:find('src/nvim/') - if curdir_start ~= nil then - curdir = dir:sub(curdir_start + #('src/nvim/')) - else - curdir = dir - end - else - declline = declline - 1 - end - elseif init < declendpos then - -- Skipping over declaration - elseif is_needed_file then - s = init - e = pattern:match(text, init) - if e ~= nil then - local declaration = text:sub(s, e - 1) - -- Comments are really handled by preprocessor, so the following is not - -- needed - declaration = declaration:gsub('/%*.-%*/', '') - declaration = declaration:gsub('//.-\n', '\n') - - declaration = declaration:gsub('# .-\n', '') - - declaration = declaration:gsub('\n', ' ') - declaration = declaration:gsub('%s+', ' ') - declaration = declaration:gsub(' ?%( ?', '(') - -- declaration = declaration:gsub(' ?%) ?', ')') - declaration = declaration:gsub(' ?, ?', ', ') - declaration = declaration:gsub(' ?(%*+) ?', ' %1') - declaration = declaration:gsub(' ?(FUNC_ATTR_)', ' %1') - declaration = declaration:gsub(' $', '') - declaration = declaration:gsub('^ ', '') - declaration = declaration .. ';' - declaration = declaration .. (' // %s/%s:%u'):format( - curdir, curfile, declline) - declaration = declaration .. '\n' - if declaration:sub(1, 6) == 'static' then - static = static .. declaration - else - non_static = non_static .. declaration - end - declendpos = e - end - end -end - -non_static = non_static .. footer -static = static .. footer - -local F -F = io.open(static_fname, 'w') -F:write(static) -F:close() - --- Before generating the non-static headers, check if the current file(if --- exists) is different from the new one. If they are the same, we won't touch --- the current version to avoid triggering an unnecessary rebuilds of modules --- that depend on this one -F = io.open(non_static_fname, 'r') -if F ~= nil then - if F:read('*a') == non_static then - os.exit(0) - end - io.close(F) -end - -F = io.open(non_static_fname, 'w') -F:write(non_static) -F:close() diff --git a/scripts/geneval.lua b/scripts/geneval.lua deleted file mode 100644 index b17ecb1a0c..0000000000 --- a/scripts/geneval.lua +++ /dev/null @@ -1,67 +0,0 @@ -mpack = require('mpack') - -local nvimsrcdir = arg[1] -local autodir = arg[2] -local metadata_file = arg[3] -local funcs_file = arg[4] - -if nvimsrcdir == '--help' then - print([[ -Usage: - lua geneval.lua src/nvim build/src/nvim/auto - -Will generate build/src/nvim/auto/funcs.generated.h with definition of functions -static const array. -]]) - os.exit(0) -end - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local funcsfname = autodir .. '/funcs.generated.h' - -local gperfpipe = io.open(funcsfname .. '.gperf', 'wb') - -local funcs = require('eval').funcs -local metadata = mpack.unpack(io.open(arg[3], 'rb'):read("*all")) -for i,fun in ipairs(metadata) do - if not fun.noeval then - funcs[fun.name] = { - args=#fun.parameters, - func='api_wrapper', - data='&handle_'..fun.name, - } - end -end - -local funcsdata = io.open(funcs_file, 'w') -funcsdata:write(mpack.pack(funcs)) -funcsdata:close() - -gperfpipe:write([[ -%language=ANSI-C -%global-table -%readonly-tables -%define initializer-suffix ,0,0,NULL,NULL -%define word-array-name functions -%define hash-function-name hash_internal_func_gperf -%define lookup-function-name find_internal_func_gperf -%omit-struct-type -%struct-type -VimLFuncDef; -%% -]]) - -for name, def in pairs(funcs) do - args = def.args or 0 - if type(args) == 'number' then - args = {args, args} - elseif #args == 1 then - args[2] = 'MAX_FUNC_ARGS' - end - func = def.func or ('f_' .. name) - data = def.data or "NULL" - gperfpipe:write(('%s, %s, %s, &%s, (FunPtr)%s\n') - :format(name, args[1], args[2], func, data)) -end -gperfpipe:close() diff --git a/scripts/genex_cmds.lua b/scripts/genex_cmds.lua deleted file mode 100644 index cb566d46ca..0000000000 --- a/scripts/genex_cmds.lua +++ /dev/null @@ -1,88 +0,0 @@ -local nvimsrcdir = arg[1] -local includedir = arg[2] -local autodir = arg[3] - -if nvimsrcdir == '--help' then - print ([[ -Usage: - lua genex_cmds.lua src/nvim build/include build/src/nvim/auto - -Will generate files build/include/ex_cmds_enum.generated.h with cmdidx_T -enum and build/src/nvim/auto/ex_cmds_defs.generated.h with main Ex commands -definitions. -]]) - os.exit(0) -end - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local enumfname = includedir .. '/ex_cmds_enum.generated.h' -local defsfname = autodir .. '/ex_cmds_defs.generated.h' - -local enumfile = io.open(enumfname, 'w') -local defsfile = io.open(defsfname, 'w') - -local defs = require('ex_cmds') -local lastchar = nil - -local i -local cmd -local first = true -local prevfirstchar = nil - -local byte_a = string.byte('a') -local byte_z = string.byte('z') - -local cmdidxs = string.format([[ -static const cmdidx_T cmdidxs[%u] = { -]], byte_z - byte_a + 2) - -enumfile:write([[ -typedef enum CMD_index { -]]) -defsfile:write(string.format([[ -static CommandDefinition cmdnames[%u] = { -]], #defs)) -for i, cmd in ipairs(defs) do - local enumname = cmd.enum or ('CMD_' .. cmd.command) - firstchar = string.byte(cmd.command) - if firstchar ~= prevfirstchar then - if (not prevfirstchar - or (byte_a <= firstchar and firstchar <= byte_z) - or (byte_a <= prevfirstchar and prevfirstchar <= byte_z)) then - if not first then - cmdidxs = cmdidxs .. ',\n' - end - cmdidxs = cmdidxs .. ' ' .. enumname - end - prevfirstchar = firstchar - end - if first then - first = false - else - defsfile:write(',\n') - end - enumfile:write(' ' .. enumname .. ',\n') - defsfile:write(string.format([[ - [%s] = { - .cmd_name = (char_u *) "%s", - .cmd_func = (ex_func_T)&%s, - .cmd_argt = %uL, - .cmd_addr_type = %i - }]], enumname, cmd.command, cmd.func, cmd.flags, cmd.addr_type)) -end -defsfile:write([[ - -}; -]]) -enumfile:write([[ - CMD_SIZE, - CMD_USER = -1, - CMD_USER_BUF = -2 -} cmdidx_T; -]]) -cmdidxs = cmdidxs .. [[ - -}; -]] -defsfile:write(cmdidxs) diff --git a/scripts/genmsgpack.lua b/scripts/genmsgpack.lua deleted file mode 100644 index c16a844586..0000000000 --- a/scripts/genmsgpack.lua +++ /dev/null @@ -1,513 +0,0 @@ -lpeg = require('lpeg') -mpack = require('mpack') - --- lpeg grammar for building api metadata from a set of header files. It --- ignores comments and preprocessor commands and parses a very small subset --- of C prototypes with a limited set of types -P, R, S = lpeg.P, lpeg.R, lpeg.S -C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg - -any = P(1) -- (consume one character) -letter = R('az', 'AZ') + S('_$') -num = R('09') -alpha = letter + num -nl = P('\r\n') + P('\n') -not_nl = any - nl -ws = S(' \t') + nl -fill = ws ^ 0 -c_comment = P('//') * (not_nl ^ 0) -c_preproc = P('#') * (not_nl ^ 0) -typed_container = - (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')') -c_id = ( - typed_container + - (letter * (alpha ^ 0)) -) -c_void = P('void') -c_param_type = ( - ((P('Error') * fill * P('*') * fill) * Cc('error')) + - (C(c_id) * (ws ^ 1)) - ) -c_type = (C(c_void) * (ws ^ 1)) + c_param_type -c_param = Ct(c_param_type * C(c_id)) -c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0) -c_params = Ct(c_void + c_param_list) -c_proto = Ct( - Cg(c_type, 'return_type') * Cg(c_id, 'name') * - fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') * - Cg(Cc(false), 'async') * - (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) * - (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(num ^ 1)) * P(')'), - 'deprecated_since') ^ -1) * - (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) * - (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) * - (fill * Cg((P('FUNC_API_NOEVAL') * Cc(true)), 'noeval') ^ -1) * - fill * P(';') - ) -grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1) - --- we need at least 4 arguments since the last two are output files -if arg[1] == '--help' then - print('Usage: genmsgpack.lua args') - print('Args: 1: source directory') - print(' 2: dispatch output file (dispatch_wrappers.generated.h)') - print(' 3: functions metadata output file (funcs_metadata.generated.h)') - print(' 4: API metadata output file (api_metadata.mpack)') - print(' 5: lua C bindings output file (msgpack_lua_c_bindings.generated.c)') - print(' rest: C files where API functions are defined') -end -assert(#arg >= 4) -functions = {} - -local nvimsrcdir = arg[1] -package.path = nvimsrcdir .. '/?.lua;' .. package.path - --- names of all headers relative to the source root (for inclusion in the --- generated file) -headers = {} - --- output h file with generated dispatch functions -dispatch_outputf = arg[2] --- output h file with packed metadata -funcs_metadata_outputf = arg[3] --- output metadata mpack file, for use by other build scripts -mpack_outputf = arg[4] -lua_c_bindings_outputf = arg[5] - --- set of function names, used to detect duplicates -function_names = {} - --- read each input file, parse and append to the api metadata -for i = 6, #arg do - local full_path = arg[i] - local parts = {} - for part in string.gmatch(full_path, '[^/]+') do - parts[#parts + 1] = part - end - headers[#headers + 1] = parts[#parts - 1]..'/'..parts[#parts] - - local input = io.open(full_path, 'rb') - - local tmp = grammar:match(input:read('*all')) - for i = 1, #tmp do - local fn = tmp[i] - if not fn.noexport then - functions[#functions + 1] = tmp[i] - function_names[fn.name] = true - if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then - -- this function should receive the channel id - fn.receives_channel_id = true - -- remove the parameter since it won't be passed by the api client - table.remove(fn.parameters, 1) - end - if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then - -- function can fail if the last parameter type is 'Error' - fn.can_fail = true - -- remove the error parameter, msgpack has it's own special field - -- for specifying errors - fn.parameters[#fn.parameters] = nil - end - end - end - input:close() -end - -local function shallowcopy(orig) - local copy = {} - for orig_key, orig_value in pairs(orig) do - copy[orig_key] = orig_value - end - return copy -end - -local function startswith(String,Start) - return string.sub(String,1,string.len(Start))==Start -end - --- Export functions under older deprecated names. --- These will be removed eventually. -local deprecated_aliases = require("api.dispatch_deprecated") -for i,f in ipairs(shallowcopy(functions)) do - local ismethod = false - if startswith(f.name, "nvim_") then - if startswith(f.name, "nvim__") then - f.since = -1 - elseif f.since == nil then - print("Function "..f.name.." lacks since field.\n") - os.exit(1) - end - f.since = tonumber(f.since) - if f.deprecated_since ~= nil then - f.deprecated_since = tonumber(f.deprecated_since) - end - - if startswith(f.name, "nvim_buf_") then - ismethod = true - elseif startswith(f.name, "nvim_win_") then - ismethod = true - elseif startswith(f.name, "nvim_tabpage_") then - ismethod = true - end - else - f.noeval = true - f.since = 0 - f.deprecated_since = 1 - end - f.method = ismethod - local newname = deprecated_aliases[f.name] - if newname ~= nil then - if function_names[newname] then - -- duplicate - print("Function "..f.name.." has deprecated alias\n" - ..newname.." which has a separate implementation.\n".. - "Please remove it from src/nvim/api/dispatch_deprecated.lua") - os.exit(1) - end - local newf = shallowcopy(f) - newf.name = newname - if newname == "ui_try_resize" then - -- The return type was incorrectly set to Object in 0.1.5. - -- Keep it that way for clients that rely on this. - newf.return_type = "Object" - end - newf.impl_name = f.name - newf.noeval = true - newf.since = 0 - newf.deprecated_since = 1 - functions[#functions+1] = newf - end -end - --- don't expose internal attributes like "impl_name" in public metadata -exported_attributes = {'name', 'parameters', 'return_type', 'method', - 'since', 'deprecated_since'} -exported_functions = {} -for _,f in ipairs(functions) do - if not startswith(f.name, "nvim__") then - local f_exported = {} - for _,attr in ipairs(exported_attributes) do - f_exported[attr] = f[attr] - end - exported_functions[#exported_functions+1] = f_exported - end -end - - -funcs_metadata_output = io.open(funcs_metadata_outputf, 'wb') -funcs_metadata_output:write([[ -static const uint8_t funcs_metadata[] = { -]]) - --- serialize the API metadata using msgpack and embed into the resulting --- binary for easy querying by clients -packed_exported_functions = mpack.pack(exported_functions) -for i = 1, #packed_exported_functions do - funcs_metadata_output:write(string.byte(packed_exported_functions, i)..', ') - if i % 10 == 0 then - funcs_metadata_output:write('\n ') - end -end -funcs_metadata_output:write([[ -}; -]]) -funcs_metadata_output:close() - --- start building the dispatch wrapper output -output = io.open(dispatch_outputf, 'wb') - -local function real_type(type) - local rv = type - if typed_container:match(rv) then - if rv:match('Array') then - rv = 'Array' - else - rv = 'Dictionary' - end - end - return rv -end - -local function attr_name(rt) - if rt == 'Float' then - return 'floating' - else - return rt:lower() - end -end - --- start the handler functions. Visit each function metadata to build the --- handler function with code generated for validating arguments and calling to --- the real API. -for i = 1, #functions do - local fn = functions[i] - if fn.impl_name == nil then - local args = {} - - output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)') - output:write('\n{') - output:write('\n Object ret = NIL;') - -- Declare/initialize variables that will hold converted arguments - for j = 1, #fn.parameters do - local param = fn.parameters[j] - local converted = 'arg_'..j - output:write('\n '..param[1]..' '..converted..';') - end - output:write('\n') - output:write('\n if (args.size != '..#fn.parameters..') {') - output:write('\n api_set_error(error, kErrorTypeException, "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);') - output:write('\n goto cleanup;') - output:write('\n }\n') - - -- Validation/conversion for each argument - for j = 1, #fn.parameters do - local converted, convert_arg, param, arg - param = fn.parameters[j] - converted = 'arg_'..j - local rt = real_type(param[1]) - if rt ~= 'Object' then - if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then - -- Buffer, Window, and Tabpage have a specific type, but are stored in integer - output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..' && args.items['..(j - 1)..'].data.integer >= 0) {') - output:write('\n '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;') - else - output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {') - output:write('\n '..converted..' = args.items['..(j - 1)..'].data.'..attr_name(rt)..';') - end - if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then - -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages - output:write('\n } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer >= 0) {') - output:write('\n '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;') - end - output:write('\n } else {') - output:write('\n api_set_error(error, kErrorTypeException, "Wrong type for argument '..j..', expecting '..param[1]..'");') - output:write('\n goto cleanup;') - output:write('\n }\n') - else - output:write('\n '..converted..' = args.items['..(j - 1)..'];\n') - end - - args[#args + 1] = converted - end - - -- function call - local call_args = table.concat(args, ', ') - output:write('\n ') - if fn.return_type ~= 'void' then - -- has a return value, prefix the call with a declaration - output:write(fn.return_type..' rv = ') - end - - -- write the function name and the opening parenthesis - output:write(fn.name..'(') - - if fn.receives_channel_id then - -- if the function receives the channel id, pass it as first argument - if #args > 0 or fn.can_fail then - output:write('channel_id, '..call_args) - else - output:write('channel_id') - end - else - output:write(call_args) - end - - if fn.can_fail then - -- if the function can fail, also pass a pointer to the local error object - if #args > 0 then - output:write(', error);\n') - else - output:write('error);\n') - end - -- and check for the error - output:write('\n if (ERROR_SET(error)) {') - output:write('\n goto cleanup;') - output:write('\n }\n') - else - output:write(');\n') - end - - if fn.return_type ~= 'void' then - output:write('\n ret = '..string.upper(real_type(fn.return_type))..'_OBJ(rv);') - end - output:write('\n\ncleanup:'); - - output:write('\n return ret;\n}\n\n'); - end -end - --- Generate a function that initializes method names with handler functions -output:write([[ -void msgpack_rpc_init_method_table(void) -{ - methods = map_new(String, MsgpackRpcRequestHandler)(); - -]]) - -for i = 1, #functions do - local fn = functions[i] - output:write(' msgpack_rpc_add_method_handler('.. - '(String) {.data = "'..fn.name..'", '.. - '.size = sizeof("'..fn.name..'") - 1}, '.. - '(MsgpackRpcRequestHandler) {.fn = handle_'.. (fn.impl_name or fn.name).. - ', .async = '..tostring(fn.async)..'});\n') - -end - -output:write('\n}\n\n') -output:close() - -mpack_output = io.open(mpack_outputf, 'wb') -mpack_output:write(mpack.pack(functions)) -mpack_output:close() - -local function include_headers(output, headers) - for i = 1, #headers do - if headers[i]:sub(-12) ~= '.generated.h' then - output:write('\n#include "nvim/'..headers[i]..'"') - end - end -end - -local function write_shifted_output(output, str) - str = str:gsub('\n ', '\n') - str = str:gsub('^ ', '') - str = str:gsub(' +$', '') - output:write(str) -end - --- start building lua output -output = io.open(lua_c_bindings_outputf, 'wb') - -output:write([[ -#include -#include -#include - -#include "nvim/func_attr.h" -#include "nvim/api/private/defs.h" -#include "nvim/api/private/helpers.h" -#include "nvim/lua/converter.h" -]]) -include_headers(output, headers) -output:write('\n') - -lua_c_functions = {} - -local function process_function(fn) - lua_c_function_name = ('nlua_msgpack_%s'):format(fn.name) - write_shifted_output(output, string.format([[ - - static int %s(lua_State *lstate) - { - Error err = ERROR_INIT; - if (lua_gettop(lstate) != %i) { - api_set_error(&err, kErrorTypeValidation, "Expected %i argument%s"); - goto exit_0; - } - ]], lua_c_function_name, #fn.parameters, #fn.parameters, - (#fn.parameters == 1) and '' or 's')) - lua_c_functions[#lua_c_functions + 1] = { - binding=lua_c_function_name, - api=fn.name - } - local cparams = '' - local free_code = {} - for j = #fn.parameters,1,-1 do - param = fn.parameters[j] - cparam = string.format('arg%u', j) - param_type = real_type(param[1]) - lc_param_type = param_type:lower() - write_shifted_output(output, string.format([[ - const %s %s = nlua_pop_%s(lstate, &err); - - if (ERROR_SET(&err)) { - goto exit_%u; - } - ]], param[1], cparam, param_type, #fn.parameters - j)) - free_code[#free_code + 1] = ('api_free_%s(%s);'):format( - lc_param_type, cparam) - cparams = cparam .. ', ' .. cparams - end - if fn.receives_channel_id then - cparams = 'LUA_INTERNAL_CALL, ' .. cparams - end - if fn.can_fail then - cparams = cparams .. '&err' - else - cparams = cparams:gsub(', $', '') - end - local free_at_exit_code = '' - for i = 1, #free_code do - local rev_i = #free_code - i + 1 - local code = free_code[rev_i] - if i == 1 then - free_at_exit_code = free_at_exit_code .. ('\n %s'):format(code) - else - free_at_exit_code = free_at_exit_code .. ('\n exit_%u:\n %s'):format( - rev_i, code) - end - end - local err_throw_code = [[ - - exit_0: - if (ERROR_SET(&err)) { - luaL_where(lstate, 1); - lua_pushstring(lstate, err.msg); - api_clear_error(&err); - lua_concat(lstate, 2); - return lua_error(lstate); - } - ]] - if fn.return_type ~= 'void' then - if fn.return_type:match('^ArrayOf') then - return_type = 'Array' - else - return_type = fn.return_type - end - write_shifted_output(output, string.format([[ - const %s ret = %s(%s); - nlua_push_%s(lstate, ret); - api_free_%s(ret); - %s - %s - return 1; - ]], fn.return_type, fn.name, cparams, return_type, return_type:lower(), - free_at_exit_code, err_throw_code)) - else - write_shifted_output(output, string.format([[ - %s(%s); - %s - %s - return 0; - ]], fn.name, cparams, free_at_exit_code, err_throw_code)) - end - write_shifted_output(output, [[ - } - ]]) -end - -for _, fn in ipairs(functions) do - if not fn.noeval or fn.name:sub(1, 4) == '_vim' then - process_function(fn) - end -end - -output:write(string.format([[ -void nlua_add_api_functions(lua_State *lstate) - FUNC_ATTR_NONNULL_ALL -{ - lua_createtable(lstate, 0, %u); -]], #lua_c_functions)) -for _, func in ipairs(lua_c_functions) do - output:write(string.format([[ - - lua_pushcfunction(lstate, &%s); - lua_setfield(lstate, -2, "%s");]], func.binding, func.api)) -end -output:write([[ - - lua_setfield(lstate, -2, "api"); -} -]]) - -output:close() diff --git a/scripts/genoptions.lua b/scripts/genoptions.lua deleted file mode 100644 index 9d7f235a3b..0000000000 --- a/scripts/genoptions.lua +++ /dev/null @@ -1,190 +0,0 @@ -if arg[1] == '--help' then - print('Usage: genoptions.lua src/nvim options_file') - os.exit(0) -end - -local nvimsrcdir = arg[1] -local options_file = arg[2] - -package.path = nvimsrcdir .. '/?.lua;' .. package.path - -local opt_fd = io.open(options_file, 'w') - -local w = function(s) - if s:match('^ %.') then - opt_fd:write(s .. ',\n') - else - opt_fd:write(s .. '\n') - end -end - -local options = require('options') - -cstr = options.cstr - -local type_flags={ - bool='P_BOOL', - number='P_NUM', - string='P_STRING', -} - -local redraw_flags={ - statuslines='P_RSTAT', - current_window='P_RWIN', - current_window_only='P_RWINONLY', - current_buffer='P_RBUF', - all_windows='P_RALL', - everything='P_RCLR', - curswant='P_CURSWANT', -} - -local list_flags={ - comma='P_COMMA', - onecomma='P_ONECOMMA', - flags='P_FLAGLIST', - flagscomma='P_COMMA|P_FLAGLIST', -} - -local get_flags = function(o) - local ret = {type_flags[o.type]} - local add_flag = function(f) - ret[1] = ret[1] .. '|' .. f - end - if o.list then - add_flag(list_flags[o.list]) - end - if o.redraw then - for _, r_flag in ipairs(o.redraw) do - add_flag(redraw_flags[r_flag]) - end - end - if o.expand then - add_flag('P_EXPAND') - if o.expand == 'nodefault' then - add_flag('P_NO_DEF_EXP') - end - end - for _, flag_desc in ipairs({ - {'alloced'}, - {'nodefault'}, - {'no_mkrc'}, - {'vi_def'}, - {'vim'}, - {'secure'}, - {'gettext'}, - {'noglob'}, - {'normal_fname_chars', 'P_NFNAME'}, - {'pri_mkrc'}, - {'deny_in_modelines', 'P_NO_ML'}, - {'deny_duplicates', 'P_NODUP'}, - }) do - local key_name = flag_desc[1] - local def_name = flag_desc[2] or ('P_' .. key_name:upper()) - if o[key_name] then - add_flag(def_name) - end - end - return ret[1] -end - -local get_cond -get_cond = function(c, base_string) - local cond_string = base_string or '#if ' - if type(c) == 'table' then - cond_string = cond_string .. get_cond(c[1], '') - for i, subc in ipairs(c) do - if i > 1 then - cond_string = cond_string .. ' && ' .. get_cond(subc, '') - end - end - elseif c:sub(1, 1) == '!' then - cond_string = cond_string .. '!defined(' .. c:sub(2) .. ')' - else - cond_string = cond_string .. 'defined(' .. c .. ')' - end - return cond_string -end - -value_dumpers = { - ['function']=function(v) return v() end, - string=cstr, - boolean=function(v) return v and 'true' or 'false' end, - number=function(v) return ('%iL'):format(v) end, - ['nil']=function(v) return '0L' end, -} - -local get_value = function(v) - return '(char_u *) ' .. value_dumpers[type(v)](v) -end - -local get_defaults = function(d) - return '{' .. get_value(d.vi) .. ', ' .. get_value(d.vim) .. '}' -end - -local defines = {} - -local dump_option = function(i, o) - w(' [' .. ('%u'):format(i - 1) .. ']={') - w(' .fullname=' .. cstr(o.full_name)) - if o.abbreviation then - w(' .shortname=' .. cstr(o.abbreviation)) - end - w(' .flags=' .. get_flags(o)) - if o.enable_if then - w(get_cond(o.enable_if)) - end - if o.varname then - w(' .var=(char_u *)&' .. o.varname) - elseif #o.scope == 1 and o.scope[1] == 'window' then - w(' .var=VAR_WIN') - end - if o.enable_if then - w('#endif') - end - if #o.scope == 1 and o.scope[1] == 'global' then - w(' .indir=PV_NONE') - else - assert (#o.scope == 1 or #o.scope == 2) - assert (#o.scope == 1 or o.scope[1] == 'global') - local min_scope = o.scope[#o.scope] - local varname = o.pv_name or o.varname or ( - 'p_' .. (o.abbreviation or o.full_name)) - local pv_name = ( - 'OPT_' .. min_scope:sub(1, 3):upper() .. '(' .. ( - min_scope:sub(1, 1):upper() .. 'V_' .. varname:sub(3):upper() - ) .. ')' - ) - if #o.scope == 2 then - pv_name = 'OPT_BOTH(' .. pv_name .. ')' - end - defines['PV_' .. varname:sub(3):upper()] = pv_name - w(' .indir=' .. pv_name) - end - if o.defaults then - if o.defaults.condition then - w(get_cond(o.defaults.condition)) - end - w(' .def_val=' .. get_defaults(o.defaults.if_true)) - if o.defaults.condition then - if o.defaults.if_false then - w('#else') - w(' .def_val=' .. get_defaults(o.defaults.if_false)) - end - w('#endif') - end - end - w(' },') -end - -w('static vimoption_T options[] = {') -for i, o in ipairs(options.options) do - dump_option(i, o) -end -w(' [' .. ('%u'):format(#options.options) .. ']={.fullname=NULL}') -w('};') -w('') - -for k, v in pairs(defines) do - w('#define ' .. k .. ' ' .. v) -end -opt_fd:close() diff --git a/scripts/genunicodetables.lua b/scripts/genunicodetables.lua deleted file mode 100644 index 66430ba26e..0000000000 --- a/scripts/genunicodetables.lua +++ /dev/null @@ -1,327 +0,0 @@ --- Script creates the following tables in unicode_tables.generated.h: --- --- 1. doublewidth and ambiguous tables: sorted list of non-overlapping closed --- intervals. Codepoints in these intervals have double (W or F) or ambiguous --- (A) east asian width respectively. --- 2. combining table: same as the above, but characters inside are combining --- characters (i.e. have general categories equal to Mn, Mc or Me). --- 3. foldCase, toLower and toUpper tables used to convert characters to --- folded/lower/upper variants. In these tables first two values are --- character ranges: like in previous tables they are sorted and must be --- non-overlapping. Third value means step inside the range: e.g. if it is --- 2 then interval applies only to first, third, fifth, … character in range. --- Fourth value is number that should be added to the codepoint to yield --- folded/lower/upper codepoint. --- 4. emoji_width and emoji_all tables: sorted lists of non-overlapping closed --- intervals of Emoji characters. emoji_width contains all the characters --- which don't have ambiguous or double width, and emoji_all has all Emojis. -if arg[1] == '--help' then - print('Usage:') - print(' genunicodetables.lua unicode/ unicode_tables.generated.h') - os.exit(0) -end - -local basedir = arg[1] -local pathsep = package.config:sub(1, 1) -local get_path = function(fname) - return basedir .. pathsep .. fname -end - -local unicodedata_fname = get_path('UnicodeData.txt') -local casefolding_fname = get_path('CaseFolding.txt') -local eastasianwidth_fname = get_path('EastAsianWidth.txt') -local emoji_fname = get_path('emoji-data.txt') - -local utf_tables_fname = arg[2] - -local split_on_semicolons = function(s) - local ret = {} - local idx = 1 - while idx <= #s + 1 do - item = s:match('^[^;]*', idx) - idx = idx + #item + 1 - if idx <= #s + 1 then - assert(s:sub(idx - 1, idx - 1) == ';') - end - item = item:gsub('^%s*', '') - item = item:gsub('%s*$', '') - table.insert(ret, item) - end - return ret -end - -local fp_lines_to_lists = function(fp, n, has_comments) - local ret = {} - local line - local i = 0 - while true do - i = i + 1 - line = fp:read('*l') - if not line then - break - end - if (not has_comments - or (line:sub(1, 1) ~= '#' and not line:match('^%s*$'))) then - local l = split_on_semicolons(line) - if #l ~= n then - io.stderr:write(('Found %s items in line %u, expected %u\n'):format( - #l, i, n)) - io.stderr:write('Line: ' .. line .. '\n') - return nil - end - table.insert(ret, l) - end - end - return ret -end - -local parse_data_to_props = function(ud_fp) - return fp_lines_to_lists(ud_fp, 15, false) -end - -local parse_fold_props = function(cf_fp) - return fp_lines_to_lists(cf_fp, 4, true) -end - -local parse_width_props = function(eaw_fp) - return fp_lines_to_lists(eaw_fp, 2, true) -end - -local parse_emoji_props = function(emoji_fp) - return fp_lines_to_lists(emoji_fp, 2, true) -end - -local make_range = function(start, end_, step, add) - if step and add then - return (' {0x%x, 0x%x, %d, %d},\n'):format( - start, end_, step == 0 and -1 or step, add) - else - return (' {0x%04x, 0x%04x},\n'):format(start, end_) - end -end - -local build_convert_table = function(ut_fp, props, cond_func, nl_index, - table_name) - ut_fp:write('static const convertStruct ' .. table_name .. '[] = {\n') - local start = -1 - local end_ = -1 - local step = 0 - local add = -1 - for _, p in ipairs(props) do - if cond_func(p) then - local n = tonumber(p[1], 16) - local nl = tonumber(p[nl_index], 16) - if start >= 0 and add == (nl - n) and (step == 0 or n - end_ == step) then - -- Continue with the same range. - step = n - end_ - end_ = n - else - if start >= 0 then - -- Produce previous range. - ut_fp:write(make_range(start, end_, step, add)) - end - start = n - end_ = n - step = 0 - add = nl - n - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_, step, add)) - end - ut_fp:write('};\n') -end - -local build_case_table = function(ut_fp, dataprops, table_name, index) - local cond_func = function(p) - return p[index] ~= '' - end - return build_convert_table(ut_fp, dataprops, cond_func, index, - 'to' .. table_name) -end - -local build_fold_table = function(ut_fp, foldprops) - local cond_func = function(p) - return (p[2] == 'C' or p[2] == 'S') - end - return build_convert_table(ut_fp, foldprops, cond_func, 3, 'foldCase') -end - -local build_combining_table = function(ut_fp, dataprops) - ut_fp:write('static const struct interval combining[] = {\n') - local start = -1 - local end_ = -1 - for _, p in ipairs(dataprops) do - if (({Mn=true, Mc=true, Me=true})[p[3]]) then - local n = tonumber(p[1], 16) - if start >= 0 and end_ + 1 == n then - -- Continue with the same range. - end_ = n - else - if start >= 0 then - -- Produce previous range. - ut_fp:write(make_range(start, end_)) - end - start = n - end_ = n - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_)) - end - ut_fp:write('};\n') -end - -local build_width_table = function(ut_fp, dataprops, widthprops, widths, - table_name) - ut_fp:write('static const struct interval ' .. table_name .. '[] = {\n') - local start = -1 - local end_ = -1 - local dataidx = 1 - local ret = {} - for _, p in ipairs(widthprops) do - if widths[p[2]:sub(1, 1)] then - local rng_start, rng_end = p[1]:find('%.%.') - local n, n_last - if rng_start then - -- It is a range. We don’t check for composing char then. - n = tonumber(p[1]:sub(1, rng_start - 1), 16) - n_last = tonumber(p[1]:sub(rng_end + 1), 16) - else - n = tonumber(p[1], 16) - n_last = n - end - local dn - while true do - dn = tonumber(dataprops[dataidx][1], 16) - if dn >= n then - break - end - dataidx = dataidx + 1 - end - if dn ~= n and n_last == n then - io.stderr:write('Cannot find character ' .. n .. ' in data table.\n') - end - -- Only use the char when it’s not a composing char. - -- But use all chars from a range. - local dp = dataprops[dataidx] - if (n_last > n) or (not (({Mn=true, Mc=true, Me=true})[dp[3]])) then - if start >= 0 and end_ + 1 == n then - -- Continue with the same range. - else - if start >= 0 then - ut_fp:write(make_range(start, end_)) - table.insert(ret, {start, end_}) - end - start = n - end - end_ = n_last - end - end - end - if start >= 0 then - ut_fp:write(make_range(start, end_)) - table.insert(ret, {start, end_}) - end - ut_fp:write('};\n') - return ret -end - -local build_emoji_table = function(ut_fp, emojiprops, doublewidth, ambiwidth) - local emojiwidth = {} - local emoji = {} - for _, p in ipairs(emojiprops) do - if p[2]:match('Emoji%s+#') then - local rng_start, rng_end = p[1]:find('%.%.') - if rng_start then - n = tonumber(p[1]:sub(1, rng_start - 1), 16) - n_last = tonumber(p[1]:sub(rng_end + 1), 16) - else - n = tonumber(p[1], 16) - n_last = n - end - if #emoji > 0 and n - 1 == emoji[#emoji][2] then - emoji[#emoji][2] = n_last - else - table.insert(emoji, { n, n_last }) - end - - -- Characters below 1F000 may be considered single width traditionally, - -- making them double width causes problems. - if n >= 0x1f000 then - -- exclude characters that are in the ambiguous/doublewidth table - for _, ambi in ipairs(ambiwidth) do - if n >= ambi[1] and n <= ambi[2] then - n = ambi[2] + 1 - end - if n_last >= ambi[1] and n_last <= ambi[2] then - n_last = ambi[1] - 1 - end - end - for _, double in ipairs(doublewidth) do - if n >= double[1] and n <= double[2] then - n = double[2] + 1 - end - if n_last >= double[1] and n_last <= double[2] then - n_last = double[1] - 1 - end - end - - if n <= n_last then - if #emojiwidth > 0 and n - 1 == emojiwidth[#emojiwidth][2] then - emojiwidth[#emojiwidth][2] = n_last - else - table.insert(emojiwidth, { n, n_last }) - end - end - end - end - end - - ut_fp:write('static const struct interval emoji_all[] = {\n') - for _, p in ipairs(emoji) do - ut_fp:write(make_range(p[1], p[2])) - end - ut_fp:write('};\n') - - ut_fp:write('static const struct interval emoji_width[] = {\n') - for _, p in ipairs(emojiwidth) do - ut_fp:write(make_range(p[1], p[2])) - end - ut_fp:write('};\n') -end - -local ud_fp = io.open(unicodedata_fname, 'r') -local dataprops = parse_data_to_props(ud_fp) -ud_fp:close() - -local ut_fp = io.open(utf_tables_fname, 'w') - -build_case_table(ut_fp, dataprops, 'Lower', 14) -build_case_table(ut_fp, dataprops, 'Upper', 13) -build_combining_table(ut_fp, dataprops) - -local cf_fp = io.open(casefolding_fname, 'r') -local foldprops = parse_fold_props(cf_fp) -cf_fp:close() - -build_fold_table(ut_fp, foldprops) - -local eaw_fp = io.open(eastasianwidth_fname, 'r') -local widthprops = parse_width_props(eaw_fp) -eaw_fp:close() - -local doublewidth = build_width_table(ut_fp, dataprops, widthprops, - {W=true, F=true}, 'doublewidth') -local ambiwidth = build_width_table(ut_fp, dataprops, widthprops, - {A=true}, 'ambiguous') - -local emoji_fp = io.open(emoji_fname, 'r') -local emojiprops = parse_emoji_props(emoji_fp) -emoji_fp:close() - -build_emoji_table(ut_fp, emojiprops, doublewidth, ambiwidth) - -ut_fp:close() diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index 94d1c00de9..3e0dadaa81 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -11,13 +11,14 @@ endif() endif() set(TOUCHES_DIR ${PROJECT_BINARY_DIR}/touches) +set(GENERATOR_DIR ${CMAKE_CURRENT_LIST_DIR}/generators) set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto) -set(MSGPACK_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genmsgpack.lua) -set(API_UI_EVENTS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gen_api_ui_events.lua) +set(API_DISPATCH_GENERATOR ${GENERATOR_DIR}/gen_api_dispatch.lua) +set(API_UI_EVENTS_GENERATOR ${GENERATOR_DIR}/gen_api_ui_events.lua) set(API_METADATA ${PROJECT_BINARY_DIR}/api_metadata.mpack) set(FUNCS_DATA ${PROJECT_BINARY_DIR}/funcs_data.mpack) set(MSGPACK_LUA_C_BINDINGS ${GENERATED_DIR}/msgpack_lua_c_bindings.generated.c) -set(HEADER_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendeclarations.lua) +set(HEADER_GENERATOR ${GENERATOR_DIR}/gen_declarations.lua) set(GENERATED_INCLUDES_DIR ${PROJECT_BINARY_DIR}/include) set(GENERATED_API_DISPATCH ${GENERATED_DIR}/api/private/dispatch_wrappers.generated.h) set(GENERATED_FUNCS_METADATA ${GENERATED_DIR}/api/private/funcs_metadata.generated.h) @@ -31,16 +32,16 @@ set(GENERATED_FUNCS ${GENERATED_DIR}/funcs.generated.h) set(GENERATED_EVENTS_ENUM ${GENERATED_INCLUDES_DIR}/auevents_enum.generated.h) set(GENERATED_EVENTS_NAMES_MAP ${GENERATED_DIR}/auevents_name_map.generated.h) set(GENERATED_OPTIONS ${GENERATED_DIR}/options.generated.h) -set(EX_CMDS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genex_cmds.lua) -set(FUNCS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/geneval.lua) -set(EVENTS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gen_events.lua) -set(OPTIONS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genoptions.lua) -set(UNICODE_TABLES_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genunicodetables.lua) +set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua) +set(FUNCS_GENERATOR ${GENERATOR_DIR}/gen_eval.lua) +set(EVENTS_GENERATOR ${GENERATOR_DIR}/gen_events.lua) +set(OPTIONS_GENERATOR ${GENERATOR_DIR}/gen_options.lua) +set(UNICODE_TABLES_GENERATOR ${GENERATOR_DIR}/gen_unicode_tables.lua) set(UNICODE_DIR ${PROJECT_SOURCE_DIR}/unicode) set(GENERATED_UNICODE_TABLES ${GENERATED_DIR}/unicode_tables.generated.h) set(VIM_MODULE_FILE ${GENERATED_DIR}/lua/vim_module.generated.h) set(VIM_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/src/nvim/lua/vim.lua) -set(CHAR_BLOB_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gencharblob.lua) +set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua) set(LINT_SUPPRESS_FILE ${PROJECT_BINARY_DIR}/errors.json) set(LINT_SUPPRESS_URL_BASE "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint") set(LINT_SUPPRESS_URL "${LINT_SUPPRESS_URL_BASE}/errors.json") @@ -238,7 +239,7 @@ add_custom_command(OUTPUT ${GENERATED_UNICODE_TABLES} add_custom_command( OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_FUNCS_METADATA} ${API_METADATA} ${MSGPACK_LUA_C_BINDINGS} - COMMAND ${LUA_PRG} ${MSGPACK_GENERATOR} ${CMAKE_CURRENT_LIST_DIR} + COMMAND ${LUA_PRG} ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR} ${GENERATED_API_DISPATCH} ${GENERATED_FUNCS_METADATA} ${API_METADATA} ${MSGPACK_LUA_C_BINDINGS} @@ -246,7 +247,7 @@ add_custom_command( DEPENDS ${API_HEADERS} ${MSGPACK_RPC_HEADERS} - ${MSGPACK_GENERATOR} + ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua ) @@ -263,10 +264,11 @@ list(APPEND NVIM_GENERATED_SOURCES "${MSGPACK_LUA_C_BINDINGS}" ) -add_custom_command(OUTPUT ${GENERATED_UI_EVENTS} - ${GENERATED_UI_EVENTS_CALL} - ${GENERATED_UI_EVENTS_REMOTE} - ${GENERATED_UI_EVENTS_BRIDGE} +add_custom_command( + OUTPUT ${GENERATED_UI_EVENTS} + ${GENERATED_UI_EVENTS_CALL} + ${GENERATED_UI_EVENTS_REMOTE} + ${GENERATED_UI_EVENTS_BRIDGE} COMMAND ${LUA_PRG} ${API_UI_EVENTS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h ${GENERATED_UI_EVENTS} diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua new file mode 100644 index 0000000000..5c52ec5930 --- /dev/null +++ b/src/nvim/generators/c_grammar.lua @@ -0,0 +1,51 @@ +lpeg = require('lpeg') + +-- lpeg grammar for building api metadata from a set of header files. It +-- ignores comments and preprocessor commands and parses a very small subset +-- of C prototypes with a limited set of types +local P, R, S = lpeg.P, lpeg.R, lpeg.S +local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg + +local any = P(1) -- (consume one character) +local letter = R('az', 'AZ') + S('_$') +local num = R('09') +local alpha = letter + num +local nl = P('\r\n') + P('\n') +local not_nl = any - nl +local ws = S(' \t') + nl +local fill = ws ^ 0 +local c_comment = P('//') * (not_nl ^ 0) +local c_preproc = P('#') * (not_nl ^ 0) +local typed_container = + (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')') +local c_id = ( + typed_container + + (letter * (alpha ^ 0)) +) +local c_void = P('void') +local c_param_type = ( + ((P('Error') * fill * P('*') * fill) * Cc('error')) + + (C(c_id) * (ws ^ 1)) + ) +local c_type = (C(c_void) * (ws ^ 1)) + c_param_type +local c_param = Ct(c_param_type * C(c_id)) +local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0) +local c_params = Ct(c_void + c_param_list) +local c_proto = Ct( + Cg(c_type, 'return_type') * Cg(c_id, 'name') * + fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') * + Cg(Cc(false), 'async') * + (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) * + (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(num ^ 1)) * P(')'), + 'deprecated_since') ^ -1) * + (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) * + (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) * + (fill * Cg((P('FUNC_API_NOEVAL') * Cc(true)), 'noeval') ^ -1) * + (fill * Cg((P('REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) * + (fill * Cg((P('REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) * + (fill * Cg((P('BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) * + fill * P(';') + ) + +local grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1) +return {grammar=grammar, typed_container=typed_container} diff --git a/src/nvim/generators/dump_bin_array.lua b/src/nvim/generators/dump_bin_array.lua new file mode 100644 index 0000000000..bee5aba73f --- /dev/null +++ b/src/nvim/generators/dump_bin_array.lua @@ -0,0 +1,17 @@ +local function dump_bin_array(output, name, data) + output:write([[ + static const uint8_t ]]..name..[[[] = { +]]) + + for i = 1, #data do + output:write(string.byte(data, i)..', ') + if i % 10 == 0 then + output:write('\n ') + end + end + output:write([[ +}; +]]) +end + +return dump_bin_array diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua new file mode 100644 index 0000000000..7adefbfc15 --- /dev/null +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -0,0 +1,458 @@ +mpack = require('mpack') + +-- we need at least 4 arguments since the last two are output files +if arg[1] == '--help' then + print('Usage: genmsgpack.lua args') + print('Args: 1: source directory') + print(' 2: dispatch output file (dispatch_wrappers.generated.h)') + print(' 3: functions metadata output file (funcs_metadata.generated.h)') + print(' 4: API metadata output file (api_metadata.mpack)') + print(' 5: lua C bindings output file (msgpack_lua_c_bindings.generated.c)') + print(' rest: C files where API functions are defined') +end +assert(#arg >= 4) +functions = {} + +local nvimdir = arg[1] +package.path = nvimdir .. '/?.lua;' .. package.path + +-- names of all headers relative to the source root (for inclusion in the +-- generated file) +headers = {} + +-- output h file with generated dispatch functions +dispatch_outputf = arg[2] +-- output h file with packed metadata +funcs_metadata_outputf = arg[3] +-- output metadata mpack file, for use by other build scripts +mpack_outputf = arg[4] +lua_c_bindings_outputf = arg[5] + +-- set of function names, used to detect duplicates +function_names = {} + +c_grammar = require('generators.c_grammar') + +-- read each input file, parse and append to the api metadata +for i = 6, #arg do + local full_path = arg[i] + local parts = {} + for part in string.gmatch(full_path, '[^/]+') do + parts[#parts + 1] = part + end + headers[#headers + 1] = parts[#parts - 1]..'/'..parts[#parts] + + local input = io.open(full_path, 'rb') + + local tmp = c_grammar.grammar:match(input:read('*all')) + for i = 1, #tmp do + local fn = tmp[i] + if not fn.noexport then + functions[#functions + 1] = tmp[i] + function_names[fn.name] = true + if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then + -- this function should receive the channel id + fn.receives_channel_id = true + -- remove the parameter since it won't be passed by the api client + table.remove(fn.parameters, 1) + end + if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then + -- function can fail if the last parameter type is 'Error' + fn.can_fail = true + -- remove the error parameter, msgpack has it's own special field + -- for specifying errors + fn.parameters[#fn.parameters] = nil + end + end + end + input:close() +end + +local function shallowcopy(orig) + local copy = {} + for orig_key, orig_value in pairs(orig) do + copy[orig_key] = orig_value + end + return copy +end + +local function startswith(String,Start) + return string.sub(String,1,string.len(Start))==Start +end + +-- Export functions under older deprecated names. +-- These will be removed eventually. +local deprecated_aliases = require("api.dispatch_deprecated") +for i,f in ipairs(shallowcopy(functions)) do + local ismethod = false + if startswith(f.name, "nvim_") then + if startswith(f.name, "nvim__") then + f.since = -1 + elseif f.since == nil then + print("Function "..f.name.." lacks since field.\n") + os.exit(1) + end + f.since = tonumber(f.since) + if f.deprecated_since ~= nil then + f.deprecated_since = tonumber(f.deprecated_since) + end + + if startswith(f.name, "nvim_buf_") then + ismethod = true + elseif startswith(f.name, "nvim_win_") then + ismethod = true + elseif startswith(f.name, "nvim_tabpage_") then + ismethod = true + end + else + f.noeval = true + f.since = 0 + f.deprecated_since = 1 + end + f.method = ismethod + local newname = deprecated_aliases[f.name] + if newname ~= nil then + if function_names[newname] then + -- duplicate + print("Function "..f.name.." has deprecated alias\n" + ..newname.." which has a separate implementation.\n".. + "Please remove it from src/nvim/api/dispatch_deprecated.lua") + os.exit(1) + end + local newf = shallowcopy(f) + newf.name = newname + if newname == "ui_try_resize" then + -- The return type was incorrectly set to Object in 0.1.5. + -- Keep it that way for clients that rely on this. + newf.return_type = "Object" + end + newf.impl_name = f.name + newf.noeval = true + newf.since = 0 + newf.deprecated_since = 1 + functions[#functions+1] = newf + end +end + +-- don't expose internal attributes like "impl_name" in public metadata +exported_attributes = {'name', 'parameters', 'return_type', 'method', + 'since', 'deprecated_since'} +exported_functions = {} +for _,f in ipairs(functions) do + if not startswith(f.name, "nvim__") then + local f_exported = {} + for _,attr in ipairs(exported_attributes) do + f_exported[attr] = f[attr] + end + exported_functions[#exported_functions+1] = f_exported + end +end + + +-- serialize the API metadata using msgpack and embed into the resulting +-- binary for easy querying by clients +funcs_metadata_output = io.open(funcs_metadata_outputf, 'wb') +packed = mpack.pack(exported_functions) +dump_bin_array = require("generators.dump_bin_array") +dump_bin_array(funcs_metadata_output, 'funcs_metadata', packed) +funcs_metadata_output:close() + +-- start building the dispatch wrapper output +output = io.open(dispatch_outputf, 'wb') + +local function real_type(type) + local rv = type + if c_grammar.typed_container:match(rv) then + if rv:match('Array') then + rv = 'Array' + else + rv = 'Dictionary' + end + end + return rv +end + +local function attr_name(rt) + if rt == 'Float' then + return 'floating' + else + return rt:lower() + end +end + +-- start the handler functions. Visit each function metadata to build the +-- handler function with code generated for validating arguments and calling to +-- the real API. +for i = 1, #functions do + local fn = functions[i] + if fn.impl_name == nil then + local args = {} + + output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)') + output:write('\n{') + output:write('\n Object ret = NIL;') + -- Declare/initialize variables that will hold converted arguments + for j = 1, #fn.parameters do + local param = fn.parameters[j] + local converted = 'arg_'..j + output:write('\n '..param[1]..' '..converted..';') + end + output:write('\n') + output:write('\n if (args.size != '..#fn.parameters..') {') + output:write('\n api_set_error(error, kErrorTypeException, "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);') + output:write('\n goto cleanup;') + output:write('\n }\n') + + -- Validation/conversion for each argument + for j = 1, #fn.parameters do + local converted, convert_arg, param, arg + param = fn.parameters[j] + converted = 'arg_'..j + local rt = real_type(param[1]) + if rt ~= 'Object' then + if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then + -- Buffer, Window, and Tabpage have a specific type, but are stored in integer + output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..' && args.items['..(j - 1)..'].data.integer >= 0) {') + output:write('\n '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;') + else + output:write('\n if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {') + output:write('\n '..converted..' = args.items['..(j - 1)..'].data.'..attr_name(rt)..';') + end + if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then + -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages + output:write('\n } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer >= 0) {') + output:write('\n '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;') + end + output:write('\n } else {') + output:write('\n api_set_error(error, kErrorTypeException, "Wrong type for argument '..j..', expecting '..param[1]..'");') + output:write('\n goto cleanup;') + output:write('\n }\n') + else + output:write('\n '..converted..' = args.items['..(j - 1)..'];\n') + end + + args[#args + 1] = converted + end + + -- function call + local call_args = table.concat(args, ', ') + output:write('\n ') + if fn.return_type ~= 'void' then + -- has a return value, prefix the call with a declaration + output:write(fn.return_type..' rv = ') + end + + -- write the function name and the opening parenthesis + output:write(fn.name..'(') + + if fn.receives_channel_id then + -- if the function receives the channel id, pass it as first argument + if #args > 0 or fn.can_fail then + output:write('channel_id, '..call_args) + else + output:write('channel_id') + end + else + output:write(call_args) + end + + if fn.can_fail then + -- if the function can fail, also pass a pointer to the local error object + if #args > 0 then + output:write(', error);\n') + else + output:write('error);\n') + end + -- and check for the error + output:write('\n if (ERROR_SET(error)) {') + output:write('\n goto cleanup;') + output:write('\n }\n') + else + output:write(');\n') + end + + if fn.return_type ~= 'void' then + output:write('\n ret = '..string.upper(real_type(fn.return_type))..'_OBJ(rv);') + end + output:write('\n\ncleanup:'); + + output:write('\n return ret;\n}\n\n'); + end +end + +-- Generate a function that initializes method names with handler functions +output:write([[ +void msgpack_rpc_init_method_table(void) +{ + methods = map_new(String, MsgpackRpcRequestHandler)(); + +]]) + +for i = 1, #functions do + local fn = functions[i] + output:write(' msgpack_rpc_add_method_handler('.. + '(String) {.data = "'..fn.name..'", '.. + '.size = sizeof("'..fn.name..'") - 1}, '.. + '(MsgpackRpcRequestHandler) {.fn = handle_'.. (fn.impl_name or fn.name).. + ', .async = '..tostring(fn.async)..'});\n') + +end + +output:write('\n}\n\n') +output:close() + +mpack_output = io.open(mpack_outputf, 'wb') +mpack_output:write(mpack.pack(functions)) +mpack_output:close() + +local function include_headers(output, headers) + for i = 1, #headers do + if headers[i]:sub(-12) ~= '.generated.h' then + output:write('\n#include "nvim/'..headers[i]..'"') + end + end +end + +local function write_shifted_output(output, str) + str = str:gsub('\n ', '\n') + str = str:gsub('^ ', '') + str = str:gsub(' +$', '') + output:write(str) +end + +-- start building lua output +output = io.open(lua_c_bindings_outputf, 'wb') + +output:write([[ +#include +#include +#include + +#include "nvim/func_attr.h" +#include "nvim/api/private/defs.h" +#include "nvim/api/private/helpers.h" +#include "nvim/lua/converter.h" +]]) +include_headers(output, headers) +output:write('\n') + +lua_c_functions = {} + +local function process_function(fn) + lua_c_function_name = ('nlua_msgpack_%s'):format(fn.name) + write_shifted_output(output, string.format([[ + + static int %s(lua_State *lstate) + { + Error err = ERROR_INIT; + if (lua_gettop(lstate) != %i) { + api_set_error(&err, kErrorTypeValidation, "Expected %i argument%s"); + goto exit_0; + } + ]], lua_c_function_name, #fn.parameters, #fn.parameters, + (#fn.parameters == 1) and '' or 's')) + lua_c_functions[#lua_c_functions + 1] = { + binding=lua_c_function_name, + api=fn.name + } + local cparams = '' + local free_code = {} + for j = #fn.parameters,1,-1 do + param = fn.parameters[j] + cparam = string.format('arg%u', j) + param_type = real_type(param[1]) + lc_param_type = param_type:lower() + write_shifted_output(output, string.format([[ + const %s %s = nlua_pop_%s(lstate, &err); + + if (ERROR_SET(&err)) { + goto exit_%u; + } + ]], param[1], cparam, param_type, #fn.parameters - j)) + free_code[#free_code + 1] = ('api_free_%s(%s);'):format( + lc_param_type, cparam) + cparams = cparam .. ', ' .. cparams + end + if fn.receives_channel_id then + cparams = 'LUA_INTERNAL_CALL, ' .. cparams + end + if fn.can_fail then + cparams = cparams .. '&err' + else + cparams = cparams:gsub(', $', '') + end + local free_at_exit_code = '' + for i = 1, #free_code do + local rev_i = #free_code - i + 1 + local code = free_code[rev_i] + if i == 1 then + free_at_exit_code = free_at_exit_code .. ('\n %s'):format(code) + else + free_at_exit_code = free_at_exit_code .. ('\n exit_%u:\n %s'):format( + rev_i, code) + end + end + local err_throw_code = [[ + + exit_0: + if (ERROR_SET(&err)) { + luaL_where(lstate, 1); + lua_pushstring(lstate, err.msg); + api_clear_error(&err); + lua_concat(lstate, 2); + return lua_error(lstate); + } + ]] + if fn.return_type ~= 'void' then + if fn.return_type:match('^ArrayOf') then + return_type = 'Array' + else + return_type = fn.return_type + end + write_shifted_output(output, string.format([[ + const %s ret = %s(%s); + nlua_push_%s(lstate, ret); + api_free_%s(ret); + %s + %s + return 1; + ]], fn.return_type, fn.name, cparams, return_type, return_type:lower(), + free_at_exit_code, err_throw_code)) + else + write_shifted_output(output, string.format([[ + %s(%s); + %s + %s + return 0; + ]], fn.name, cparams, free_at_exit_code, err_throw_code)) + end + write_shifted_output(output, [[ + } + ]]) +end + +for _, fn in ipairs(functions) do + if not fn.noeval or fn.name:sub(1, 4) == '_vim' then + process_function(fn) + end +end + +output:write(string.format([[ +void nlua_add_api_functions(lua_State *lstate) + FUNC_ATTR_NONNULL_ALL +{ + lua_createtable(lstate, 0, %u); +]], #lua_c_functions)) +for _, func in ipairs(lua_c_functions) do + output:write(string.format([[ + + lua_pushcfunction(lstate, &%s); + lua_setfield(lstate, -2, "%s");]], func.binding, func.api)) +end +output:write([[ + + lua_setfield(lstate, -2, "api"); +} +]]) + +output:close() diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/nvim/generators/gen_api_ui_events.lua new file mode 100644 index 0000000000..28007d2266 --- /dev/null +++ b/src/nvim/generators/gen_api_ui_events.lua @@ -0,0 +1,136 @@ +mpack = require('mpack') + +local nvimdir = arg[1] +package.path = nvimdir .. '/?.lua;' .. package.path + +assert(#arg == 6) +input = io.open(arg[2], 'rb') +proto_output = io.open(arg[3], 'wb') +call_output = io.open(arg[4], 'wb') +remote_output = io.open(arg[5], 'wb') +bridge_output = io.open(arg[6], 'wb') + +c_grammar = require('generators.c_grammar') +local events = c_grammar.grammar:match(input:read('*all')) + +function write_signature(output, ev, prefix, notype) + output:write('('..prefix) + if prefix == "" and #ev.parameters == 0 then + output:write('void') + end + for j = 1, #ev.parameters do + if j > 1 or prefix ~= '' then + output:write(', ') + end + local param = ev.parameters[j] + if not notype then + output:write(param[1]..' ') + end + output:write(param[2]) + end + output:write(')') +end + +function write_arglist(output, ev, need_copy) + output:write(' Array args = ARRAY_DICT_INIT;\n') + for j = 1, #ev.parameters do + local param = ev.parameters[j] + local kind = string.upper(param[1]) + local do_copy = need_copy and (kind == "ARRAY" or kind == "DICTIONARY" or kind == "STRING") + output:write(' ADD(args, ') + if do_copy then + output:write('copy_object(') + end + output:write(kind..'_OBJ('..param[2]..')') + if do_copy then + output:write(')') + end + output:write(');\n') + end +end + +for i = 1, #events do + ev = events[i] + assert(ev.return_type == 'void') + + if not ev.remote_only then + proto_output:write(' void (*'..ev.name..')') + write_signature(proto_output, ev, 'UI *ui') + proto_output:write(';\n') + + if not ev.remote_impl then + remote_output:write('static void remote_ui_'..ev.name) + write_signature(remote_output, ev, 'UI *ui') + remote_output:write('\n{\n') + write_arglist(remote_output, ev, true) + remote_output:write(' push_call(ui, "'..ev.name..'", args);\n') + remote_output:write('}\n\n') + end + + if not ev.bridge_impl then + + send, argv, recv, recv_argv, recv_cleanup = '', '', '', '', '' + argc = 1 + for j = 1, #ev.parameters do + local param = ev.parameters[j] + copy = 'copy_'..param[2] + if param[1] == 'String' then + send = send..' String copy_'..param[2]..' = copy_string('..param[2]..');\n' + argv = argv..', '..copy..'.data, INT2PTR('..copy..'.size)' + recv = (recv..' String '..param[2].. + ' = (String){.data = argv['..argc..'],'.. + '.size = (size_t)argv['..(argc+1)..']};\n') + recv_argv = recv_argv..', '..param[2] + recv_cleanup = recv_cleanup..' api_free_string('..param[2]..');\n' + argc = argc+2 + elseif param[1] == 'Array' then + send = send..' Array copy_'..param[2]..' = copy_array('..param[2]..');\n' + argv = argv..', '..copy..'.items, INT2PTR('..copy..'.size)' + recv = (recv..' Array '..param[2].. + ' = (Array){.items = argv['..argc..'],'.. + '.size = (size_t)argv['..(argc+1)..']};\n') + recv_argv = recv_argv..', '..param[2] + recv_cleanup = recv_cleanup..' api_free_array('..param[2]..');\n' + argc = argc+2 + elseif param[1] == 'Integer' or param[1] == 'Boolean' then + argv = argv..', INT2PTR('..param[2]..')' + recv_argv = recv_argv..', PTR2INT(argv['..argc..'])' + argc = argc+1 + else + assert(false) + end + end + bridge_output:write('static void ui_bridge_'..ev.name.. + '_event(void **argv)\n{\n') + bridge_output:write(' UI *ui = UI(argv[0]);\n') + bridge_output:write(recv) + bridge_output:write(' ui->'..ev.name..'(ui'..recv_argv..');\n') + bridge_output:write(recv_cleanup) + bridge_output:write('}\n\n') + + bridge_output:write('static void ui_bridge_'..ev.name) + write_signature(bridge_output, ev, 'UI *ui') + bridge_output:write('\n{\n') + bridge_output:write(send) + bridge_output:write(' UI_CALL(ui, '..ev.name..', '..argc..', ui'..argv..');\n}\n') + end + end + + call_output:write('void ui_call_'..ev.name) + write_signature(call_output, ev, '') + call_output:write('\n{\n') + if ev.remote_only then + write_arglist(call_output, ev, false) + call_output:write(' ui_event("'..ev.name..'", args);\n') + else + call_output:write(' UI_CALL') + write_signature(call_output, ev, ev.name, true) + call_output:write(";\n") + end + call_output:write("}\n\n") + +end + +proto_output:close() +call_output:close() +remote_output:close() diff --git a/src/nvim/generators/gen_char_blob.lua b/src/nvim/generators/gen_char_blob.lua new file mode 100644 index 0000000000..d860375e26 --- /dev/null +++ b/src/nvim/generators/gen_char_blob.lua @@ -0,0 +1,48 @@ +if arg[1] == '--help' then + print('Usage:') + print(' gencharblob.lua source target varname') + print('') + print('Generates C file with big uint8_t blob.') + print('Blob will be stored in a static const array named varname.') + os.exit() +end + +assert(#arg == 3) + +local source_file = arg[1] +local target_file = arg[2] +local varname = arg[3] + +source = io.open(source_file, 'r') +target = io.open(target_file, 'w') + +target:write('#include \n\n') +target:write(('static const uint8_t %s[] = {\n'):format(varname)) + +num_bytes = 0 +MAX_NUM_BYTES = 15 -- 78 / 5: maximum number of bytes on one line +target:write(' ') + +increase_num_bytes = function() + num_bytes = num_bytes + 1 + if num_bytes == MAX_NUM_BYTES then + num_bytes = 0 + target:write('\n ') + end +end + +for line in source:lines() do + for i = 1,string.len(line) do + byte = string.byte(line, i) + assert(byte ~= 0) + target:write(string.format(' %3u,', byte)) + increase_num_bytes() + end + target:write(string.format(' %3u,', string.byte('\n', 1))) + increase_num_bytes() +end + +target:write(' 0};\n') + +source:close() +target:close() diff --git a/src/nvim/generators/gen_declarations.lua b/src/nvim/generators/gen_declarations.lua new file mode 100755 index 0000000000..e999e53e4a --- /dev/null +++ b/src/nvim/generators/gen_declarations.lua @@ -0,0 +1,287 @@ +#!/usr/bin/lua + +local fname = arg[1] +local static_fname = arg[2] +local non_static_fname = arg[3] +local preproc_fname = arg[4] + + +local lpeg = require('lpeg') + +local fold = function (func, ...) + local result = nil + for i, v in ipairs({...}) do + if result == nil then + result = v + else + result = func(result, v) + end + end + return result +end + +local folder = function (func) + return function (...) + return fold(func, ...) + end +end + +local lit = lpeg.P +local set = function(...) + return lpeg.S(fold(function (a, b) return a .. b end, ...)) +end +local any_character = lpeg.P(1) +local rng = function(s, e) return lpeg.R(s .. e) end +local concat = folder(function (a, b) return a * b end) +local branch = folder(function (a, b) return a + b end) +local one_or_more = function(v) return v ^ 1 end +local two_or_more = function(v) return v ^ 2 end +local any_amount = function(v) return v ^ 0 end +local one_or_no = function(v) return v ^ -1 end +local look_behind = lpeg.B +local look_ahead = function(v) return #v end +local neg_look_ahead = function(v) return -v end +local neg_look_behind = function(v) return -look_behind(v) end + +local w = branch( + rng('a', 'z'), + rng('A', 'Z'), + lit('_') +) +local aw = branch( + w, + rng('0', '9') +) +local s = set(' ', '\n', '\t') +local raw_word = concat(w, any_amount(aw)) +local right_word = concat( + raw_word, + neg_look_ahead(aw) +) +local word = branch( + concat( + branch(lit('ArrayOf('), lit('DictionaryOf(')), -- typed container macro + one_or_more(any_character - lit(')')), + lit(')') + ), + concat( + neg_look_behind(aw), + right_word + ) +) +local inline_comment = concat( + lit('/*'), + any_amount(concat( + neg_look_ahead(lit('*/')), + any_character + )), + lit('*/') +) +local spaces = any_amount(branch( + s, + -- Comments are really handled by preprocessor, so the following is not needed + inline_comment, + concat( + lit('//'), + any_amount(concat( + neg_look_ahead(lit('\n')), + any_character + )), + lit('\n') + ), + -- Linemarker inserted by preprocessor + concat( + lit('# '), + any_amount(concat( + neg_look_ahead(lit('\n')), + any_character + )), + lit('\n') + ) +)) +local typ_part = concat( + word, + any_amount(concat( + spaces, + lit('*') + )), + spaces +) +local typ = one_or_more(typ_part) +local typ_id = two_or_more(typ_part) +local arg = typ_id -- argument name is swallowed by typ +local pattern = concat( + any_amount(branch(set(' ', '\t'), inline_comment)), + typ_id, -- return type with function name + spaces, + lit('('), + spaces, + one_or_no(branch( -- function arguments + concat( + arg, -- first argument, does not require comma + any_amount(concat( -- following arguments, start with a comma + spaces, + lit(','), + spaces, + arg, + any_amount(concat( + lit('['), + spaces, + any_amount(aw), + spaces, + lit(']') + )) + )), + one_or_no(concat( + spaces, + lit(','), + spaces, + lit('...') + )) + ), + lit('void') -- also accepts just void + )), + spaces, + lit(')'), + any_amount(concat( -- optional attributes + spaces, + lit('FUNC_'), + any_amount(aw), + one_or_no(concat( -- attribute argument + spaces, + lit('('), + any_amount(concat( + neg_look_ahead(lit(')')), + any_character + )), + lit(')') + )) + )), + look_ahead(concat( -- definition must be followed by "{" + spaces, + lit('{') + )) +) + +if fname == '--help' then + print'Usage:' + print() + print' gendeclarations.lua definitions.c static.h non-static.h preprocessor.i' + os.exit() +end + +local preproc_f = io.open(preproc_fname) +local text = preproc_f:read("*all") +preproc_f:close() + + +local header = [[ +#ifndef DEFINE_FUNC_ATTRIBUTES +# define DEFINE_FUNC_ATTRIBUTES +#endif +#include "nvim/func_attr.h" +#undef DEFINE_FUNC_ATTRIBUTES +]] + +local footer = [[ +#include "nvim/func_attr.h" +]] + +local non_static = header +local static = header + +local filepattern = '^#%a* (%d+) "([^"]-)/?([^"/]+)"' +local curfile + +local init = 0 +local curfile = nil +local neededfile = fname:match('[^/]+$') +local declline = 0 +local declendpos = 0 +local curdir = nil +local is_needed_file = false +while init ~= nil do + init = text:find('[\n;}]', init) + if init == nil then + break + end + local init_is_nl = text:sub(init, init) == '\n' + init = init + 1 + if init_is_nl and is_needed_file then + declline = declline + 1 + end + if init_is_nl and text:sub(init, init) == '#' then + local line, dir, file = text:match(filepattern, init) + if file ~= nil then + curfile = file + is_needed_file = (curfile == neededfile) + declline = tonumber(line) - 1 + local curdir_start = dir:find('src/nvim/') + if curdir_start ~= nil then + curdir = dir:sub(curdir_start + #('src/nvim/')) + else + curdir = dir + end + else + declline = declline - 1 + end + elseif init < declendpos then + -- Skipping over declaration + elseif is_needed_file then + s = init + e = pattern:match(text, init) + if e ~= nil then + local declaration = text:sub(s, e - 1) + -- Comments are really handled by preprocessor, so the following is not + -- needed + declaration = declaration:gsub('/%*.-%*/', '') + declaration = declaration:gsub('//.-\n', '\n') + + declaration = declaration:gsub('# .-\n', '') + + declaration = declaration:gsub('\n', ' ') + declaration = declaration:gsub('%s+', ' ') + declaration = declaration:gsub(' ?%( ?', '(') + -- declaration = declaration:gsub(' ?%) ?', ')') + declaration = declaration:gsub(' ?, ?', ', ') + declaration = declaration:gsub(' ?(%*+) ?', ' %1') + declaration = declaration:gsub(' ?(FUNC_ATTR_)', ' %1') + declaration = declaration:gsub(' $', '') + declaration = declaration:gsub('^ ', '') + declaration = declaration .. ';' + declaration = declaration .. (' // %s/%s:%u'):format( + curdir, curfile, declline) + declaration = declaration .. '\n' + if declaration:sub(1, 6) == 'static' then + static = static .. declaration + else + non_static = non_static .. declaration + end + declendpos = e + end + end +end + +non_static = non_static .. footer +static = static .. footer + +local F +F = io.open(static_fname, 'w') +F:write(static) +F:close() + +-- Before generating the non-static headers, check if the current file(if +-- exists) is different from the new one. If they are the same, we won't touch +-- the current version to avoid triggering an unnecessary rebuilds of modules +-- that depend on this one +F = io.open(non_static_fname, 'r') +if F ~= nil then + if F:read('*a') == non_static then + os.exit(0) + end + io.close(F) +end + +F = io.open(non_static_fname, 'w') +F:write(non_static) +F:close() diff --git a/src/nvim/generators/gen_eval.lua b/src/nvim/generators/gen_eval.lua new file mode 100644 index 0000000000..b17ecb1a0c --- /dev/null +++ b/src/nvim/generators/gen_eval.lua @@ -0,0 +1,67 @@ +mpack = require('mpack') + +local nvimsrcdir = arg[1] +local autodir = arg[2] +local metadata_file = arg[3] +local funcs_file = arg[4] + +if nvimsrcdir == '--help' then + print([[ +Usage: + lua geneval.lua src/nvim build/src/nvim/auto + +Will generate build/src/nvim/auto/funcs.generated.h with definition of functions +static const array. +]]) + os.exit(0) +end + +package.path = nvimsrcdir .. '/?.lua;' .. package.path + +local funcsfname = autodir .. '/funcs.generated.h' + +local gperfpipe = io.open(funcsfname .. '.gperf', 'wb') + +local funcs = require('eval').funcs +local metadata = mpack.unpack(io.open(arg[3], 'rb'):read("*all")) +for i,fun in ipairs(metadata) do + if not fun.noeval then + funcs[fun.name] = { + args=#fun.parameters, + func='api_wrapper', + data='&handle_'..fun.name, + } + end +end + +local funcsdata = io.open(funcs_file, 'w') +funcsdata:write(mpack.pack(funcs)) +funcsdata:close() + +gperfpipe:write([[ +%language=ANSI-C +%global-table +%readonly-tables +%define initializer-suffix ,0,0,NULL,NULL +%define word-array-name functions +%define hash-function-name hash_internal_func_gperf +%define lookup-function-name find_internal_func_gperf +%omit-struct-type +%struct-type +VimLFuncDef; +%% +]]) + +for name, def in pairs(funcs) do + args = def.args or 0 + if type(args) == 'number' then + args = {args, args} + elseif #args == 1 then + args[2] = 'MAX_FUNC_ARGS' + end + func = def.func or ('f_' .. name) + data = def.data or "NULL" + gperfpipe:write(('%s, %s, %s, &%s, (FunPtr)%s\n') + :format(name, args[1], args[2], func, data)) +end +gperfpipe:close() diff --git a/src/nvim/generators/gen_events.lua b/src/nvim/generators/gen_events.lua new file mode 100644 index 0000000000..75e0b3da3a --- /dev/null +++ b/src/nvim/generators/gen_events.lua @@ -0,0 +1,65 @@ +if arg[1] == '--help' then + print('Usage: gen_events.lua src/nvim enum_file event_names_file') + os.exit(0) +end + +local nvimsrcdir = arg[1] +local fileio_enum_file = arg[2] +local names_file = arg[3] + +package.path = nvimsrcdir .. '/?.lua;' .. package.path + +local auevents = require('auevents') +local events = auevents.events +local aliases = auevents.aliases + +enum_tgt = io.open(fileio_enum_file, 'w') +names_tgt = io.open(names_file, 'w') + +enum_tgt:write('typedef enum auto_event {') +names_tgt:write([[ +static const struct event_name { + size_t len; + char *name; + event_T event; +} event_names[] = {]]) + +for i, event in ipairs(events) do + if i > 1 then + comma = ',\n' + else + comma = '\n' + end + enum_tgt:write(('%s EVENT_%s = %u'):format(comma, event:upper(), i - 1)) + names_tgt:write(('%s {%u, "%s", EVENT_%s}'):format(comma, #event, event, event:upper())) +end + +for alias, event in pairs(aliases) do + names_tgt:write((',\n {%u, "%s", EVENT_%s}'):format(#alias, alias, event:upper())) +end + +names_tgt:write(',\n {0, NULL, (event_T)0}') + +enum_tgt:write('\n} event_T;\n') +names_tgt:write('\n};\n') + +enum_tgt:write(('\n#define NUM_EVENTS %u\n'):format(#events)) +names_tgt:write('\nstatic AutoPat *first_autopat[NUM_EVENTS] = {\n ') +line_len = 1 +for i = 1,((#events) - 1) do + line_len = line_len + #(' NULL,') + if line_len > 80 then + names_tgt:write('\n ') + line_len = 1 + #(' NULL,') + end + names_tgt:write(' NULL,') +end +if line_len + #(' NULL') > 80 then + names_tgt:write('\n NULL') +else + names_tgt:write(' NULL') +end +names_tgt:write('\n};\n') + +enum_tgt:close() +names_tgt:close() diff --git a/src/nvim/generators/gen_ex_cmds.lua b/src/nvim/generators/gen_ex_cmds.lua new file mode 100644 index 0000000000..cb566d46ca --- /dev/null +++ b/src/nvim/generators/gen_ex_cmds.lua @@ -0,0 +1,88 @@ +local nvimsrcdir = arg[1] +local includedir = arg[2] +local autodir = arg[3] + +if nvimsrcdir == '--help' then + print ([[ +Usage: + lua genex_cmds.lua src/nvim build/include build/src/nvim/auto + +Will generate files build/include/ex_cmds_enum.generated.h with cmdidx_T +enum and build/src/nvim/auto/ex_cmds_defs.generated.h with main Ex commands +definitions. +]]) + os.exit(0) +end + +package.path = nvimsrcdir .. '/?.lua;' .. package.path + +local enumfname = includedir .. '/ex_cmds_enum.generated.h' +local defsfname = autodir .. '/ex_cmds_defs.generated.h' + +local enumfile = io.open(enumfname, 'w') +local defsfile = io.open(defsfname, 'w') + +local defs = require('ex_cmds') +local lastchar = nil + +local i +local cmd +local first = true +local prevfirstchar = nil + +local byte_a = string.byte('a') +local byte_z = string.byte('z') + +local cmdidxs = string.format([[ +static const cmdidx_T cmdidxs[%u] = { +]], byte_z - byte_a + 2) + +enumfile:write([[ +typedef enum CMD_index { +]]) +defsfile:write(string.format([[ +static CommandDefinition cmdnames[%u] = { +]], #defs)) +for i, cmd in ipairs(defs) do + local enumname = cmd.enum or ('CMD_' .. cmd.command) + firstchar = string.byte(cmd.command) + if firstchar ~= prevfirstchar then + if (not prevfirstchar + or (byte_a <= firstchar and firstchar <= byte_z) + or (byte_a <= prevfirstchar and prevfirstchar <= byte_z)) then + if not first then + cmdidxs = cmdidxs .. ',\n' + end + cmdidxs = cmdidxs .. ' ' .. enumname + end + prevfirstchar = firstchar + end + if first then + first = false + else + defsfile:write(',\n') + end + enumfile:write(' ' .. enumname .. ',\n') + defsfile:write(string.format([[ + [%s] = { + .cmd_name = (char_u *) "%s", + .cmd_func = (ex_func_T)&%s, + .cmd_argt = %uL, + .cmd_addr_type = %i + }]], enumname, cmd.command, cmd.func, cmd.flags, cmd.addr_type)) +end +defsfile:write([[ + +}; +]]) +enumfile:write([[ + CMD_SIZE, + CMD_USER = -1, + CMD_USER_BUF = -2 +} cmdidx_T; +]]) +cmdidxs = cmdidxs .. [[ + +}; +]] +defsfile:write(cmdidxs) diff --git a/src/nvim/generators/gen_options.lua b/src/nvim/generators/gen_options.lua new file mode 100644 index 0000000000..9d7f235a3b --- /dev/null +++ b/src/nvim/generators/gen_options.lua @@ -0,0 +1,190 @@ +if arg[1] == '--help' then + print('Usage: genoptions.lua src/nvim options_file') + os.exit(0) +end + +local nvimsrcdir = arg[1] +local options_file = arg[2] + +package.path = nvimsrcdir .. '/?.lua;' .. package.path + +local opt_fd = io.open(options_file, 'w') + +local w = function(s) + if s:match('^ %.') then + opt_fd:write(s .. ',\n') + else + opt_fd:write(s .. '\n') + end +end + +local options = require('options') + +cstr = options.cstr + +local type_flags={ + bool='P_BOOL', + number='P_NUM', + string='P_STRING', +} + +local redraw_flags={ + statuslines='P_RSTAT', + current_window='P_RWIN', + current_window_only='P_RWINONLY', + current_buffer='P_RBUF', + all_windows='P_RALL', + everything='P_RCLR', + curswant='P_CURSWANT', +} + +local list_flags={ + comma='P_COMMA', + onecomma='P_ONECOMMA', + flags='P_FLAGLIST', + flagscomma='P_COMMA|P_FLAGLIST', +} + +local get_flags = function(o) + local ret = {type_flags[o.type]} + local add_flag = function(f) + ret[1] = ret[1] .. '|' .. f + end + if o.list then + add_flag(list_flags[o.list]) + end + if o.redraw then + for _, r_flag in ipairs(o.redraw) do + add_flag(redraw_flags[r_flag]) + end + end + if o.expand then + add_flag('P_EXPAND') + if o.expand == 'nodefault' then + add_flag('P_NO_DEF_EXP') + end + end + for _, flag_desc in ipairs({ + {'alloced'}, + {'nodefault'}, + {'no_mkrc'}, + {'vi_def'}, + {'vim'}, + {'secure'}, + {'gettext'}, + {'noglob'}, + {'normal_fname_chars', 'P_NFNAME'}, + {'pri_mkrc'}, + {'deny_in_modelines', 'P_NO_ML'}, + {'deny_duplicates', 'P_NODUP'}, + }) do + local key_name = flag_desc[1] + local def_name = flag_desc[2] or ('P_' .. key_name:upper()) + if o[key_name] then + add_flag(def_name) + end + end + return ret[1] +end + +local get_cond +get_cond = function(c, base_string) + local cond_string = base_string or '#if ' + if type(c) == 'table' then + cond_string = cond_string .. get_cond(c[1], '') + for i, subc in ipairs(c) do + if i > 1 then + cond_string = cond_string .. ' && ' .. get_cond(subc, '') + end + end + elseif c:sub(1, 1) == '!' then + cond_string = cond_string .. '!defined(' .. c:sub(2) .. ')' + else + cond_string = cond_string .. 'defined(' .. c .. ')' + end + return cond_string +end + +value_dumpers = { + ['function']=function(v) return v() end, + string=cstr, + boolean=function(v) return v and 'true' or 'false' end, + number=function(v) return ('%iL'):format(v) end, + ['nil']=function(v) return '0L' end, +} + +local get_value = function(v) + return '(char_u *) ' .. value_dumpers[type(v)](v) +end + +local get_defaults = function(d) + return '{' .. get_value(d.vi) .. ', ' .. get_value(d.vim) .. '}' +end + +local defines = {} + +local dump_option = function(i, o) + w(' [' .. ('%u'):format(i - 1) .. ']={') + w(' .fullname=' .. cstr(o.full_name)) + if o.abbreviation then + w(' .shortname=' .. cstr(o.abbreviation)) + end + w(' .flags=' .. get_flags(o)) + if o.enable_if then + w(get_cond(o.enable_if)) + end + if o.varname then + w(' .var=(char_u *)&' .. o.varname) + elseif #o.scope == 1 and o.scope[1] == 'window' then + w(' .var=VAR_WIN') + end + if o.enable_if then + w('#endif') + end + if #o.scope == 1 and o.scope[1] == 'global' then + w(' .indir=PV_NONE') + else + assert (#o.scope == 1 or #o.scope == 2) + assert (#o.scope == 1 or o.scope[1] == 'global') + local min_scope = o.scope[#o.scope] + local varname = o.pv_name or o.varname or ( + 'p_' .. (o.abbreviation or o.full_name)) + local pv_name = ( + 'OPT_' .. min_scope:sub(1, 3):upper() .. '(' .. ( + min_scope:sub(1, 1):upper() .. 'V_' .. varname:sub(3):upper() + ) .. ')' + ) + if #o.scope == 2 then + pv_name = 'OPT_BOTH(' .. pv_name .. ')' + end + defines['PV_' .. varname:sub(3):upper()] = pv_name + w(' .indir=' .. pv_name) + end + if o.defaults then + if o.defaults.condition then + w(get_cond(o.defaults.condition)) + end + w(' .def_val=' .. get_defaults(o.defaults.if_true)) + if o.defaults.condition then + if o.defaults.if_false then + w('#else') + w(' .def_val=' .. get_defaults(o.defaults.if_false)) + end + w('#endif') + end + end + w(' },') +end + +w('static vimoption_T options[] = {') +for i, o in ipairs(options.options) do + dump_option(i, o) +end +w(' [' .. ('%u'):format(#options.options) .. ']={.fullname=NULL}') +w('};') +w('') + +for k, v in pairs(defines) do + w('#define ' .. k .. ' ' .. v) +end +opt_fd:close() diff --git a/src/nvim/generators/gen_unicode_tables.lua b/src/nvim/generators/gen_unicode_tables.lua new file mode 100644 index 0000000000..66430ba26e --- /dev/null +++ b/src/nvim/generators/gen_unicode_tables.lua @@ -0,0 +1,327 @@ +-- Script creates the following tables in unicode_tables.generated.h: +-- +-- 1. doublewidth and ambiguous tables: sorted list of non-overlapping closed +-- intervals. Codepoints in these intervals have double (W or F) or ambiguous +-- (A) east asian width respectively. +-- 2. combining table: same as the above, but characters inside are combining +-- characters (i.e. have general categories equal to Mn, Mc or Me). +-- 3. foldCase, toLower and toUpper tables used to convert characters to +-- folded/lower/upper variants. In these tables first two values are +-- character ranges: like in previous tables they are sorted and must be +-- non-overlapping. Third value means step inside the range: e.g. if it is +-- 2 then interval applies only to first, third, fifth, … character in range. +-- Fourth value is number that should be added to the codepoint to yield +-- folded/lower/upper codepoint. +-- 4. emoji_width and emoji_all tables: sorted lists of non-overlapping closed +-- intervals of Emoji characters. emoji_width contains all the characters +-- which don't have ambiguous or double width, and emoji_all has all Emojis. +if arg[1] == '--help' then + print('Usage:') + print(' genunicodetables.lua unicode/ unicode_tables.generated.h') + os.exit(0) +end + +local basedir = arg[1] +local pathsep = package.config:sub(1, 1) +local get_path = function(fname) + return basedir .. pathsep .. fname +end + +local unicodedata_fname = get_path('UnicodeData.txt') +local casefolding_fname = get_path('CaseFolding.txt') +local eastasianwidth_fname = get_path('EastAsianWidth.txt') +local emoji_fname = get_path('emoji-data.txt') + +local utf_tables_fname = arg[2] + +local split_on_semicolons = function(s) + local ret = {} + local idx = 1 + while idx <= #s + 1 do + item = s:match('^[^;]*', idx) + idx = idx + #item + 1 + if idx <= #s + 1 then + assert(s:sub(idx - 1, idx - 1) == ';') + end + item = item:gsub('^%s*', '') + item = item:gsub('%s*$', '') + table.insert(ret, item) + end + return ret +end + +local fp_lines_to_lists = function(fp, n, has_comments) + local ret = {} + local line + local i = 0 + while true do + i = i + 1 + line = fp:read('*l') + if not line then + break + end + if (not has_comments + or (line:sub(1, 1) ~= '#' and not line:match('^%s*$'))) then + local l = split_on_semicolons(line) + if #l ~= n then + io.stderr:write(('Found %s items in line %u, expected %u\n'):format( + #l, i, n)) + io.stderr:write('Line: ' .. line .. '\n') + return nil + end + table.insert(ret, l) + end + end + return ret +end + +local parse_data_to_props = function(ud_fp) + return fp_lines_to_lists(ud_fp, 15, false) +end + +local parse_fold_props = function(cf_fp) + return fp_lines_to_lists(cf_fp, 4, true) +end + +local parse_width_props = function(eaw_fp) + return fp_lines_to_lists(eaw_fp, 2, true) +end + +local parse_emoji_props = function(emoji_fp) + return fp_lines_to_lists(emoji_fp, 2, true) +end + +local make_range = function(start, end_, step, add) + if step and add then + return (' {0x%x, 0x%x, %d, %d},\n'):format( + start, end_, step == 0 and -1 or step, add) + else + return (' {0x%04x, 0x%04x},\n'):format(start, end_) + end +end + +local build_convert_table = function(ut_fp, props, cond_func, nl_index, + table_name) + ut_fp:write('static const convertStruct ' .. table_name .. '[] = {\n') + local start = -1 + local end_ = -1 + local step = 0 + local add = -1 + for _, p in ipairs(props) do + if cond_func(p) then + local n = tonumber(p[1], 16) + local nl = tonumber(p[nl_index], 16) + if start >= 0 and add == (nl - n) and (step == 0 or n - end_ == step) then + -- Continue with the same range. + step = n - end_ + end_ = n + else + if start >= 0 then + -- Produce previous range. + ut_fp:write(make_range(start, end_, step, add)) + end + start = n + end_ = n + step = 0 + add = nl - n + end + end + end + if start >= 0 then + ut_fp:write(make_range(start, end_, step, add)) + end + ut_fp:write('};\n') +end + +local build_case_table = function(ut_fp, dataprops, table_name, index) + local cond_func = function(p) + return p[index] ~= '' + end + return build_convert_table(ut_fp, dataprops, cond_func, index, + 'to' .. table_name) +end + +local build_fold_table = function(ut_fp, foldprops) + local cond_func = function(p) + return (p[2] == 'C' or p[2] == 'S') + end + return build_convert_table(ut_fp, foldprops, cond_func, 3, 'foldCase') +end + +local build_combining_table = function(ut_fp, dataprops) + ut_fp:write('static const struct interval combining[] = {\n') + local start = -1 + local end_ = -1 + for _, p in ipairs(dataprops) do + if (({Mn=true, Mc=true, Me=true})[p[3]]) then + local n = tonumber(p[1], 16) + if start >= 0 and end_ + 1 == n then + -- Continue with the same range. + end_ = n + else + if start >= 0 then + -- Produce previous range. + ut_fp:write(make_range(start, end_)) + end + start = n + end_ = n + end + end + end + if start >= 0 then + ut_fp:write(make_range(start, end_)) + end + ut_fp:write('};\n') +end + +local build_width_table = function(ut_fp, dataprops, widthprops, widths, + table_name) + ut_fp:write('static const struct interval ' .. table_name .. '[] = {\n') + local start = -1 + local end_ = -1 + local dataidx = 1 + local ret = {} + for _, p in ipairs(widthprops) do + if widths[p[2]:sub(1, 1)] then + local rng_start, rng_end = p[1]:find('%.%.') + local n, n_last + if rng_start then + -- It is a range. We don’t check for composing char then. + n = tonumber(p[1]:sub(1, rng_start - 1), 16) + n_last = tonumber(p[1]:sub(rng_end + 1), 16) + else + n = tonumber(p[1], 16) + n_last = n + end + local dn + while true do + dn = tonumber(dataprops[dataidx][1], 16) + if dn >= n then + break + end + dataidx = dataidx + 1 + end + if dn ~= n and n_last == n then + io.stderr:write('Cannot find character ' .. n .. ' in data table.\n') + end + -- Only use the char when it’s not a composing char. + -- But use all chars from a range. + local dp = dataprops[dataidx] + if (n_last > n) or (not (({Mn=true, Mc=true, Me=true})[dp[3]])) then + if start >= 0 and end_ + 1 == n then + -- Continue with the same range. + else + if start >= 0 then + ut_fp:write(make_range(start, end_)) + table.insert(ret, {start, end_}) + end + start = n + end + end_ = n_last + end + end + end + if start >= 0 then + ut_fp:write(make_range(start, end_)) + table.insert(ret, {start, end_}) + end + ut_fp:write('};\n') + return ret +end + +local build_emoji_table = function(ut_fp, emojiprops, doublewidth, ambiwidth) + local emojiwidth = {} + local emoji = {} + for _, p in ipairs(emojiprops) do + if p[2]:match('Emoji%s+#') then + local rng_start, rng_end = p[1]:find('%.%.') + if rng_start then + n = tonumber(p[1]:sub(1, rng_start - 1), 16) + n_last = tonumber(p[1]:sub(rng_end + 1), 16) + else + n = tonumber(p[1], 16) + n_last = n + end + if #emoji > 0 and n - 1 == emoji[#emoji][2] then + emoji[#emoji][2] = n_last + else + table.insert(emoji, { n, n_last }) + end + + -- Characters below 1F000 may be considered single width traditionally, + -- making them double width causes problems. + if n >= 0x1f000 then + -- exclude characters that are in the ambiguous/doublewidth table + for _, ambi in ipairs(ambiwidth) do + if n >= ambi[1] and n <= ambi[2] then + n = ambi[2] + 1 + end + if n_last >= ambi[1] and n_last <= ambi[2] then + n_last = ambi[1] - 1 + end + end + for _, double in ipairs(doublewidth) do + if n >= double[1] and n <= double[2] then + n = double[2] + 1 + end + if n_last >= double[1] and n_last <= double[2] then + n_last = double[1] - 1 + end + end + + if n <= n_last then + if #emojiwidth > 0 and n - 1 == emojiwidth[#emojiwidth][2] then + emojiwidth[#emojiwidth][2] = n_last + else + table.insert(emojiwidth, { n, n_last }) + end + end + end + end + end + + ut_fp:write('static const struct interval emoji_all[] = {\n') + for _, p in ipairs(emoji) do + ut_fp:write(make_range(p[1], p[2])) + end + ut_fp:write('};\n') + + ut_fp:write('static const struct interval emoji_width[] = {\n') + for _, p in ipairs(emojiwidth) do + ut_fp:write(make_range(p[1], p[2])) + end + ut_fp:write('};\n') +end + +local ud_fp = io.open(unicodedata_fname, 'r') +local dataprops = parse_data_to_props(ud_fp) +ud_fp:close() + +local ut_fp = io.open(utf_tables_fname, 'w') + +build_case_table(ut_fp, dataprops, 'Lower', 14) +build_case_table(ut_fp, dataprops, 'Upper', 13) +build_combining_table(ut_fp, dataprops) + +local cf_fp = io.open(casefolding_fname, 'r') +local foldprops = parse_fold_props(cf_fp) +cf_fp:close() + +build_fold_table(ut_fp, foldprops) + +local eaw_fp = io.open(eastasianwidth_fname, 'r') +local widthprops = parse_width_props(eaw_fp) +eaw_fp:close() + +local doublewidth = build_width_table(ut_fp, dataprops, widthprops, + {W=true, F=true}, 'doublewidth') +local ambiwidth = build_width_table(ut_fp, dataprops, widthprops, + {A=true}, 'ambiguous') + +local emoji_fp = io.open(emoji_fname, 'r') +local emojiprops = parse_emoji_props(emoji_fp) +emoji_fp:close() + +build_emoji_table(ut_fp, emojiprops, doublewidth, ambiwidth) + +ut_fp:close() -- cgit