generators: separate source generators from scripts

12 files changed, 1751 insertions, 15 deletions

diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index 94d1c00de9..3e0dadaa81 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -11,13 +11,14 @@ endif()
 endif()
 
 set(TOUCHES_DIR ${PROJECT_BINARY_DIR}/touches)
+set(GENERATOR_DIR ${CMAKE_CURRENT_LIST_DIR}/generators)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
-set(MSGPACK_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genmsgpack.lua)
-set(API_UI_EVENTS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gen_api_ui_events.lua)
+set(API_DISPATCH_GENERATOR ${GENERATOR_DIR}/gen_api_dispatch.lua)
+set(API_UI_EVENTS_GENERATOR ${GENERATOR_DIR}/gen_api_ui_events.lua)
 set(API_METADATA ${PROJECT_BINARY_DIR}/api_metadata.mpack)
 set(FUNCS_DATA ${PROJECT_BINARY_DIR}/funcs_data.mpack)
 set(MSGPACK_LUA_C_BINDINGS ${GENERATED_DIR}/msgpack_lua_c_bindings.generated.c)
-set(HEADER_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendeclarations.lua)
+set(HEADER_GENERATOR ${GENERATOR_DIR}/gen_declarations.lua)
 set(GENERATED_INCLUDES_DIR ${PROJECT_BINARY_DIR}/include)
 set(GENERATED_API_DISPATCH ${GENERATED_DIR}/api/private/dispatch_wrappers.generated.h)
 set(GENERATED_FUNCS_METADATA ${GENERATED_DIR}/api/private/funcs_metadata.generated.h)
@@ -31,16 +32,16 @@ set(GENERATED_FUNCS ${GENERATED_DIR}/funcs.generated.h)
 set(GENERATED_EVENTS_ENUM ${GENERATED_INCLUDES_DIR}/auevents_enum.generated.h)
 set(GENERATED_EVENTS_NAMES_MAP ${GENERATED_DIR}/auevents_name_map.generated.h)
 set(GENERATED_OPTIONS ${GENERATED_DIR}/options.generated.h)
-set(EX_CMDS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genex_cmds.lua)
-set(FUNCS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/geneval.lua)
-set(EVENTS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gen_events.lua)
-set(OPTIONS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genoptions.lua)
-set(UNICODE_TABLES_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genunicodetables.lua)
+set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua)
+set(FUNCS_GENERATOR ${GENERATOR_DIR}/gen_eval.lua)
+set(EVENTS_GENERATOR ${GENERATOR_DIR}/gen_events.lua)
+set(OPTIONS_GENERATOR ${GENERATOR_DIR}/gen_options.lua)
+set(UNICODE_TABLES_GENERATOR ${GENERATOR_DIR}/gen_unicode_tables.lua)
 set(UNICODE_DIR ${PROJECT_SOURCE_DIR}/unicode)
 set(GENERATED_UNICODE_TABLES ${GENERATED_DIR}/unicode_tables.generated.h)
 set(VIM_MODULE_FILE ${GENERATED_DIR}/lua/vim_module.generated.h)
 set(VIM_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/src/nvim/lua/vim.lua)
-set(CHAR_BLOB_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gencharblob.lua)
+set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua)
 set(LINT_SUPPRESS_FILE ${PROJECT_BINARY_DIR}/errors.json)
 set(LINT_SUPPRESS_URL_BASE "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint")
 set(LINT_SUPPRESS_URL "${LINT_SUPPRESS_URL_BASE}/errors.json")
@@ -238,7 +239,7 @@ add_custom_command(OUTPUT ${GENERATED_UNICODE_TABLES}
 add_custom_command(
   OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_FUNCS_METADATA}
          ${API_METADATA} ${MSGPACK_LUA_C_BINDINGS}
-  COMMAND ${LUA_PRG} ${MSGPACK_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
+  COMMAND ${LUA_PRG} ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
                      ${GENERATED_API_DISPATCH}
                      ${GENERATED_FUNCS_METADATA} ${API_METADATA}
                      ${MSGPACK_LUA_C_BINDINGS}
@@ -246,7 +247,7 @@ add_custom_command(
   DEPENDS
     ${API_HEADERS}
     ${MSGPACK_RPC_HEADERS}
-    ${MSGPACK_GENERATOR}
+    ${API_DISPATCH_GENERATOR}
     ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
 )
 
@@ -263,10 +264,11 @@ list(APPEND NVIM_GENERATED_SOURCES
   "${MSGPACK_LUA_C_BINDINGS}"
 )
 
-add_custom_command(OUTPUT ${GENERATED_UI_EVENTS}
-                          ${GENERATED_UI_EVENTS_CALL}
-                          ${GENERATED_UI_EVENTS_REMOTE}
-                          ${GENERATED_UI_EVENTS_BRIDGE}
+add_custom_command(
+  OUTPUT ${GENERATED_UI_EVENTS}
+         ${GENERATED_UI_EVENTS_CALL}
+         ${GENERATED_UI_EVENTS_REMOTE}
+         ${GENERATED_UI_EVENTS_BRIDGE}
   COMMAND ${LUA_PRG} ${API_UI_EVENTS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
                      ${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h
                      ${GENERATED_UI_EVENTS}
diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
new file mode 100644
index 0000000000..5c52ec5930
--- /dev/null
+++ b/src/nvim/generators/c_grammar.lua
@@ -0,0 +1,51 @@
+lpeg = require('lpeg')
+
+-- lpeg grammar for building api metadata from a set of header files. It
+-- ignores comments and preprocessor commands and parses a very small subset
+-- of C prototypes with a limited set of types
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
+
+local any = P(1) -- (consume one character)
+local letter = R('az', 'AZ') + S('_$')
+local num = R('09')
+local alpha = letter + num
+local nl = P('\r\n') + P('\n')
+local not_nl = any - nl
+local ws = S(' \t') + nl
+local fill = ws ^ 0
+local c_comment = P('//') * (not_nl ^ 0)
+local c_preproc = P('#') * (not_nl ^ 0)
+local typed_container =
+  (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')')
+local c_id = (
+  typed_container +
+  (letter * (alpha ^ 0))
+)
+local c_void = P('void')
+local c_param_type = (
+  ((P('Error') * fill * P('*') * fill) * Cc('error')) +
+  (C(c_id) * (ws ^ 1))
+  )
+local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
+local c_param = Ct(c_param_type * C(c_id))
+local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
+local c_params = Ct(c_void + c_param_list)
+local c_proto = Ct(
+  Cg(c_type, 'return_type') * Cg(c_id, 'name') *
+  fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') *
+  Cg(Cc(false), 'async') *
+  (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) *
+  (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(num ^ 1)) * P(')'),
+              'deprecated_since') ^ -1) *
+  (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) *
+  (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) *
+  (fill * Cg((P('FUNC_API_NOEVAL') * Cc(true)), 'noeval') ^ -1) *
+  (fill * Cg((P('REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) *
+  (fill * Cg((P('REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) *
+  (fill * Cg((P('BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) *
+  fill * P(';')
+  )
+
+local grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1)
+return {grammar=grammar, typed_container=typed_container}
diff --git a/src/nvim/generators/dump_bin_array.lua b/src/nvim/generators/dump_bin_array.lua
new file mode 100644
index 0000000000..bee5aba73f
--- /dev/null
+++ b/src/nvim/generators/dump_bin_array.lua
@@ -0,0 +1,17 @@
+local function dump_bin_array(output, name, data)
+  output:write([[
+  static const uint8_t ]]..name..[[[] = {
+]])
+
+  for i = 1, #data do
+    output:write(string.byte(data, i)..', ')
+    if i % 10 == 0 then
+      output:write('\n  ')
+    end
+  end
+  output:write([[
+};
+]])
+end
+
+return dump_bin_array
diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
new file mode 100644
index 0000000000..7adefbfc15
--- /dev/null
+++ b/src/nvim/generators/gen_api_dispatch.lua
@@ -0,0 +1,458 @@
+mpack = require('mpack')
+
+-- we need at least 4 arguments since the last two are output files
+if arg[1] == '--help' then
+  print('Usage: genmsgpack.lua args')
+  print('Args: 1: source directory')
+  print('      2: dispatch output file (dispatch_wrappers.generated.h)')
+  print('      3: functions metadata output file (funcs_metadata.generated.h)')
+  print('      4: API metadata output file (api_metadata.mpack)')
+  print('      5: lua C bindings output file (msgpack_lua_c_bindings.generated.c)')
+  print('      rest: C files where API functions are defined')
+end
+assert(#arg >= 4)
+functions = {}
+
+local nvimdir = arg[1]
+package.path = nvimdir .. '/?.lua;' .. package.path
+
+-- names of all headers relative to the source root (for inclusion in the
+-- generated file)
+headers = {}
+
+-- output h file with generated dispatch functions
+dispatch_outputf = arg[2]
+-- output h file with packed metadata
+funcs_metadata_outputf = arg[3]
+-- output metadata mpack file, for use by other build scripts
+mpack_outputf = arg[4]
+lua_c_bindings_outputf = arg[5]
+
+-- set of function names, used to detect duplicates
+function_names = {}
+
+c_grammar = require('generators.c_grammar')
+
+-- read each input file, parse and append to the api metadata
+for i = 6, #arg do
+  local full_path = arg[i]
+  local parts = {}
+  for part in string.gmatch(full_path, '[^/]+') do
+    parts[#parts + 1] = part
+  end
+  headers[#headers + 1] = parts[#parts - 1]..'/'..parts[#parts]
+
+  local input = io.open(full_path, 'rb')
+
+  local tmp = c_grammar.grammar:match(input:read('*all'))
+  for i = 1, #tmp do
+    local fn = tmp[i]
+    if not fn.noexport then
+      functions[#functions + 1] = tmp[i]
+      function_names[fn.name] = true
+      if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
+        -- this function should receive the channel id
+        fn.receives_channel_id = true
+        -- remove the parameter since it won't be passed by the api client
+        table.remove(fn.parameters, 1)
+      end
+      if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then
+        -- function can fail if the last parameter type is 'Error'
+        fn.can_fail = true
+        -- remove the error parameter, msgpack has it's own special field
+        -- for specifying errors
+        fn.parameters[#fn.parameters] = nil
+      end
+    end
+  end
+  input:close()
+end
+
+local function shallowcopy(orig)
+  local copy = {}
+  for orig_key, orig_value in pairs(orig) do
+    copy[orig_key] = orig_value
+  end
+  return copy
+end
+
+local function startswith(String,Start)
+  return string.sub(String,1,string.len(Start))==Start
+end
+
+-- Export functions under older deprecated names.
+-- These will be removed eventually.
+local deprecated_aliases = require("api.dispatch_deprecated")
+for i,f in ipairs(shallowcopy(functions)) do
+  local ismethod = false
+  if startswith(f.name, "nvim_") then
+    if startswith(f.name, "nvim__") then
+      f.since = -1
+    elseif f.since == nil then
+      print("Function "..f.name.." lacks since field.\n")
+      os.exit(1)
+    end
+    f.since = tonumber(f.since)
+    if f.deprecated_since ~= nil then
+      f.deprecated_since = tonumber(f.deprecated_since)
+    end
+
+    if startswith(f.name, "nvim_buf_") then
+      ismethod = true
+    elseif startswith(f.name, "nvim_win_") then
+      ismethod = true
+    elseif startswith(f.name, "nvim_tabpage_") then
+      ismethod = true
+    end
+  else
+    f.noeval = true
+    f.since = 0
+    f.deprecated_since = 1
+  end
+  f.method = ismethod
+  local newname = deprecated_aliases[f.name]
+  if newname ~= nil then
+    if function_names[newname] then
+      -- duplicate
+      print("Function "..f.name.." has deprecated alias\n"
+            ..newname.." which has a separate implementation.\n"..
+            "Please remove it from src/nvim/api/dispatch_deprecated.lua")
+      os.exit(1)
+    end
+    local newf = shallowcopy(f)
+    newf.name = newname
+    if newname == "ui_try_resize" then
+      -- The return type was incorrectly set to Object in 0.1.5.
+      -- Keep it that way for clients that rely on this.
+      newf.return_type = "Object"
+    end
+    newf.impl_name = f.name
+    newf.noeval = true
+    newf.since = 0
+    newf.deprecated_since = 1
+    functions[#functions+1] = newf
+  end
+end
+
+-- don't expose internal attributes like "impl_name" in public metadata
+exported_attributes = {'name', 'parameters', 'return_type', 'method',
+                       'since', 'deprecated_since'}
+exported_functions = {}
+for _,f in ipairs(functions) do
+  if not startswith(f.name, "nvim__") then
+    local f_exported = {}
+    for _,attr in ipairs(exported_attributes) do
+      f_exported[attr] = f[attr]
+    end
+    exported_functions[#exported_functions+1] = f_exported
+  end
+end
+
+
+-- serialize the API metadata using msgpack and embed into the resulting
+-- binary for easy querying by clients
+funcs_metadata_output = io.open(funcs_metadata_outputf, 'wb')
+packed = mpack.pack(exported_functions)
+dump_bin_array = require("generators.dump_bin_array")
+dump_bin_array(funcs_metadata_output, 'funcs_metadata', packed)
+funcs_metadata_output:close()
+
+-- start building the dispatch wrapper output
+output = io.open(dispatch_outputf, 'wb')
+
+local function real_type(type)
+  local rv = type
+  if c_grammar.typed_container:match(rv) then
+    if rv:match('Array') then
+      rv = 'Array'
+    else
+      rv = 'Dictionary'
+    end
+  end
+  return rv
+end
+
+local function attr_name(rt)
+  if rt == 'Float' then
+    return 'floating'
+  else
+    return rt:lower()
+  end
+end
+
+-- start the handler functions. Visit each function metadata to build the
+-- handler function with code generated for validating arguments and calling to
+-- the real API.
+for i = 1, #functions do
+  local fn = functions[i]
+  if fn.impl_name == nil then
+    local args = {}
+
+    output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)')
+    output:write('\n{')
+    output:write('\n  Object ret = NIL;')
+    -- Declare/initialize variables that will hold converted arguments
+    for j = 1, #fn.parameters do
+      local param = fn.parameters[j]
+      local converted = 'arg_'..j
+      output:write('\n  '..param[1]..' '..converted..';')
+    end
+    output:write('\n')
+    output:write('\n  if (args.size != '..#fn.parameters..') {')
+    output:write('\n    api_set_error(error, kErrorTypeException, "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);')
+    output:write('\n    goto cleanup;')
+    output:write('\n  }\n')
+
+    -- Validation/conversion for each argument
+    for j = 1, #fn.parameters do
+      local converted, convert_arg, param, arg
+      param = fn.parameters[j]
+      converted = 'arg_'..j
+      local rt = real_type(param[1])
+      if rt ~= 'Object' then
+        if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then
+          -- Buffer, Window, and Tabpage have a specific type, but are stored in integer
+          output:write('\n  if (args.items['..(j - 1)..'].type == kObjectType'..rt..' && args.items['..(j - 1)..'].data.integer >= 0) {')
+          output:write('\n    '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;')
+        else
+          output:write('\n  if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {')
+          output:write('\n    '..converted..' = args.items['..(j - 1)..'].data.'..attr_name(rt)..';')
+        end
+        if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then
+          -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages
+          output:write('\n  } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer >= 0) {')
+          output:write('\n    '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;')
+        end
+        output:write('\n  } else {')
+        output:write('\n    api_set_error(error, kErrorTypeException, "Wrong type for argument '..j..', expecting '..param[1]..'");')
+        output:write('\n    goto cleanup;')
+        output:write('\n  }\n')
+      else
+        output:write('\n  '..converted..' = args.items['..(j - 1)..'];\n')
+      end
+
+      args[#args + 1] = converted
+    end
+
+    -- function call
+    local call_args = table.concat(args, ', ')
+    output:write('\n  ')
+    if fn.return_type ~= 'void' then
+      -- has a return value, prefix the call with a declaration
+      output:write(fn.return_type..' rv = ')
+    end
+
+    -- write the function name and the opening parenthesis
+    output:write(fn.name..'(')
+
+    if fn.receives_channel_id then
+      -- if the function receives the channel id, pass it as first argument
+      if #args > 0 or fn.can_fail then
+        output:write('channel_id, '..call_args)
+      else
+        output:write('channel_id')
+      end
+    else
+      output:write(call_args)
+    end
+
+    if fn.can_fail then
+      -- if the function can fail, also pass a pointer to the local error object
+      if #args > 0 then
+        output:write(', error);\n')
+      else
+        output:write('error);\n')
+      end
+      -- and check for the error
+      output:write('\n  if (ERROR_SET(error)) {')
+      output:write('\n    goto cleanup;')
+      output:write('\n  }\n')
+    else
+      output:write(');\n')
+    end
+
+    if fn.return_type ~= 'void' then
+      output:write('\n  ret = '..string.upper(real_type(fn.return_type))..'_OBJ(rv);')
+    end
+    output:write('\n\ncleanup:');
+
+    output:write('\n  return ret;\n}\n\n');
+  end
+end
+
+-- Generate a function that initializes method names with handler functions
+output:write([[
+void msgpack_rpc_init_method_table(void)
+{
+  methods = map_new(String, MsgpackRpcRequestHandler)();
+
+]])
+
+for i = 1, #functions do
+  local fn = functions[i]
+  output:write('  msgpack_rpc_add_method_handler('..
+               '(String) {.data = "'..fn.name..'", '..
+               '.size = sizeof("'..fn.name..'") - 1}, '..
+               '(MsgpackRpcRequestHandler) {.fn = handle_'..  (fn.impl_name or fn.name)..
+               ', .async = '..tostring(fn.async)..'});\n')
+
+end
+
+output:write('\n}\n\n')
+output:close()
+
+mpack_output = io.open(mpack_outputf, 'wb')
+mpack_output:write(mpack.pack(functions))
+mpack_output:close()
+
+local function include_headers(output, headers)
+  for i = 1, #headers do
+    if headers[i]:sub(-12) ~= '.generated.h' then
+      output:write('\n#include "nvim/'..headers[i]..'"')
+    end
+  end
+end
+
+local function write_shifted_output(output, str)
+  str = str:gsub('\n  ', '\n')
+  str = str:gsub('^  ', '')
+  str = str:gsub(' +$', '')
+  output:write(str)
+end
+
+-- start building lua output
+output = io.open(lua_c_bindings_outputf, 'wb')
+
+output:write([[
+#include <lua.h>
+#include <lualib.h>
+#include <lauxlib.h>
+
+#include "nvim/func_attr.h"
+#include "nvim/api/private/defs.h"
+#include "nvim/api/private/helpers.h"
+#include "nvim/lua/converter.h"
+]])
+include_headers(output, headers)
+output:write('\n')
+
+lua_c_functions = {}
+
+local function process_function(fn)
+  lua_c_function_name = ('nlua_msgpack_%s'):format(fn.name)
+  write_shifted_output(output, string.format([[
+
+  static int %s(lua_State *lstate)
+  {
+    Error err = ERROR_INIT;
+    if (lua_gettop(lstate) != %i) {
+      api_set_error(&err, kErrorTypeValidation, "Expected %i argument%s");
+      goto exit_0;
+    }
+  ]], lua_c_function_name, #fn.parameters, #fn.parameters,
+      (#fn.parameters == 1) and '' or 's'))
+  lua_c_functions[#lua_c_functions + 1] = {
+    binding=lua_c_function_name,
+    api=fn.name
+  }
+  local cparams = ''
+  local free_code = {}
+  for j = #fn.parameters,1,-1 do
+    param = fn.parameters[j]
+    cparam = string.format('arg%u', j)
+    param_type = real_type(param[1])
+    lc_param_type = param_type:lower()
+    write_shifted_output(output, string.format([[
+    const %s %s = nlua_pop_%s(lstate, &err);
+
+    if (ERROR_SET(&err)) {
+      goto exit_%u;
+    }
+    ]], param[1], cparam, param_type, #fn.parameters - j))
+    free_code[#free_code + 1] = ('api_free_%s(%s);'):format(
+      lc_param_type, cparam)
+    cparams = cparam .. ', ' .. cparams
+  end
+  if fn.receives_channel_id then
+    cparams = 'LUA_INTERNAL_CALL, ' .. cparams
+  end
+  if fn.can_fail then
+    cparams = cparams .. '&err'
+  else
+    cparams = cparams:gsub(', $', '')
+  end
+  local free_at_exit_code = ''
+  for i = 1, #free_code do
+    local rev_i = #free_code - i + 1
+    local code = free_code[rev_i]
+    if i == 1 then
+      free_at_exit_code = free_at_exit_code .. ('\n    %s'):format(code)
+    else
+      free_at_exit_code = free_at_exit_code .. ('\n  exit_%u:\n    %s'):format(
+        rev_i, code)
+    end
+  end
+  local err_throw_code = [[
+
+  exit_0:
+    if (ERROR_SET(&err)) {
+      luaL_where(lstate, 1);
+      lua_pushstring(lstate, err.msg);
+      api_clear_error(&err);
+      lua_concat(lstate, 2);
+      return lua_error(lstate);
+    }
+  ]]
+  if fn.return_type ~= 'void' then
+    if fn.return_type:match('^ArrayOf') then
+      return_type = 'Array'
+    else
+      return_type = fn.return_type
+    end
+    write_shifted_output(output, string.format([[
+    const %s ret = %s(%s);
+    nlua_push_%s(lstate, ret);
+    api_free_%s(ret);
+  %s
+  %s
+    return 1;
+    ]], fn.return_type, fn.name, cparams, return_type, return_type:lower(),
+        free_at_exit_code, err_throw_code))
+  else
+    write_shifted_output(output, string.format([[
+    %s(%s);
+  %s
+  %s
+    return 0;
+    ]], fn.name, cparams, free_at_exit_code, err_throw_code))
+  end
+  write_shifted_output(output, [[
+  }
+  ]])
+end
+
+for _, fn in ipairs(functions) do
+  if not fn.noeval or fn.name:sub(1, 4) == '_vim' then
+    process_function(fn)
+  end
+end
+
+output:write(string.format([[
+void nlua_add_api_functions(lua_State *lstate)
+  FUNC_ATTR_NONNULL_ALL
+{
+  lua_createtable(lstate, 0, %u);
+]], #lua_c_functions))
+for _, func in ipairs(lua_c_functions) do
+  output:write(string.format([[
+
+  lua_pushcfunction(lstate, &%s);
+  lua_setfield(lstate, -2, "%s");]], func.binding, func.api))
+end
+output:write([[
+
+  lua_setfield(lstate, -2, "api");
+}
+]])
+
+output:close()
diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/nvim/generators/gen_api_ui_events.lua
new file mode 100644
index 0000000000..28007d2266
--- /dev/null
+++ b/src/nvim/generators/gen_api_ui_events.lua
@@ -0,0 +1,136 @@
+mpack = require('mpack')
+
+local nvimdir = arg[1]
+package.path = nvimdir .. '/?.lua;' .. package.path
+
+assert(#arg == 6)
+input = io.open(arg[2], 'rb')
+proto_output = io.open(arg[3], 'wb')
+call_output = io.open(arg[4], 'wb')
+remote_output = io.open(arg[5], 'wb')
+bridge_output = io.open(arg[6], 'wb')
+
+c_grammar = require('generators.c_grammar')
+local events = c_grammar.grammar:match(input:read('*all'))
+
+function write_signature(output, ev, prefix, notype)
+  output:write('('..prefix)
+  if prefix == "" and #ev.parameters == 0 then
+    output:write('void')
+  end
+  for j = 1, #ev.parameters do
+    if j > 1 or prefix ~= '' then
+      output:write(', ')
+    end
+    local param = ev.parameters[j]
+    if not notype then
+      output:write(param[1]..' ')
+    end
+    output:write(param[2])
+  end
+  output:write(')')
+end
+
+function write_arglist(output, ev, need_copy)
+  output:write('  Array args = ARRAY_DICT_INIT;\n')
+  for j = 1, #ev.parameters do
+    local param = ev.parameters[j]
+    local kind = string.upper(param[1])
+    local do_copy = need_copy and (kind == "ARRAY" or kind == "DICTIONARY" or kind == "STRING")
+    output:write('  ADD(args, ')
+    if do_copy then
+      output:write('copy_object(')
+    end
+    output:write(kind..'_OBJ('..param[2]..')')
+    if do_copy then
+      output:write(')')
+    end
+    output:write(');\n')
+  end
+end
+
+for i = 1, #events do
+  ev = events[i]
+  assert(ev.return_type == 'void')
+
+  if not ev.remote_only then
+    proto_output:write('  void (*'..ev.name..')')
+    write_signature(proto_output, ev, 'UI *ui')
+    proto_output:write(';\n')
+
+    if not ev.remote_impl then
+      remote_output:write('static void remote_ui_'..ev.name)
+      write_signature(remote_output, ev, 'UI *ui')
+      remote_output:write('\n{\n')
+      write_arglist(remote_output, ev, true)
+      remote_output:write('  push_call(ui, "'..ev.name..'", args);\n')
+      remote_output:write('}\n\n')
+    end
+
+    if not ev.bridge_impl then
+
+      send, argv, recv, recv_argv, recv_cleanup = '', '', '', '', ''
+      argc = 1
+      for j = 1, #ev.parameters do
+        local param = ev.parameters[j]
+        copy = 'copy_'..param[2]
+        if param[1] == 'String' then
+          send = send..'  String copy_'..param[2]..' = copy_string('..param[2]..');\n'
+          argv = argv..', '..copy..'.data, INT2PTR('..copy..'.size)'
+          recv = (recv..'  String '..param[2]..
+                          ' = (String){.data = argv['..argc..'],'..
+                          '.size = (size_t)argv['..(argc+1)..']};\n')
+          recv_argv = recv_argv..', '..param[2]
+          recv_cleanup = recv_cleanup..'  api_free_string('..param[2]..');\n'
+          argc = argc+2
+        elseif param[1] == 'Array' then
+          send = send..'  Array copy_'..param[2]..' = copy_array('..param[2]..');\n'
+          argv = argv..', '..copy..'.items, INT2PTR('..copy..'.size)'
+          recv = (recv..'  Array '..param[2]..
+                          ' = (Array){.items = argv['..argc..'],'..
+                          '.size = (size_t)argv['..(argc+1)..']};\n')
+          recv_argv = recv_argv..', '..param[2]
+          recv_cleanup = recv_cleanup..'  api_free_array('..param[2]..');\n'
+          argc = argc+2
+        elseif param[1] == 'Integer' or param[1] == 'Boolean' then
+          argv = argv..', INT2PTR('..param[2]..')'
+          recv_argv = recv_argv..', PTR2INT(argv['..argc..'])'
+          argc = argc+1
+        else
+          assert(false)
+        end
+      end
+      bridge_output:write('static void ui_bridge_'..ev.name..
+                          '_event(void **argv)\n{\n')
+      bridge_output:write('  UI *ui = UI(argv[0]);\n')
+      bridge_output:write(recv)
+      bridge_output:write('  ui->'..ev.name..'(ui'..recv_argv..');\n')
+      bridge_output:write(recv_cleanup)
+      bridge_output:write('}\n\n')
+
+      bridge_output:write('static void ui_bridge_'..ev.name)
+      write_signature(bridge_output, ev, 'UI *ui')
+      bridge_output:write('\n{\n')
+      bridge_output:write(send)
+      bridge_output:write('  UI_CALL(ui, '..ev.name..', '..argc..', ui'..argv..');\n}\n')
+    end
+  end
+
+  call_output:write('void ui_call_'..ev.name)
+  write_signature(call_output, ev, '')
+  call_output:write('\n{\n')
+  if ev.remote_only then
+    write_arglist(call_output, ev, false)
+    call_output:write('  ui_event("'..ev.name..'", args);\n')
+  else
+    call_output:write('  UI_CALL')
+    write_signature(call_output, ev, ev.name, true)
+    call_output:write(";\n")
+  end
+  call_output:write("}\n\n")
+
+end
+
+proto_output:close()
+call_output:close()
+remote_output:close()
diff --git a/src/nvim/generators/gen_char_blob.lua b/src/nvim/generators/gen_char_blob.lua
new file mode 100644
index 0000000000..d860375e26
--- /dev/null
+++ b/src/nvim/generators/gen_char_blob.lua
@@ -0,0 +1,48 @@
+if arg[1] == '--help' then
+  print('Usage:')
+  print('  gencharblob.lua source target varname')
+  print('')
+  print('Generates C file with big uint8_t blob.')
+  print('Blob will be stored in a static const array named varname.')
+  os.exit()
+end
+
+assert(#arg == 3)
+
+local source_file = arg[1]
+local target_file = arg[2]
+local varname = arg[3]
+
+source = io.open(source_file, 'r')
+target = io.open(target_file, 'w')
+
+target:write('#include <stdint.h>\n\n')
+target:write(('static const uint8_t %s[] = {\n'):format(varname))
+
+num_bytes = 0
+MAX_NUM_BYTES = 15  -- 78 / 5: maximum number of bytes on one line
+target:write(' ')
+
+increase_num_bytes = function()
+  num_bytes = num_bytes + 1
+  if num_bytes == MAX_NUM_BYTES then
+    num_bytes = 0
+    target:write('\n ')
+  end
+end
+
+for line in source:lines() do
+  for i = 1,string.len(line) do
+    byte = string.byte(line, i)
+    assert(byte ~= 0)
+    target:write(string.format(' %3u,', byte))
+    increase_num_bytes()
+  end
+  target:write(string.format(' %3u,', string.byte('\n', 1)))
+  increase_num_bytes()
+end
+
+target:write('   0};\n')
+
+source:close()
+target:close()
diff --git a/src/nvim/generators/gen_declarations.lua b/src/nvim/generators/gen_declarations.lua
new file mode 100755
index 0000000000..e999e53e4a
--- /dev/null
+++ b/src/nvim/generators/gen_declarations.lua
@@ -0,0 +1,287 @@
+#!/usr/bin/lua
+
+local fname = arg[1]
+local static_fname = arg[2]
+local non_static_fname = arg[3]
+local preproc_fname = arg[4]
+
+
+local lpeg = require('lpeg')
+
+local fold = function (func, ...)
+  local result = nil
+  for i, v in ipairs({...}) do
+    if result == nil then
+      result = v
+    else
+      result = func(result, v)
+    end
+  end
+  return result
+end
+
+local folder = function (func)
+  return function (...)
+    return fold(func, ...)
+  end
+end
+
+local lit = lpeg.P
+local set = function(...)
+  return lpeg.S(fold(function (a, b) return a .. b end, ...))
+end
+local any_character = lpeg.P(1)
+local rng = function(s, e) return lpeg.R(s .. e) end
+local concat = folder(function (a, b) return a * b end)
+local branch = folder(function (a, b) return a + b end)
+local one_or_more = function(v) return v ^ 1 end
+local two_or_more = function(v) return v ^ 2 end
+local any_amount = function(v) return v ^ 0 end
+local one_or_no = function(v) return v ^ -1 end
+local look_behind = lpeg.B
+local look_ahead = function(v) return #v end
+local neg_look_ahead = function(v) return -v end
+local neg_look_behind = function(v) return -look_behind(v) end
+
+local w = branch(
+  rng('a', 'z'),
+  rng('A', 'Z'),
+  lit('_')
+)
+local aw = branch(
+  w,
+  rng('0', '9')
+)
+local s = set(' ', '\n', '\t')
+local raw_word = concat(w, any_amount(aw))
+local right_word = concat(
+  raw_word,
+  neg_look_ahead(aw)
+)
+local word = branch(
+  concat(
+    branch(lit('ArrayOf('), lit('DictionaryOf(')), -- typed container macro
+    one_or_more(any_character - lit(')')),
+    lit(')')
+  ),
+  concat(
+    neg_look_behind(aw),
+    right_word
+  )
+)
+local inline_comment = concat(
+  lit('/*'),
+  any_amount(concat(
+    neg_look_ahead(lit('*/')),
+    any_character
+  )),
+  lit('*/')
+)
+local spaces = any_amount(branch(
+  s,
+  -- Comments are really handled by preprocessor, so the following is not needed
+  inline_comment,
+  concat(
+    lit('//'),
+    any_amount(concat(
+      neg_look_ahead(lit('\n')),
+      any_character
+    )),
+    lit('\n')
+  ),
+  -- Linemarker inserted by preprocessor
+  concat(
+    lit('# '),
+    any_amount(concat(
+      neg_look_ahead(lit('\n')),
+      any_character
+    )),
+    lit('\n')
+  )
+))
+local typ_part = concat(
+  word,
+  any_amount(concat(
+    spaces,
+    lit('*')
+  )),
+  spaces
+)
+local typ = one_or_more(typ_part)
+local typ_id = two_or_more(typ_part)
+local arg = typ_id         -- argument name is swallowed by typ
+local pattern = concat(
+  any_amount(branch(set(' ', '\t'), inline_comment)),
+  typ_id,                  -- return type with function name
+  spaces,
+  lit('('),
+  spaces,
+  one_or_no(branch(        -- function arguments
+    concat(
+      arg,                 -- first argument, does not require comma
+      any_amount(concat(   -- following arguments, start with a comma
+        spaces,
+        lit(','),
+        spaces,
+        arg,
+        any_amount(concat(
+          lit('['),
+          spaces,
+          any_amount(aw),
+          spaces,
+          lit(']')
+        ))
+      )),
+      one_or_no(concat(
+        spaces,
+        lit(','),
+        spaces,
+        lit('...')
+      ))
+    ),