From c0d60526541a3cf977ae623471ae4a347b492af1 Mon Sep 17 00:00:00 2001 From: bfredl Date: Tue, 23 Aug 2022 09:33:08 +0200 Subject: perf(api): allow to use an arena for return values --- src/nvim/generators/gen_api_dispatch.lua | 43 ++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) (limited to 'src/nvim/generators/gen_api_dispatch.lua') diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index b167767f7a..e4798bf5ce 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -17,6 +17,7 @@ local nvimdir = arg[1] package.path = nvimdir .. '/?.lua;' .. package.path _G.vim = loadfile(nvimdir..'/../../runtime/lua/vim/shared.lua')() +_G.vim.inspect = loadfile(nvimdir..'/../../runtime/lua/vim/inspect.lua')() local hashy = require'generators.hashy' @@ -72,6 +73,11 @@ for i = 6, #arg do -- for specifying errors fn.parameters[#fn.parameters] = nil end + if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'arena' then + -- return value is allocated in an arena + fn.arena_return = true + fn.parameters[#fn.parameters] = nil + end end end input:close() @@ -210,7 +216,7 @@ for i = 1, #functions do if fn.impl_name == nil and fn.remote then local args = {} - output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)') + output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Arena* arena, Error *error)') output:write('\n{') output:write('\n#if MIN_LOG_LEVEL <= LOGLVL_DBG') output:write('\n logmsg(LOGLVL_DBG, "RPC: ", NULL, -1, true, "ch %" PRIu64 ": invoke ' @@ -319,6 +325,10 @@ for i = 1, #functions do output:write(call_args) end + if fn.arena_return then + output:write(', arena') + end + if fn.can_fail then -- if the function can fail, also pass a pointer to the local error object if #args > 0 then @@ -355,11 +365,12 @@ local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", vim.t return "method_handlers["..idx.."].name" end) -output:write("static const MsgpackRpcRequestHandler method_handlers[] = {\n") -for _, name in ipairs(hashorder) do +output:write("const MsgpackRpcRequestHandler method_handlers[] = {\n") +for n, name in ipairs(hashorder) do local fn = remote_fns[name] + fn.handler_id = n-1 output:write(' { .name = "'..name..'", .fn = handle_'.. (fn.impl_name or fn.name).. - ', .fast = '..tostring(fn.fast)..'},\n') + ', .fast = '..tostring(fn.fast)..', .arena_return = '..tostring(not not fn.arena_return)..'},\n') end output:write("};\n\n") output:write(hashfun) @@ -400,6 +411,10 @@ output:write([[ #include "nvim/api/private/helpers.h" #include "nvim/lua/converter.h" #include "nvim/lua/executor.h" +#include "nvim/memory.h" + +static ArenaMem lua_reuse_blk = { 0 }; + ]]) include_headers(output, headers) output:write('\n') @@ -477,6 +492,14 @@ local function process_function(fn) if fn.receives_channel_id then cparams = 'LUA_INTERNAL_CALL, ' .. cparams end + if fn.arena_return then + cparams = cparams .. '&arena, ' + write_shifted_output(output, [[ + Arena arena = ARENA_EMPTY; + arena_start(&arena, &lua_reuse_blk); + ]]) + end + if fn.can_fail then cparams = cparams .. '&err' else @@ -511,15 +534,21 @@ local function process_function(fn) else return_type = fn.return_type end + local free_retval + if fn.arena_return then + free_retval = "arena_mem_free(arena_finish(&arena), &lua_reuse_blk);" + else + free_retval = "api_free_"..return_type:lower().."(ret);" + end write_shifted_output(output, string.format([[ const %s ret = %s(%s); nlua_push_%s(lstate, ret, true); - api_free_%s(ret); + %s %s %s return 1; - ]], fn.return_type, fn.name, cparams, return_type, return_type:lower(), - free_at_exit_code, err_throw_code)) + ]], fn.return_type, fn.name, cparams, return_type, + free_retval, free_at_exit_code, err_throw_code)) else write_shifted_output(output, string.format([[ %s(%s); -- cgit From bcf5ee328e228d5a536b4de2069a79234f9f3e9e Mon Sep 17 00:00:00 2001 From: bfredl Date: Tue, 23 Aug 2022 10:36:46 +0200 Subject: refactor(arena): use a shared block freelist This is both simpler in client code and more effective (always reuse block hottest in cache) --- src/nvim/generators/gen_api_dispatch.lua | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src/nvim/generators/gen_api_dispatch.lua') diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index e4798bf5ce..67b8f5f0f5 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -413,8 +413,6 @@ output:write([[ #include "nvim/lua/executor.h" #include "nvim/memory.h" -static ArenaMem lua_reuse_blk = { 0 }; - ]]) include_headers(output, headers) output:write('\n') @@ -496,7 +494,6 @@ local function process_function(fn) cparams = cparams .. '&arena, ' write_shifted_output(output, [[ Arena arena = ARENA_EMPTY; - arena_start(&arena, &lua_reuse_blk); ]]) end @@ -536,7 +533,7 @@ local function process_function(fn) end local free_retval if fn.arena_return then - free_retval = "arena_mem_free(arena_finish(&arena), &lua_reuse_blk);" + free_retval = "arena_mem_free(arena_finish(&arena));" else free_retval = "api_free_"..return_type:lower().."(ret);" end -- cgit From f8c671827710c6e9cca3bfd60c32098b2be8239a Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Mon, 14 Nov 2022 18:04:36 +0000 Subject: feat(lua-api): avoid unnecessary allocations (#19877) Lua makes (or reuses) an internal copy of strings, so we can safely push buf pointers onto the stack. --- src/nvim/generators/gen_api_dispatch.lua | 34 ++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) (limited to 'src/nvim/generators/gen_api_dispatch.lua') diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index 67b8f5f0f5..6894b7a6df 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -78,6 +78,10 @@ for i = 6, #arg do fn.arena_return = true fn.parameters[#fn.parameters] = nil end + if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'lstate' then + fn.has_lua_imp = true + fn.parameters[#fn.parameters] = nil + end end end input:close() @@ -329,6 +333,14 @@ for i = 1, #functions do output:write(', arena') end + if fn.has_lua_imp then + if #args > 0 then + output:write(', NULL') + else + output:write('NULL') + end + end + if fn.can_fail then -- if the function can fail, also pass a pointer to the local error object if #args > 0 then @@ -497,6 +509,10 @@ local function process_function(fn) ]]) end + if fn.has_lua_imp then + cparams = cparams .. 'lstate, ' + end + if fn.can_fail then cparams = cparams .. '&err' else @@ -539,13 +555,27 @@ local function process_function(fn) end write_shifted_output(output, string.format([[ const %s ret = %s(%s); + ]], fn.return_type, fn.name, cparams)) + + if fn.has_lua_imp then + -- only push onto the Lua stack if we haven't already + write_shifted_output(output, string.format([[ + if (lua_gettop(lstate) == 0) { + nlua_push_%s(lstate, ret, true); + } + ]], return_type)) + else + write_shifted_output(output, string.format([[ nlua_push_%s(lstate, ret, true); + ]], return_type)) + end + + write_shifted_output(output, string.format([[ %s %s %s return 1; - ]], fn.return_type, fn.name, cparams, return_type, - free_retval, free_at_exit_code, err_throw_code)) + ]], free_retval, free_at_exit_code, err_throw_code)) else write_shifted_output(output, string.format([[ %s(%s); -- cgit From 66360675cf4d091b7460e4a8e1435c13216c1929 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Sun, 11 Sep 2022 17:12:44 +0200 Subject: build: allow IWYU to fix includes for all .c files Allow Include What You Use to remove unnecessary includes and only include what is necessary. This helps with reducing compilation times and makes it easier to visualise which dependencies are actually required. Work on https://github.com/neovim/neovim/issues/549, but doesn't close it since this only works fully for .c files and not headers. --- src/nvim/generators/gen_api_dispatch.lua | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'src/nvim/generators/gen_api_dispatch.lua') diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index 6894b7a6df..d4fe455f82 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -189,6 +189,35 @@ funcs_metadata_output:close() -- start building the dispatch wrapper output local output = io.open(dispatch_outputf, 'wb') + +-- =========================================================================== +-- NEW API FILES MUST GO HERE. +-- +-- When creating a new API file, you must include it here, +-- so that the dispatcher can find the C functions that you are creating! +-- =========================================================================== +output:write([[ +#include "nvim/log.h" +#include "nvim/map.h" +#include "nvim/msgpack_rpc/helpers.h" +#include "nvim/vim.h" + +#include "nvim/api/autocmd.h" +#include "nvim/api/buffer.h" +#include "nvim/api/command.h" +#include "nvim/api/deprecated.h" +#include "nvim/api/extmark.h" +#include "nvim/api/options.h" +#include "nvim/api/tabpage.h" +#include "nvim/api/ui.h" +#include "nvim/api/vim.h" +#include "nvim/api/vimscript.h" +#include "nvim/api/win_config.h" +#include "nvim/api/window.h" +#include "nvim/ui_client.h" + +]]) + local function real_type(type) local rv = type local rmatch = string.match(type, "Dict%(([_%w]+)%)") -- cgit From 24488169564c39a506c235bf6a33b8e23a8cb528 Mon Sep 17 00:00:00 2001 From: hlpr98 Date: Mon, 27 May 2019 22:04:24 +0530 Subject: feat(tui): run TUI as external process --- src/nvim/generators/gen_api_dispatch.lua | 40 +++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) (limited to 'src/nvim/generators/gen_api_dispatch.lua') diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index d4fe455f82..240b99ca29 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -60,6 +60,12 @@ for i = 6, #arg do if public and not fn.noexport then functions[#functions + 1] = tmp[j] function_names[fn.name] = true + if #fn.parameters >= 2 and fn.parameters[2][1] == 'Array' and fn.parameters[2][2] == 'uidata' then + -- function recieves the "args" as a parameter + fn.receives_array_args = true + -- remove the args parameter + table.remove(fn.parameters, 2) + end if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then -- this function should receive the channel id fn.receives_channel_id = true @@ -159,7 +165,7 @@ local exported_attributes = {'name', 'return_type', 'method', 'since', 'deprecated_since'} local exported_functions = {} for _,f in ipairs(functions) do - if not (startswith(f.name, "nvim__") or f.name == "nvim_error_event") then + if not (startswith(f.name, "nvim__") or f.name == "nvim_error_event" or f.name == "redraw") then local f_exported = {} for _,attr in ipairs(exported_attributes) do f_exported[attr] = f[attr] @@ -264,11 +270,13 @@ for i = 1, #functions do output:write('\n '..rt..' '..converted..';') end output:write('\n') - output:write('\n if (args.size != '..#fn.parameters..') {') - output:write('\n api_set_error(error, kErrorTypeException, \ - "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);') - output:write('\n goto cleanup;') - output:write('\n }\n') + if not fn.receives_array_args then + output:write('\n if (args.size != '..#fn.parameters..') {') + output:write('\n api_set_error(error, kErrorTypeException, \ + "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);') + output:write('\n goto cleanup;') + output:write('\n }\n') + end -- Validation/conversion for each argument for j = 1, #fn.parameters do @@ -350,12 +358,28 @@ for i = 1, #functions do if fn.receives_channel_id then -- if the function receives the channel id, pass it as first argument if #args > 0 or fn.can_fail then - output:write('channel_id, '..call_args) + output:write('channel_id, ') + if fn.receives_array_args then + -- if the function recieves the array args, pass it the second argument + output:write('args, ') + end + output:write(call_args) else output:write('channel_id') + if fn.receives_array_args then + output:write(', args') + end end else - output:write(call_args) + if fn.receives_array_args then + if #args > 0 or fn.call_fail then + output:write('args, '..call_args) + else + output:write('args') + end + else + output:write(call_args) + end end if fn.arena_return then -- cgit From 43e8ec92de9e0850e7d202cb7ff9051bc408447e Mon Sep 17 00:00:00 2001 From: bfredl Date: Mon, 2 May 2022 21:10:01 +0200 Subject: fix(tui): more work in the TUI --- src/nvim/generators/gen_api_dispatch.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/nvim/generators/gen_api_dispatch.lua') diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index 240b99ca29..55ab9fdf0e 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -363,7 +363,7 @@ for i = 1, #functions do -- if the function recieves the array args, pass it the second argument output:write('args, ') end - output:write(call_args) + output:write(call_args) else output:write('channel_id') if fn.receives_array_args then -- cgit From 936e191fef9865600af211c29ea4959ffbce81dd Mon Sep 17 00:00:00 2001 From: dundargoc <33953936+dundargoc@users.noreply.github.com> Date: Wed, 4 Jan 2023 00:38:48 +0100 Subject: docs: fix typos (#21427) Co-authored-by: Gustavo Sampaio Co-authored-by: C.D. MacEachern Co-authored-by: Sean Dewar Co-authored-by: Tomas Nemec --- src/nvim/generators/gen_api_dispatch.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/nvim/generators/gen_api_dispatch.lua') diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index 55ab9fdf0e..96c9b21b8f 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -61,7 +61,7 @@ for i = 6, #arg do functions[#functions + 1] = tmp[j] function_names[fn.name] = true if #fn.parameters >= 2 and fn.parameters[2][1] == 'Array' and fn.parameters[2][2] == 'uidata' then - -- function recieves the "args" as a parameter + -- function receives the "args" as a parameter fn.receives_array_args = true -- remove the args parameter table.remove(fn.parameters, 2) @@ -360,7 +360,7 @@ for i = 1, #functions do if #args > 0 or fn.can_fail then output:write('channel_id, ') if fn.receives_array_args then - -- if the function recieves the array args, pass it the second argument + -- if the function receives the array args, pass it the second argument output:write('args, ') end output:write(call_args) -- cgit From cb757f2663e6950e655c6306d713338dfa66b18d Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Mon, 23 Jan 2023 10:26:46 +0100 Subject: build: make generated source files reproducible #21586 Problem: Build is not reproducible, because generated source files (.c/.h/) are not deterministic, mostly because Lua pairs() is unordered by design (for security). https://github.com/LuaJIT/LuaJIT/issues/626#issuecomment-707005671 https://www.lua.org/manual/5.1/manual.html#pdf-next > The order in which the indices are enumerated is not specified [...] > >> The hardening of the VM deliberately randomizes string hashes. This in >> turn randomizes the iteration order of tables with string keys. Solution: - Update the code generation scripts to be deterministic. - That is only a partial solution: the exported function (funcs_metadata.generated.h) and ui event (ui_events_metadata.generated.h) metadata have some mpack'ed tables, which are not serialized deterministically. - As a workaround, introduce `PRG_GEN_LUA` cmake setting, so you can inject a modified build of luajit (with LUAJIT_SECURITY_PRN=0) that preserves table order. - Longer-term we should change the mpack'ed data structure so it no longer uses tables keyed by strings. Closes #20124 Co-Authored-By: dundargoc Co-Authored-By: Arnout Engelen --- src/nvim/generators/gen_api_dispatch.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/nvim/generators/gen_api_dispatch.lua') diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua index 96c9b21b8f..35f6bf8455 100644 --- a/src/nvim/generators/gen_api_dispatch.lua +++ b/src/nvim/generators/gen_api_dispatch.lua @@ -426,7 +426,9 @@ for _,fn in ipairs(functions) do end remote_fns.redraw = {impl_name="ui_client_redraw", fast=true} -local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", vim.tbl_keys(remote_fns), function (idx) +local names = vim.tbl_keys(remote_fns) +table.sort(names) +local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", names, function (idx) return "method_handlers["..idx.."].name" end) -- cgit