build: make generated source files reproducible #21586

Problem:
Build is not reproducible, because generated source files (.c/.h/) are not
deterministic, mostly because Lua pairs() is unordered by design (for security).

https://github.com/LuaJIT/LuaJIT/issues/626#issuecomment-707005671
https://www.lua.org/manual/5.1/manual.html#pdf-next
> The order in which the indices are enumerated is not specified [...]
>
>> The hardening of the VM deliberately randomizes string hashes. This in
>> turn randomizes the iteration order of tables with string keys.

Solution:
- Update the code generation scripts to be deterministic.
    - That is only a partial solution: the exported function
      (funcs_metadata.generated.h) and ui event
      (ui_events_metadata.generated.h) metadata have some mpack'ed
      tables, which are not serialized deterministically.
    - As a workaround, introduce `PRG_GEN_LUA` cmake setting, so you can
      inject a modified build of luajit (with LUAJIT_SECURITY_PRN=0)
      that preserves table order.
    - Longer-term we should change the mpack'ed data structure so it no
      longer uses tables keyed by strings.

Closes #20124

Co-Authored-By: dundargoc <gocdundar@gmail.com>
Co-Authored-By: Arnout Engelen <arnout@bzzt.net>

5 files changed, 16 insertions, 12 deletions

diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
index 96c9b21b8f..35f6bf8455 100644
--- a/src/nvim/generators/gen_api_dispatch.lua
+++ b/src/nvim/generators/gen_api_dispatch.lua
@@ -426,7 +426,9 @@ for _,fn in ipairs(functions) do
 end
 remote_fns.redraw = {impl_name="ui_client_redraw", fast=true}
 
-local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", vim.tbl_keys(remote_fns), function (idx)
+local names = vim.tbl_keys(remote_fns)
+table.sort(names)
+local hashorder, hashfun = hashy.hashy_hash("msgpack_rpc_get_handler_for", names, function (idx)
   return "method_handlers["..idx.."].name"
 end)
 
diff --git a/src/nvim/generators/gen_eval.lua b/src/nvim/generators/gen_eval.lua
index a50e058e00..baed6a74c2 100644
--- a/src/nvim/generators/gen_eval.lua
+++ b/src/nvim/generators/gen_eval.lua
@@ -73,14 +73,13 @@ for _,fun in ipairs(metadata) do
   end
 end
 
+local func_names = vim.tbl_keys(funcs)
+table.sort(func_names)
 local funcsdata = io.open(funcs_file, 'w')
-funcsdata:write(mpack.pack(funcs))
+funcsdata:write(mpack.pack(func_names))
 funcsdata:close()
 
-
-local names = vim.tbl_keys(funcs)
-
-local neworder, hashfun = hashy.hashy_hash("find_internal_func", names, function (idx)
+local neworder, hashfun = hashy.hashy_hash("find_internal_func", func_names, function (idx)
   return "functions["..idx.."].name"
 end)
 hashpipe:write("static const EvalFuncDef functions[] = {\n")
diff --git a/src/nvim/generators/gen_events.lua b/src/nvim/generators/gen_events.lua
index 6ee45a14af..8db7f22452 100644
--- a/src/nvim/generators/gen_events.lua
+++ b/src/nvim/generators/gen_events.lua
@@ -32,7 +32,9 @@ for i, event in ipairs(events) do
   end
 end
 
-for alias, event in pairs(aliases) do
+for _, v in ipairs(aliases) do
+  local alias = v[1]
+  local event = v[2]
   names_tgt:write(('\n  {%u, "%s", EVENT_%s},'):format(#alias, alias, event:upper()))
 end
 
diff --git a/src/nvim/generators/gen_keysets.lua b/src/nvim/generators/gen_keysets.lua
index 633c5da184..b1c1f3e2d8 100644
--- a/src/nvim/generators/gen_keysets.lua
+++ b/src/nvim/generators/gen_keysets.lua
@@ -1,4 +1,3 @@
-
 local nvimsrcdir = arg[1]
 local shared_file = arg[2]
 local funcs_file = arg[3]
@@ -38,7 +37,9 @@ local function sanitize(key)
   return key
 end
 
-for name, keys in pairs(keysets) do
+for _, v in ipairs(keysets) do
+  local name = v[1]
+  local keys = v[2]
   local neworder, hashfun = hashy.hashy_hash(name, keys, function (idx)
     return name.."_table["..idx.."].str"
   end)
diff --git a/src/nvim/generators/gen_options.lua b/src/nvim/generators/gen_options.lua
index af21d44eaf..edb7dae159 100644
--- a/src/nvim/generators/gen_options.lua
+++ b/src/nvim/generators/gen_options.lua
@@ -159,7 +159,7 @@ local dump_option = function(i, o)
     if #o.scope == 2 then
       pv_name = 'OPT_BOTH(' .. pv_name .. ')'
     end
-    defines['PV_' .. varname:sub(3):upper()] = pv_name
+    table.insert(defines,  { 'PV_' .. varname:sub(3):upper() , pv_name})
     w('    .indir=' .. pv_name)
   end
   if o.enable_if then
@@ -192,7 +192,7 @@ w('  [' .. ('%u'):format(#options.options) .. ']={.fullname=NULL}')
 w('};')
 w('')
 
-for k, v in pairs(defines) do
-  w('#define ' .. k .. ' ' .. v)
+for _, v in ipairs(defines) do
+  w('#define ' .. v[1] .. ' ' .. v[2])
 end
 opt_fd:close()