build: make generated source files reproducible #21586

Problem:
Build is not reproducible, because generated source files (.c/.h/) are not
deterministic, mostly because Lua pairs() is unordered by design (for security).

https://github.com/LuaJIT/LuaJIT/issues/626#issuecomment-707005671
https://www.lua.org/manual/5.1/manual.html#pdf-next
> The order in which the indices are enumerated is not specified [...]
>
>> The hardening of the VM deliberately randomizes string hashes. This in
>> turn randomizes the iteration order of tables with string keys.

Solution:
- Update the code generation scripts to be deterministic.
    - That is only a partial solution: the exported function
      (funcs_metadata.generated.h) and ui event
      (ui_events_metadata.generated.h) metadata have some mpack'ed
      tables, which are not serialized deterministically.
    - As a workaround, introduce `PRG_GEN_LUA` cmake setting, so you can
      inject a modified build of luajit (with LUAJIT_SECURITY_PRN=0)
      that preserves table order.
    - Longer-term we should change the mpack'ed data structure so it no
      longer uses tables keyed by strings.

Closes #20124

Co-Authored-By: dundargoc <gocdundar@gmail.com>
Co-Authored-By: Arnout Engelen <arnout@bzzt.net>

1 files changed, 46 insertions, 47 deletions

diff --git a/src/nvim/api/keysets.lua b/src/nvim/api/keysets.lua
index 8f909e937f..bd709b7b25 100644
--- a/src/nvim/api/keysets.lua
+++ b/src/nvim/api/keysets.lua
@@ -1,8 +1,8 @@
 return {
-  context = {
+  { 'context', {
     "types";
-  };
-  set_decoration_provider = {
+  }};
+  { 'set_decoration_provider', {
     "on_start";
     "on_buf";
     "on_win";
@@ -10,8 +10,8 @@ return {
     "on_end";
     "_on_hl_def";
     "_on_spell_nav";
-  };
-  set_extmark = {
+  }};
+  { 'set_extmark', {
     "id";
     "end_line";
     "end_row";
@@ -39,8 +39,8 @@ return {
     "conceal";
     "spell";
     "ui_watched";
-  };
-  keymap = {
+  }};
+  { 'keymap', {
     "noremap";
     "nowait";
     "silent";
@@ -50,11 +50,11 @@ return {
     "callback";
     "desc";
     "replace_keycodes";
-  };
-  get_commands = {
+  }};
+  { 'get_commands', {
     "builtin";
-  };
-  user_command = {
+  }};
+  { 'user_command', {
     "addr";
     "bang";
     "bar";
@@ -67,8 +67,8 @@ return {
     "preview";
     "range";
     "register";
-  };
-  float_config = {
+  }};
+  { 'float_config', {
     "row";
     "col";
     "width";
@@ -85,25 +85,25 @@ return {
     "title_pos";
     "style";
     "noautocmd";
-  };
-  runtime = {
+  }};
+  { 'runtime', {
     "is_lua";
     "do_source";
-  };
-  eval_statusline = {
+  }};
+  { 'eval_statusline', {
     "winid";
     "maxwidth";
     "fillchar";
     "highlights";
     "use_winbar";
     "use_tabline";
-  };
-  option = {
+  }};
+  { 'option', {
     "scope";
     "win";
     "buf";
-  };
-  highlight = {
+  }};
+  { 'highlight', {
     "bold";
     "standout";
     "strikethrough";
@@ -128,8 +128,8 @@ return {
     "blend";
     "fg_indexed";
     "bg_indexed";
-  };
-  highlight_cterm = {
+  }};
+  { 'highlight_cterm', {
     "bold";
     "standout";
     "strikethrough";
@@ -141,15 +141,15 @@ return {
     "italic";
     "reverse";
     "nocombine";
-  };
+  }};
   -- Autocmds
-  clear_autocmds = {
+  { 'clear_autocmds', {
     "buffer";
     "event";
     "group";
     "pattern";
-  };
-  create_autocmd = {
+  }};
+  { 'create_autocmd', {
     "buffer";
     "callback";
     "command";
@@ -158,24 +158,24 @@ return {
     "nested";
     "once";
     "pattern";
-  };
-  exec_autocmds = {
+  }};
+  { 'exec_autocmds', {
     "buffer";
     "group";
     "modeline";
     "pattern";
     "data";
-  };
-  get_autocmds = {
+  }};
+  { 'get_autocmds', {
     "event";
     "group";
     "pattern";
     "buffer";
-  };
-  create_augroup = {
+  }};
+  { 'create_augroup', {
     "clear";
-  };
-  cmd = {
+  }};
+  { 'cmd', {
     "cmd";
     "range";
     "count";
@@ -187,12 +187,12 @@ return {
     "nargs";
     "addr";
     "nextcmd";
-  };
-  cmd_magic = {
+  }};
+  { 'cmd_magic', {
     "file";
     "bar";
-  };
-  cmd_mods = {
+  }};
+  { 'cmd_mods', {
     "silent";
     "emsg_silent";
     "unsilent";
@@ -213,16 +213,15 @@ return {
     "verbose";
     "vertical";
     "split";
-  };
-  cmd_mods_filter = {
+  }};
+  { 'cmd_mods_filter', {
     "pattern";
     "force";
-  };
-  cmd_opts = {
+  }};
+  { 'cmd_opts', {
     "output";
-  };
-  echo_opts = {
+  }};
+  { 'echo_opts', {
     "verbose";
-  };
+  }};
 }
-