perf: pre-compile embedded Lua source into bytecode (#16631)

The Lua modules that make up vim.lua are embedded as raw source files into the
nvim binary. These sources are loaded by the Lua runtime on startuptime. We can
pre-compile these sources into Lua bytecode before embedding them into the
binary, which minimizes the size of the binary and improves startuptime.

3 files changed, 50 insertions, 25 deletions

diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index bb16459a7f..9c4b778169 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -326,7 +326,9 @@ add_custom_command(
 
 add_custom_command(
   OUTPUT ${VIM_MODULE_FILE}
-  COMMAND ${LUA_PRG} ${CHAR_BLOB_GENERATOR} ${VIM_MODULE_FILE}
+  COMMAND ${CMAKE_COMMAND} -E env
+      "LUAC_PRG=${LUAC_PRG}"
+      ${LUA_PRG} ${CHAR_BLOB_GENERATOR} -c ${VIM_MODULE_FILE}
       ${LUA_VIM_MODULE_SOURCE} vim_module
       ${LUA_SHARED_MODULE_SOURCE} shared_module
       ${LUA_INSPECT_MODULE_SOURCE} inspect_module
@@ -339,6 +341,7 @@ add_custom_command(
     ${LUA_INSPECT_MODULE_SOURCE}
     ${LUA_F_MODULE_SOURCE}
     ${LUA_META_MODULE_SOURCE}
+  VERBATIM
 )
 
 list(APPEND NVIM_GENERATED_SOURCES
diff --git a/src/nvim/generators/gen_char_blob.lua b/src/nvim/generators/gen_char_blob.lua
index a7dad50d48..70c034abc5 100644
--- a/src/nvim/generators/gen_char_blob.lua
+++ b/src/nvim/generators/gen_char_blob.lua
@@ -1,12 +1,26 @@
 if arg[1] == '--help' then
   print('Usage:')
-  print('  '..arg[0]..' target source varname [source varname]...')
+  print('  '..arg[0]..' [-c] target source varname [source varname]...')
   print('')
   print('Generates C file with big uint8_t blob.')
   print('Blob will be stored in a static const array named varname.')
   os.exit()
 end
 
+-- Recognized options:
+--   -c   compile Lua bytecode
+local options = {}
+
+while true do
+  local opt = string.match(arg[1], "^-(%w)")
+  if not opt then
+    break
+  end
+
+  options[opt] = true
+  table.remove(arg, 1)
+end
+
 assert(#arg >= 3 and (#arg - 1) % 2 == 0)
 
 local target_file = arg[1] or error('Need a target file')
@@ -23,11 +37,25 @@ for argi = 2, #arg, 2 do
   end
   varnames[varname] = source_file
 
-  local source = io.open(source_file, 'r')
-      or error(string.format("source_file %q doesn't exist", source_file))
-
   target:write(('static const uint8_t %s[] = {\n'):format(varname))
 
+  local output
+  if options.c then
+    local luac = os.getenv("LUAC_PRG")
+    if luac then
+      output = io.popen(luac:format(source_file), "r"):read("*a")
+    else
+      print("LUAC_PRG is undefined")
+    end
+  end
+
+  if not output then
+    local source = io.open(source_file, "r")
+        or error(string.format("source_file %q doesn't exist", source_file))
+    output = source:read("*a")
+    source:close()
+  end
+
   local num_bytes = 0
   local MAX_NUM_BYTES = 15  -- 78 / 5: maximum number of bytes on one line
   target:write(' ')
@@ -41,19 +69,13 @@ for argi = 2, #arg, 2 do
     end
   end
 
-  for line in source:lines() do
-    for i = 1, string.len(line) do
-      local byte = line:byte(i)
-      assert(byte ~= 0)
-      target:write(string.format(' %3u,', byte))
-      increase_num_bytes()
-    end
-    target:write(string.format(' %3u,', string.byte('\n', 1)))
+  for i = 1, string.len(output) do
+    local byte = output:byte(i)
+    target:write(string.format(' %3u,', byte))
     increase_num_bytes()
   end
 
-  target:write('   0};\n')
-  source:close()
+  target:write('  0};\n')
 end
 
 target:close()
diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c
index b09d133495..107ff22913 100644
--- a/src/nvim/lua/executor.c
+++ b/src/nvim/lua/executor.c
@@ -404,9 +404,9 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
 
   {
     const char *code = (char *)&shared_module[0];
-    if (luaL_loadbuffer(lstate, code, strlen(code), "@vim/shared.lua")
+    if (luaL_loadbuffer(lstate, code, sizeof(shared_module) - 1, "@vim/shared.lua")
         || nlua_pcall(lstate, 0, 0)) {
-      nlua_error(lstate, _("E5106: Error while creating shared module: %.*s"));
+      nlua_error(lstate, _("E5106: Error while creating shared module: %.*s\n"));
       return 1;
     }
   }
@@ -416,18 +416,18 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
     lua_getfield(lstate, -1, "loaded");  // [package, loaded]
 
     const char *code = (char *)&inspect_module[0];
-    if (luaL_loadbuffer(lstate, code, strlen(code), "@vim/inspect.lua")
+    if (luaL_loadbuffer(lstate, code, sizeof(inspect_module) - 1, "@vim/inspect.lua")
         || nlua_pcall(lstate, 0, 1)) {
-      nlua_error(lstate, _("E5106: Error while creating inspect module: %.*s"));
+      nlua_error(lstate, _("E5106: Error while creating inspect module: %.*s\n"));
       return 1;
     }
     // [package, loaded, inspect]
     lua_setfield(lstate, -2, "vim.inspect");  // [package, loaded]
 
     code = (char *)&lua_F_module[0];
-    if (luaL_loadbuffer(lstate, code, strlen(code), "@vim/F.lua")
+    if (luaL_loadbuffer(lstate, code, sizeof(lua_F_module) - 1, "@vim/F.lua")
         || nlua_pcall(lstate, 0, 1)) {
-      nlua_error(lstate, _("E5106: Error while creating vim.F module: %.*s"));
+      nlua_error(lstate, _("E5106: Error while creating vim.F module: %.*s\n"));
       return 1;
     }
     // [package, loaded, module]
@@ -438,9 +438,9 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
 
   {
     const char *code = (char *)&vim_module[0];
-    if (luaL_loadbuffer(lstate, code, strlen(code), "@vim.lua")
+    if (luaL_loadbuffer(lstate, code, sizeof(vim_module) - 1, "@vim.lua")
         || nlua_pcall(lstate, 0, 0)) {
-      nlua_error(lstate, _("E5106: Error while creating vim module: %.*s"));
+      nlua_error(lstate, _("E5106: Error while creating vim module: %.*s\n"));
       return 1;
     }
   }
@@ -450,9 +450,9 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
     lua_getfield(lstate, -1, "loaded");  // [package, loaded]
 
     const char *code = (char *)&lua_meta_module[0];
-    if (luaL_loadbuffer(lstate, code, strlen(code), "@vim/_meta.lua")
+    if (luaL_loadbuffer(lstate, code, sizeof(lua_meta_module) - 1, "@vim/_meta.lua")
         || nlua_pcall(lstate, 0, 1)) {
-      nlua_error(lstate, _("E5106: Error while creating vim._meta module: %.*s"));
+      nlua_error(lstate, _("E5106: Error while creating vim._meta module: %.*s\n"));
       return 1;
     }
     // [package, loaded, module]