Merge pull request #28344 from bfredl/wonderland

feat(build): build.zig MVP: build and run functionaltests on linux

17 files changed, 562 insertions, 34 deletions

diff --git a/src/build_lua.zig b/src/build_lua.zig
new file mode 100644
index 0000000000..be3c709aca
--- /dev/null
+++ b/src/build_lua.zig
@@ -0,0 +1,138 @@
+const std = @import("std");
+const LazyPath = std.Build.LazyPath;
+
+pub fn build_nlua0(
+    b: *std.Build,
+    target: std.Build.ResolvedTarget,
+    optimize: std.builtin.OptimizeMode,
+    use_luajit: bool,
+    ziglua: *std.Build.Dependency,
+    lpeg: *std.Build.Dependency,
+) *std.Build.Step.Compile {
+    const options = b.addOptions();
+    options.addOption(bool, "use_luajit", use_luajit);
+
+    const nlua0_exe = b.addExecutable(.{
+        .name = "nlua0",
+        .root_source_file = b.path("src/nlua0.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    const nlua0_mod = nlua0_exe.root_module;
+
+    const exe_unit_tests = b.addTest(.{
+        .root_source_file = b.path("src/nlua0.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+
+    const embedded_data = b.addModule("embedded_data", .{
+        .root_source_file = b.path("runtime/embedded_data.zig"),
+    });
+
+    for ([2]*std.Build.Module{ nlua0_mod, exe_unit_tests.root_module }) |mod| {
+        mod.addImport("ziglua", ziglua.module("lua_wrapper"));
+        mod.addImport("embedded_data", embedded_data);
+        // addImport already links by itself. but we need headers as well..
+        mod.linkLibrary(ziglua.artifact("lua"));
+
+        mod.addOptions("options", options);
+
+        mod.addIncludePath(b.path("src"));
+        mod.addIncludePath(b.path("src/includes_fixmelater"));
+        add_lua_modules(mod, lpeg, use_luajit, true);
+    }
+
+    // for debugging the nlua0 environment
+    // like this: `zig build nlua0 -- script.lua {args}`
+    const run_cmd = b.addRunArtifact(nlua0_exe);
+    if (b.args) |args| {
+        run_cmd.addArgs(args);
+    }
+    const run_step = b.step("nlua0", "Run nlua0 build tool");
+    run_step.dependOn(&run_cmd.step);
+
+    b.installArtifact(nlua0_exe); // DEBUG
+
+    const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests);
+
+    const test_step = b.step("test_nlua0", "Run unit tests for nlua0");
+    test_step.dependOn(&run_exe_unit_tests.step);
+
+    return nlua0_exe;
+}
+
+pub fn add_lua_modules(mod: *std.Build.Module, lpeg: *std.Build.Dependency, use_luajit: bool, is_nlua0: bool) void {
+    const flags = [_][]const u8{
+        // Standard version used in Lua Makefile
+        "-std=gnu99",
+        if (is_nlua0) "-DNVIM_NLUA0" else "",
+    };
+
+    mod.addIncludePath(lpeg.path(""));
+    mod.addCSourceFiles(.{
+        .files = &.{
+            "src/mpack/lmpack.c",
+            "src/mpack/mpack_core.c",
+            "src/mpack/object.c",
+            "src/mpack/conv.c",
+            "src/mpack/rpc.c",
+        },
+        .flags = &flags,
+    });
+    mod.addCSourceFiles(.{
+        .root = .{ .dependency = .{ .dependency = lpeg, .sub_path = "" } },
+        .files = &.{
+            "lpcap.c",
+            "lpcode.c",
+            "lpcset.c",
+            "lpprint.c",
+            "lptree.c",
+            "lpvm.c",
+        },
+        .flags = &flags,
+    });
+
+    if (!use_luajit) {
+        mod.addCSourceFiles(.{
+            .files = &.{
+                "src/bit.c",
+            },
+            .flags = &flags,
+        });
+    }
+}
+
+pub fn build_libluv(
+    b: *std.Build,
+    target: std.Build.ResolvedTarget,
+    optimize: std.builtin.OptimizeMode,
+    lua: *std.Build.Step.Compile,
+    libuv: *std.Build.Step.Compile,
+) !*std.Build.Step.Compile {
+    const upstream = b.dependency("libluv", .{});
+    const compat53 = b.dependency("lua_compat53", .{});
+    const lib = b.addStaticLibrary(.{
+        .name = "luv",
+        .target = target,
+        .optimize = optimize,
+    });
+
+    lib.linkLibrary(lua);
+    lib.linkLibrary(libuv);
+
+    lib.addIncludePath(upstream.path("src"));
+    lib.addIncludePath(compat53.path("c-api"));
+
+    lib.installHeader(upstream.path("src/luv.h"), "luv/luv.h");
+
+    lib.addCSourceFiles(.{ .root = upstream.path("src/"), .files = &.{
+        "luv.c",
+    } });
+
+    lib.addCSourceFiles(.{ .root = compat53.path("c-api"), .files = &.{
+        "compat-5.3.c",
+    } });
+
+    return lib;
+}
diff --git a/src/gen/gen_api_dispatch.lua b/src/gen/gen_api_dispatch.lua
index b5433527bc..6480e66489 100644
--- a/src/gen/gen_api_dispatch.lua
+++ b/src/gen/gen_api_dispatch.lua
@@ -11,8 +11,6 @@ local mpack = vim.mpack
 
 local hashy = require 'gen.hashy'
 
-local pre_args = 7
-assert(#arg >= pre_args)
 -- output h file with generated dispatch functions (dispatch_wrappers.generated.h)
 local dispatch_outputf = arg[1]
 -- output h file with packed metadata (api_metadata.generated.h)
@@ -23,6 +21,11 @@ local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c
 local keysets_outputf = arg[5] -- keysets_defs.generated.h
 local ui_metadata_inputf = arg[6] -- ui events metadata
 local git_version_inputf = arg[7] -- git version header
+local nvim_version_inputf = arg[8] -- nvim version
+local dump_bin_array_inputf = arg[9]
+local dispatch_deprecated_inputf = arg[10]
+local pre_args = 10
+assert(#arg >= pre_args)
 
 local functions = {}
 
@@ -152,7 +155,7 @@ end
 
 -- Export functions under older deprecated names.
 -- These will be removed eventually.
-local deprecated_aliases = require('nvim.api.dispatch_deprecated')
+local deprecated_aliases = loadfile(dispatch_deprecated_inputf)()
 for _, f in ipairs(shallowcopy(functions)) do
   local ismethod = false
   if startswith(f.name, 'nvim_') then
@@ -244,7 +247,7 @@ for x in string.gmatch(ui_options_text, '"([a-z][a-z_]+)"') do
   table.insert(ui_options, x)
 end
 
-local version = require 'nvim_version' -- `build/nvim_version.lua` file.
+local version = loadfile(nvim_version_inputf)()
 local git_version = io.open(git_version_inputf):read '*a'
 local version_build = string.match(git_version, '#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
 
@@ -302,7 +305,7 @@ for i, item in ipairs(types) do
 end
 
 local packed = table.concat(pieces)
-local dump_bin_array = require('gen.dump_bin_array')
+local dump_bin_array = loadfile(dump_bin_array_inputf)()
 dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
 api_metadata_output:close()
 
diff --git a/src/gen/gen_eval.lua b/src/gen/gen_eval.lua
index 2f2d09485b..66c2a7c015 100644
--- a/src/gen/gen_eval.lua
+++ b/src/gen/gen_eval.lua
@@ -1,10 +1,9 @@
 local mpack = vim.mpack
 
-local autodir = arg[1]
+local funcsfname = arg[1]
 local metadata_file = arg[2]
 local funcs_file = arg[3]
-
-local funcsfname = autodir .. '/funcs.generated.h'
+local eval_file = arg[4]
 
 --Will generate funcs.generated.h with definition of functions static const array.
 
@@ -48,7 +47,7 @@ hashpipe:write([[
 
 ]])
 
-local funcs = require('nvim.eval').funcs
+local funcs = loadfile(eval_file)().funcs
 for _, func in pairs(funcs) do
   if func.float_func then
     func.func = 'float_op_wrapper'
diff --git a/src/gen/gen_eval_files.lua b/src/gen/gen_eval_files.lua
index 4bf648f934..cfa4d096a3 100755
--- a/src/gen/gen_eval_files.lua
+++ b/src/gen/gen_eval_files.lua
@@ -5,7 +5,7 @@
 local util = require('gen.util')
 local fmt = string.format
 
-local DEP_API_METADATA = 'build/funcs_metadata.mpack'
+local DEP_API_METADATA = arg[1]
 local TEXT_WIDTH = 78
 
 --- @class vim.api.metadata
diff --git a/src/gen/gen_events.lua b/src/gen/gen_events.lua
index c6d97405af..76072c59d4 100644
--- a/src/gen/gen_events.lua
+++ b/src/gen/gen_events.lua
@@ -1,8 +1,9 @@
 local fileio_enum_file = arg[1]
 local names_file = arg[2]
+local auevents_file = arg[3]
 
 local hashy = require('gen.hashy')
-local auevents = require('nvim.auevents')
+local auevents = loadfile(auevents_file)()
 local events = auevents.events
 local aliases = auevents.aliases
 
diff --git a/src/gen/gen_ex_cmds.lua b/src/gen/gen_ex_cmds.lua
index 6c03e8fc4d..3662cb149b 100644
--- a/src/gen/gen_ex_cmds.lua
+++ b/src/gen/gen_ex_cmds.lua
@@ -1,17 +1,15 @@
-local includedir = arg[1]
-local autodir = arg[2]
-
 -- Will generate files ex_cmds_enum.generated.h with cmdidx_T enum
 -- and ex_cmds_defs.generated.h with main Ex commands definitions.
 
-local enumfname = includedir .. '/ex_cmds_enum.generated.h'
-local defsfname = autodir .. '/ex_cmds_defs.generated.h'
+local enumfname = arg[1] -- '/ex_cmds_enum.generated.h'
+local defsfname = arg[2] -- '/ex_cmds_defs.generated.h'
+local ex_cmds_name = arg[3] -- 'ex_cmds.lua'
 
 local enumfile = io.open(enumfname, 'w')
 local defsfile = io.open(defsfname, 'w')
 
 local bit = require 'bit'
-local ex_cmds = require('nvim.ex_cmds')
+local ex_cmds = loadfile(ex_cmds_name)()
 local defs = ex_cmds.cmds
 local flags = ex_cmds.flags
 
diff --git a/src/gen/gen_keycodes.lua b/src/gen/gen_keycodes.lua
index 584f0a3d36..98b714dffe 100644
--- a/src/gen/gen_keycodes.lua
+++ b/src/gen/gen_keycodes.lua
@@ -1,7 +1,8 @@
 local names_file = arg[1]
+local keycodes_file = arg[2]
 
 local hashy = require('gen.hashy')
-local keycodes = require('nvim.keycodes')
+local keycodes = loadfile(keycodes_file)()
 
 local keycode_names = keycodes.names
 
diff --git a/src/gen/gen_options.lua b/src/gen/gen_options.lua
index 1947297a0e..9c88f96ae2 100644
--- a/src/gen/gen_options.lua
+++ b/src/gen/gen_options.lua
@@ -1,5 +1,7 @@
+local options_input_file = arg[5]
+
 --- @module 'nvim.options'
-local options = require('nvim.options')
+local options = loadfile(options_input_file)()
 local options_meta = options.options
 local cstr = options.cstr
 local valid_scopes = options.valid_scopes
diff --git a/src/gen/gen_steps.zig b/src/gen/gen_steps.zig
new file mode 100644
index 0000000000..563a4418df
--- /dev/null
+++ b/src/gen/gen_steps.zig
@@ -0,0 +1,237 @@
+const std = @import("std");
+const LazyPath = std.Build.LazyPath;
+
+pub const SourceItem = struct { name: []u8, api_export: bool };
+
+pub fn nvim_gen_sources(
+    b: *std.Build,
+    nlua0: *std.Build.Step.Compile,
+    nvim_sources: *std.ArrayList(SourceItem),
+    nvim_headers: *std.ArrayList([]u8),
+    api_headers: *std.ArrayList(LazyPath),
+    include_path: []const LazyPath,
+    target: std.Build.ResolvedTarget,
+    versiondef_git: LazyPath,
+    version_lua: LazyPath,
+) !struct { *std.Build.Step.WriteFile, LazyPath } {
+    const gen_headers = b.addWriteFiles();
+
+    for (nvim_sources.items) |s| {
+        const api_export = if (s.api_export) api_headers else null;
+        const input_file = b.path(b.fmt("src/nvim/{s}", .{s.name}));
+        _ = try generate_header_for(b, s.name, input_file, api_export, nlua0, include_path, target, gen_headers, false);
+    }
+
+    for (nvim_headers.items) |s| {
+        const input_file = b.path(b.fmt("src/nvim/{s}", .{s}));
+        _ = try generate_header_for(b, s, input_file, null, nlua0, include_path, target, gen_headers, true);
+    }
+
+    {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/gen/gen_ex_cmds.lua"));
+        _ = gen_header(b, gen_step, "ex_cmds_enum.generated.h", gen_headers);
+        _ = gen_header(b, gen_step, "ex_cmds_defs.generated.h", gen_headers);
+        gen_step.addFileArg(b.path("src/nvim/ex_cmds.lua"));
+    }
+
+    {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/gen/gen_options.lua"));
+        _ = gen_header(b, gen_step, "options.generated.h", gen_headers);
+        _ = gen_header(b, gen_step, "options_enum.generated.h", gen_headers);
+        _ = gen_header(b, gen_step, "options_map.generated.h", gen_headers);
+        _ = gen_header(b, gen_step, "option_vars.generated.h", gen_headers);
+        gen_step.addFileArg(b.path("src/nvim/options.lua"));
+
+        const test_gen_step = b.step("wipopt", "debug one nlua0 (options)");
+        test_gen_step.dependOn(&gen_step.step);
+    }
+
+    {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/gen/gen_events.lua"));
+        _ = gen_header(b, gen_step, "auevents_enum.generated.h", gen_headers);
+        _ = gen_header(b, gen_step, "auevents_name_map.generated.h", gen_headers);
+        gen_step.addFileArg(b.path("src/nvim/auevents.lua"));
+    }
+
+    {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/gen/gen_keycodes.lua"));
+        _ = gen_header(b, gen_step, "keycode_names.generated.h", gen_headers);
+        gen_step.addFileArg(b.path("src/nvim/keycodes.lua"));
+    }
+
+    {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/gen/gen_char_blob.lua"));
+        // TODO(bfredl): LUAC_PRG is missing. tricky with cross-compiling..
+        // gen_step.addArg("-c");
+        _ = gen_header(b, gen_step, "lua/vim_module.generated.h", gen_headers);
+        // NB: vim._init_packages and vim.inspect must be be first and second ones
+        // respectively, otherwise --luamod-dev won't work properly.
+        const names = [_][]const u8{
+            "_init_packages",
+            "inspect",
+            "_editor",
+            "filetype",
+            "fs",
+            "F",
+            "keymap",
+            "loader",
+            "_defaults",
+            "_options",
+            "shared",
+        };
+        for (names) |n| {
+            gen_step.addFileArg(b.path(b.fmt("runtime/lua/vim/{s}.lua", .{n})));
+            gen_step.addArg(b.fmt("vim.{s}", .{n}));
+        }
+    }
+
+    const ui_metadata = ui_step: {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/gen/gen_api_ui_events.lua"));
+        gen_step.addFileArg(b.path("src/nvim/api/ui_events.in.h"));
+        _ = try gen_header_with_header(b, gen_step, "ui_events_call.generated.h", nlua0, include_path, target, gen_headers);
+        _ = try gen_header_with_header(b, gen_step, "ui_events_remote.generated.h", nlua0, include_path, target, gen_headers);
+        const ui_metadata = gen_step.addOutputFileArg("ui_metadata.mpack");
+        _ = try gen_header_with_header(b, gen_step, "ui_events_client.generated.h", nlua0, include_path, target, gen_headers);
+        break :ui_step ui_metadata;
+    };
+
+    const funcs_metadata = api_step: {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/gen/gen_api_dispatch.lua"));
+        _ = try gen_header_with_header(b, gen_step, "api/private/dispatch_wrappers.generated.h", nlua0, include_path, target, gen_headers);
+        _ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers);
+        const funcs_metadata = gen_step.addOutputFileArg("funcs_metadata.mpack");
+        _ = gen_header(b, gen_step, "lua_api_c_bindings.generated.h", gen_headers);
+        _ = gen_header(b, gen_step, "keysets_defs.generated.h", gen_headers);
+        gen_step.addFileArg(ui_metadata);
+        gen_step.addFileArg(versiondef_git);
+        gen_step.addFileArg(version_lua);
+        gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua"));
+        gen_step.addFileArg(b.path("src/nvim/api/dispatch_deprecated.lua"));
+        // now follows all .h files with exported functions
+        for (api_headers.items) |h| {
+            gen_step.addFileArg(h);
+        }
+
+        break :api_step funcs_metadata;
+    };
+
+    const funcs_data = eval_step: {
+        const gen_step = b.addRunArtifact(nlua0);
+        gen_step.addFileArg(b.path("src/gen/gen_eval.lua"));
+        _ = gen_header(b, gen_step, "funcs.generated.h", gen_headers);
+        gen_step.addFileArg(funcs_metadata);
+        const funcs_data = gen_step.addOutputFileArg("funcs_data.mpack");
+        gen_step.addFileArg(b.path("src/nvim/eval.lua"));
+        break :eval_step funcs_data;
+    };
+
+    return .{ gen_headers, funcs_data };
+}
+
+fn gen_header(
+    b: *std.Build,
+    gen_step: *std.Build.Step.Run,
+    name: []const u8,
+    gen_headers: *std.Build.Step.WriteFile,
+) std.Build.LazyPath {
+    _ = b;
+    const header = gen_step.addOutputFileArg(name);
+    _ = gen_headers.addCopyFile(header, name);
+    return header;
+}
+
+fn gen_header_with_header(
+    b: *std.Build,
+    gen_step: *std.Build.Step.Run,
+    name: []const u8,
+    nlua0: *std.Build.Step.Compile,
+    include_path: []const LazyPath,
+    target: ?std.Build.ResolvedTarget,
+    gen_headers: *std.Build.Step.WriteFile,
+) !std.Build.LazyPath {
+    if (name.len < 12 or !std.mem.eql(u8, ".generated.h", name[name.len - 12 ..])) return error.InvalidBaseName;
+    const h = gen_header(b, gen_step, name, gen_headers);
+    _ = try generate_header_for(b, b.fmt("{s}.h", .{name[0 .. name.len - 12]}), h, null, nlua0, include_path, target, gen_headers, false);
+    return h;
+}
+
+pub const PreprocessorOptions = struct {
+    include_dirs: []const LazyPath = &.{},
+    c_macros: []const []const u8 = &.{},
+    target: ?std.Build.ResolvedTarget = null,
+};
+
+fn run_preprocessor(
+    b: *std.Build,
+    src: LazyPath,
+    output_name: []const u8,
+    options: PreprocessorOptions,
+) !LazyPath {
+    const run_step = std.Build.Step.Run.create(b, b.fmt("preprocess to get {s}", .{output_name}));
+    run_step.addArgs(&.{ b.graph.zig_exe, "cc", "-E" });
+    run_step.addFileArg(src);
+    run_step.addArg("-o");
+    const output = run_step.addOutputFileArg(output_name);
+    // upstream issue: include path logic for addCSourceFiles and TranslateC is _very_ different
+    for (options.include_dirs) |include_dir| {
+        run_step.addArg("-I");
+        run_step.addDirectoryArg(include_dir);
+    }
+    for (options.c_macros) |c_macro| {
+        run_step.addArg(b.fmt("-D{s}", .{c_macro}));
+    }
+    if (options.target) |t| {
+        if (!t.query.isNative()) {
+            run_step.addArgs(&.{
+                "-target", try t.query.zigTriple(b.allocator),
+            });
+        }
+    }
+    run_step.addArgs(&.{ "-MMD", "-MF" });
+    _ = run_step.addDepFileOutputArg(b.fmt("{s}.d", .{output_name}));
+    return output;
+}
+
+fn generate_header_for(
+    b: *std.Build,
+    name: []const u8,
+    input_file: LazyPath,
+    api_export: ?*std.ArrayList(LazyPath),
+    nlua0: *std.Build.Step.Compile,
+    include_path: []const LazyPath,
+    target: ?std.Build.ResolvedTarget,
+    gen_headers: *std.Build.Step.WriteFile,
+    nvim_header: bool,
+) !*std.Build.Step.Run {
+    if (name.len < 2 or !(std.mem.eql(u8, ".c", name[name.len - 2 ..]) or std.mem.eql(u8, ".h", name[name.len - 2 ..]))) return error.InvalidBaseName;
+    const basename = name[0 .. name.len - 2];
+    const i_file = try run_preprocessor(b, input_file, b.fmt("{s}.i", .{basename}), .{
+        .include_dirs = include_path,
+        .c_macros = &.{ "_GNU_SOURCE", "ZIG_BUILD" },
+        .target = target,
+    });
+    const run_step = b.addRunArtifact(nlua0);
+    run_step.addFileArg(b.path("src/gen/gen_declarations.lua"));
+    run_step.addFileArg(input_file);
+    const gen_name = b.fmt("{s}.{s}.generated.h", .{ basename, if (nvim_header) "h.inline" else "c" });
+    _ = gen_header(b, run_step, gen_name, gen_headers);
+    if (nvim_header) {
+        run_step.addArg("SKIP");
+    } else {
+        const h_file = gen_header(b, run_step, b.fmt("{s}.h.generated.h", .{basename}), gen_headers);
+        if (api_export) |api_files| {
+            try api_files.append(h_file);
+        }
+    }
+
+    run_step.addFileArg(i_file);
+    run_step.addArg(gen_name);
+    return run_step;
+}
diff --git a/src/gen/gen_vimvim.lua b/src/gen/gen_vimvim.lua
index 4cd88f0188..f2d3e18e72 100644
--- a/src/gen/gen_vimvim.lua
+++ b/src/gen/gen_vimvim.lua
@@ -2,6 +2,10 @@ local mpack = vim.mpack
 
 local syntax_file = arg[1]
 local funcs_file = arg[2]
+local options_file = arg[3]
+local auevents_file = arg[4]
+local ex_cmds_file = arg[5]
+local vvars_file = arg[6]
 
 local lld = {}
 local syn_fd = assert(io.open(syntax_file, 'w'))
@@ -15,10 +19,10 @@ local function w(s)
   end
 end
 
-local options = require('nvim.options')
-local auevents = require('nvim.auevents')
-local ex_cmds = require('nvim.ex_cmds')
-local vvars = require('nvim.vvars')
+local options = loadfile(options_file)()
+local auevents = loadfile(auevents_file)()
+local ex_cmds = loadfile(ex_cmds_file)()
+local vvars = loadfile(vvars_file)()
 
 local function cmd_kw(prev_cmd, cmd)
   if not prev_cmd then
diff --git a/src/nlua0.zig b/src/nlua0.zig
new file mode 100644
index 0000000000..042ff01609
--- /dev/null
+++ b/src/nlua0.zig
@@ -0,0 +1,120 @@
+//! "nlua" is an abbreviation for nvim flavored lua, i e lua with the
+//! extended standard library functionality added by nvim such as json, mpack
+//! and libuv and a range of vim.* utility functions.
+//!
+//! nlua0 is an interpreter for the "bootstrap" lua code we need to run before
+//! nvim can be built, in order to run lua scripts which process and generate
+//! more .c code, which still need these extensions.
+const std = @import("std");
+const ziglua = @import("ziglua");
+const options = @import("options");
+
+const embedded_data = @import("embedded_data");
+
+// these are common dependencies used by many generators
+const hashy = @embedFile("gen/hashy.lua");
+const c_grammar = @embedFile("gen/c_grammar.lua");
+
+const Lua = ziglua.Lua;
+
+extern "c" fn luaopen_mpack(ptr: *anyopaque) c_int;
+extern "c" fn luaopen_lpeg(ptr: *anyopaque) c_int;
+extern "c" fn luaopen_bit(ptr: *anyopaque) c_int;
+
+fn init() !*Lua {
+    // Initialize the Lua vm
+    var lua = try Lua.init(std.heap.c_allocator);
+    lua.openLibs();
+
+    // this sets _G.vim by itself, so we don't need to
+    try lua.loadBuffer(embedded_data.shared_module, "shared.lua");
+    lua.call(.{ .results = 1 });
+
+    try lua.loadBuffer(embedded_data.inspect_module, "inspect.lua");
+    lua.call(.{ .results = 1 });
+    lua.setField(-2, "inspect");
+
+    try lua.loadBuffer(embedded_data.iter_module, "iter.lua");
+    lua.call(.{ .results = 1 });
+    lua.setField(-2, "iter");
+
+    _ = try lua.getGlobal("package");
+    _ = lua.getField(-1, "preload");
+    try lua.loadBuffer(hashy, "hashy.lua"); // [package, preload, hashy]
+    lua.setField(-2, "gen.hashy");
+    try lua.loadBuffer(c_grammar, "c_grammar.lua"); // [package, preload, c_grammar]
+    lua.setField(-2, "gen.c_grammar");
+    lua.pop(2);
+
+    const retval = luaopen_mpack(lua);
+    if (retval != 1) return error.LoadError;
+    _ = lua.getField(-1, "NIL"); // [vim, mpack, NIL]
+    lua.setField(-3, "NIL"); // vim.NIL = mpack.NIL (wow BOB wow)
+    lua.setField(-2, "mpack");
+
+    const retval2 = luaopen_lpeg(lua);
+    if (retval2 != 1) return error.LoadError;
+    lua.setField(-3, "lpeg");
+
+    lua.pop(2);
+
+    if (!options.use_luajit) {
+        lua.pop(luaopen_bit(lua));
+    }
+    return lua;
+}
+
+pub fn main() !void {
+    const argv = std.os.argv;
+
+    const lua = try init();
+    defer lua.deinit();
+
+    if (argv.len < 2) {
+        std.debug.print("USAGE: nlua0 script.lua args...\n\n", .{});
+        return;
+    }
+    lua.createTable(@intCast(argv.len - 2), 1);
+    for (0.., argv[1..]) |i, arg| {
+        _ = lua.pushString(std.mem.span(arg));
+        lua.rawSetIndex(-2, @intCast(i));
+    }
+    lua.setGlobal("arg");
+
+    _ = try lua.getGlobal("debug");
+    _ = lua.getField(-1, "traceback");
+    try lua.loadFile(std.mem.span(argv[1]));
+    lua.protectedCall(.{ .msg_handler = -2 }) catch |e| {
+        if (e == error.LuaRuntime) {
+            const msg = try lua.toString(-1);
+            std.debug.print("{s}\n", .{msg});
+        }
+        return e;
+    };
+}
+
+fn do_ret1(lua: *Lua, str: [:0]const u8) !void {
+    try lua.loadString(str);
+    try lua.protectedCall(.{ .results = 1 });
+}
+
+test "simple test" {
+    const lua = try init();
+    defer lua.deinit();
+
+    try do_ret1(lua, "return vim.isarray({2,3})");
+    try std.testing.expectEqual(true, lua.toBoolean(-1));
+    lua.pop(1);
+
+    try do_ret1(lua, "return vim.isarray({a=2,b=3})");
+    try std.testing.expectEqual(false, lua.toBoolean(-1));
+    lua.pop(1);
+
+    try do_ret1(lua, "return vim.inspect(vim.mpack.decode('\\146\\42\\69'))");
+    try std.testing.expectEqualStrings("{ 42, 69 }", try lua.toString(-1));
+    lua.pop(1);
+
+    try do_ret1(lua, "return require'bit'.band(7,12)");
+    try std.testing.expectEqualStrings("4", try lua.toString(-1));
+    lua.pop(1);
+}
diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index 2a60eefe7c..f6112a3c2a 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -595,8 +595,11 @@ add_custom_command(
                          ${LUA_API_C_BINDINGS}
                          ${GENERATED_KEYSETS_DEFS}
                          ${UI_METADATA}
-                         ${NVIM_VERSION_GIT_H}
+                         ${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA}
+                         ${GENERATOR_DIR}/dump_bin_array.lua
+                         ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
                          ${API_HEADERS}
+
   DEPENDS
     ${LUA_GEN_DEPS}
     ${API_HEADERS}
@@ -604,6 +607,7 @@ add_custom_command(
     ${API_DISPATCH_GENERATOR}
     ${GENERATOR_C_GRAMMAR}
     ${GENERATOR_HASHY}
+    ${GENERATOR_DIR}/dump_bin_array.lua
     ${UI_METADATA}
     ${NVIM_VERSION_LUA}
     ${NVIM_VERSION_GIT_H}
@@ -666,27 +670,27 @@ add_custom_command(
 )
 
 add_custom_command(OUTPUT ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS}
-  COMMAND ${LUA_GEN} ${EX_CMDS_GENERATOR} ${GENERATED_INCLUDES_DIR} ${GENERATED_DIR}
+  COMMAND ${LUA_GEN} ${EX_CMDS_GENERATOR} ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS} ${CMAKE_CURRENT_LIST_DIR}/ex_cmds.lua
   DEPENDS ${LUA_GEN_DEPS} ${EX_CMDS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/ex_cmds.lua
 )
 
 add_custom_command(OUTPUT ${GENERATED_FUNCS} ${FUNCS_DATA}
-  COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_DIR} ${FUNCS_METADATA} ${FUNCS_DATA}
+  COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua
   DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${FUNCS_METADATA}
 )
 
 add_custom_command(OUTPUT ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP}
-  COMMAND ${LUA_GEN} ${EVENTS_GENERATOR} ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP}
+  COMMAND ${LUA_GEN} ${EVENTS_GENERATOR} ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP} ${CMAKE_CURRENT_LIST_DIR}/auevents.lua
   DEPENDS ${LUA_GEN_DEPS} ${EVENTS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/auevents.lua
 )
 
 add_custom_command(OUTPUT ${GENERATED_KEYCODE_NAMES}
-  COMMAND ${LUA_GEN} ${KEYCODES_GENERATOR} ${GENERATED_KEYCODE_NAMES}
+  COMMAND ${LUA_GEN} ${KEYCODES_GENERATOR} ${GENERATED_KEYCODE_NAMES} ${CMAKE_CURRENT_LIST_DIR}/keycodes.lua
   DEPENDS ${LUA_GEN_DEPS} ${KEYCODES_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/keycodes.lua
 )
 
 add_custom_command(OUTPUT ${GENERATED_OPTIONS} ${GENERATED_OPTIONS_ENUM} ${GENERATED_OPTIONS_MAP} ${GENERATED_OPTION_VARS}
-  COMMAND ${LUA_GEN} ${OPTIONS_GENERATOR} ${GENERATED_OPTIONS} ${GENERATED_OPTIONS_ENUM} ${GENERATED_OPTIONS_MAP} ${GENERATED_OPTION_VARS}
+  COMMAND ${LUA_GEN} ${OPTIONS_GENERATOR} ${GENERATED_OPTIONS} ${GENERATED_OPTIONS_ENUM} ${GENERATED_OPTIONS_MAP} ${GENERATED_OPTION_VARS} ${CMAKE_CURRENT_LIST_DIR}/options.lua
   DEPENDS ${LUA_GEN_DEPS} ${OPTIONS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/options.lua
 )
 
@@ -967,7 +971,7 @@ add_target(doc-vim
   )
 
 add_target(doc-eval
-  COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua
+  COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${FUNCS_METADATA}
   DEPENDS
     nvim
     ${FUNCS_METADATA}
diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c
index 292588ee64..98d99c1c95 100644
--- a/src/nvim/lua/converter.c
+++ b/src/nvim/lua/converter.c
@@ -661,7 +661,7 @@ static inline void nlua_create_typed_table(lua_State *lstate, const size_t narr,
 void nlua_push_String(lua_State *lstate, const String s, int flags)
   FUNC_ATTR_NONNULL_ALL
 {
-  lua_pushlstring(lstate, s.data, s.size);
+  lua_pushlstring(lstate, s.size ? s.data : "", s.size);
 }
 
 /// Convert given Integer to Lua number
diff --git a/src/nvim/macros_defs.h b/src/nvim/macros_defs.h
index 67da29031c..84417659e9 100644
--- a/src/nvim/macros_defs.h
+++ b/src/nvim/macros_defs.h
@@ -1,6 +1,8 @@
 #pragma once
 
-#include "auto/config.h"
+#ifndef NVIM_NLUA0
+# include "auto/config.h"
+#endif
 
 // EXTERN is only defined in main.c. That's where global variables are
 // actually defined and initialized.
diff --git a/src/nvim/os/lang.c b/src/nvim/os/lang.c
index 6f6eba805a..47eeef702a 100644
--- a/src/nvim/os/lang.c
+++ b/src/nvim/os/lang.c
@@ -1,4 +1,4 @@
-#ifdef __APPLE__
+#if defined(__APPLE__) && !defined(ZIG_BUILD)
 # define Boolean CFBoolean  // Avoid conflict with API's Boolean
 # define FileInfo CSFileInfo  // Avoid conflict with API's Fileinfo
 # include <CoreServices/CoreServices.h>
@@ -340,7 +340,7 @@ char *get_locales(expand_T *xp, int idx)
 
 void lang_init(void)
 {
-#ifdef __APPLE__
+#if defined(__APPLE__) && !defined(ZIG_BUILD)
   if (!os_env_exists("LANG", true)) {
     char buf[50] = { 0 };
 
diff --git a/src/nvim/os/pty_proc_unix.c b/src/nvim/os/pty_proc_unix.c
index 10a464bbc3..98d6f7a2f9 100644
--- a/src/nvim/os/pty_proc_unix.c
+++ b/src/nvim/os/pty_proc_unix.c
@@ -13,6 +13,9 @@
 // forkpty is not in POSIX, so headers are platform-specific
 #if defined(__FreeBSD__) || defined(__DragonFly__)
 # include <libutil.h>
+// TODO(bfredl): this is avaliable on darwin, but there is an issue with cross-compile headers
+#elif defined(__APPLE__) && !defined(HAVE_FORKPTY)
+int forkpty(int *, char *, const struct termios *, const struct winsize *);
 #elif defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
 # include <util.h>
 #elif defined(__sun)