fix(lua): blast radius of broken `_G.debug` #41507

Problem:
`nlua_pcall()` references `_G.debug.traceback`. If user code deletes it
or breaks it some other way, various Lua features are broken.

    _G.debug = nil
    vim.schedule(function() end)
    vim.wait(100)

    E5113: Lua chunk: attempt to index a nil value
    stack traceback:
            [C]: in function 'loop_poll'
            [string "vim/_core/editor"]:176: in function 'wait'
            crash.lua:3: in main chunk
    PANIC: unprotected error in call to Lua API (attempt to index a nil value)

Solution:
Check `_G.debug.traceback` before using it as errfunc. If it's broken,
omit the traceback and say so in the error message.

Note: We could cache `_G.debug` in LUA_REGISTRYINDEX on startup, but
that would prevent plugins from providing custom functionality there
(and we happen to do so in `tui_spec.lua` for example).

1 files changed, 22 insertions, 3 deletions

diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c
index 402fbcd52f..5ac5e7203b 100644
--- a/src/nvim/lua/executor.c
+++ b/src/nvim/lua/executor.c
@@ -314,21 +314,40 @@ void nlua_error(lua_State *const lstate, const char *const msg)
   lua_pop(lstate, 1);
 }
 
-/// Like lua_pcall, but use debug.traceback as errfunc.
+/// Dummy errfunc used if `_G.debug.traceback` is broken: returns the error object unchanged.
+static int nlua_no_traceback(lua_State *lstate)
+{
+  return 1;
+}
+
+/// Like lua_pcall, but use `debug.traceback` as errfunc. Called by most Lua entrypoints (`:lua`,
+/// `luaeval()`, `v:lua`, API calls, …). Not used by `pcall()` itself.
 ///
 /// @param lstate Lua interpreter state
 /// @param[in] nargs Number of arguments expected by the function being called.
 /// @param[in] nresults Number of results the function returns.
 int nlua_pcall(lua_State *lstate, int nargs, int nresults)
 {
+  // Detect broken `_G.debug` (by user code). #41504
   lua_getglobal(lstate, "debug");
-  lua_getfield(lstate, -1, "traceback");
-  lua_remove(lstate, -2);
+  if (lua_istable(lstate, -1)) {
+    lua_getfield(lstate, -1, "traceback");
+    lua_remove(lstate, -2);
+  }
+  bool no_traceback = !lua_isfunction(lstate, -1);  // `_G.debug.traceback` is a function?
+  if (no_traceback) {
+    lua_pop(lstate, 1);
+    lua_pushcfunction(lstate, nlua_no_traceback);
+  }
   lua_insert(lstate, -2 - nargs);
   int pre_top = lua_gettop(lstate);
   int status = lua_pcall(lstate, nargs, nresults, -2 - nargs);
   if (status) {
     lua_remove(lstate, -2);
+    if (no_traceback && status == LUA_ERRRUN && lua_isstring(lstate, -1)) {
+      lua_pushliteral(lstate, "\nNo traceback: debug.traceback is not a function");
+      lua_concat(lstate, 2);
+    }
   } else {
     if (nresults == LUA_MULTRET) {
       nresults = lua_gettop(lstate) - (pre_top - nargs - 1);