fix(api): revert unintended change of optional bool params

Currently (as of nvim 0.9), the behavior of boolean params in
vim.api lua wrappers is inconsistent for optional parameters
(part of an `opts` dict) compared to positional parameters.

This was inadvertently changed in #24524 . While cleaning up this
inconsistency is something we might want eventually, it needs
to be discussed separately and the impact of existing code considered.

2 files changed, 58 insertions, 1 deletions

diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c
index 2df6458198..385ac11546 100644
--- a/src/nvim/lua/converter.c
+++ b/src/nvim/lua/converter.c
@@ -849,6 +849,9 @@ Integer nlua_pop_Integer(lua_State *lstate, Error *err)
 
 /// Convert lua value to boolean
 ///
+/// Despite the name of the function, this uses lua semantics for booleans.
+/// thus `err` is never set as any lua value can be co-erced into a lua bool
+///
 /// Always pops one value from the stack.
 Boolean nlua_pop_Boolean(lua_State *lstate, Error *err)
   FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
@@ -858,6 +861,36 @@ Boolean nlua_pop_Boolean(lua_State *lstate, Error *err)
   return ret;
 }
 
+/// Convert lua value to boolean
+///
+/// This follows API conventions for a Boolean value, compare api_object_to_bool
+///
+/// Always pops one value from the stack.
+Boolean nlua_pop_Boolean_strict(lua_State *lstate, Error *err)
+  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  Boolean ret = false;
+  switch (lua_type(lstate, -1)) {
+  case LUA_TBOOLEAN:
+    ret = lua_toboolean(lstate, -1);
+    break;
+
+  case LUA_TNUMBER:
+    ret = (lua_tonumber(lstate, -1) != 0);
+    break;
+
+  case LUA_TNIL:
+    ret = false;
+    break;
+
+  default:
+    api_set_error(err, kErrorTypeValidation, "not a boolean");
+  }
+
+  lua_pop(lstate, 1);
+  return ret;
+}
+
 /// Check whether typed table on top of the stack has given type
 ///
 /// @param[in]  lstate  Lua state.
@@ -1318,7 +1351,7 @@ void nlua_pop_keydict(lua_State *L, void *retval, FieldHashfn hashy, char **err_
     } else if (field->type == kObjectTypeInteger) {
       *(Integer *)mem = nlua_pop_Integer(L, err);
     } else if (field->type == kObjectTypeBoolean) {
-      *(Boolean *)mem = nlua_pop_Boolean(L, err);
+      *(Boolean *)mem = nlua_pop_Boolean_strict(L, err);
     } else if (field->type == kObjectTypeString) {
       *(String *)mem = nlua_pop_String(L, err);
     } else if (field->type == kObjectTypeFloat) {
diff --git a/test/functional/lua/api_spec.lua b/test/functional/lua/api_spec.lua
index 5dfc2eb83b..d808693a9e 100644
--- a/test/functional/lua/api_spec.lua
+++ b/test/functional/lua/api_spec.lua
@@ -9,6 +9,7 @@ local eval = helpers.eval
 local NIL = helpers.NIL
 local eq = helpers.eq
 local exec_lua = helpers.exec_lua
+local pcall_err = helpers.pcall_err
 
 before_each(clear)
 
@@ -171,6 +172,29 @@ describe('luaeval(vim.api.…)', function()
     eq(4, eval([[type(luaeval('vim.api.nvim__id_dictionary({})'))]]))
   end)
 
+  it('converts booleans in positional args', function()
+    eq({''}, exec_lua [[ return vim.api.nvim_buf_get_lines(0, 0, 10, false) ]])
+    eq({''}, exec_lua [[ return vim.api.nvim_buf_get_lines(0, 0, 10, nil) ]])
+    eq('Index out of bounds', pcall_err(exec_lua, [[ return vim.api.nvim_buf_get_lines(0, 0, 10, true) ]]))
+    eq('Index out of bounds', pcall_err(exec_lua, [[ return vim.api.nvim_buf_get_lines(0, 0, 10, 1) ]]))
+
+    -- this follows lua conventions for bools (not api convention for Boolean)
+    eq('Index out of bounds', pcall_err(exec_lua, [[ return vim.api.nvim_buf_get_lines(0, 0, 10, 0) ]]))
+    eq('Index out of bounds', pcall_err(exec_lua, [[ return vim.api.nvim_buf_get_lines(0, 0, 10, {}) ]]))
+  end)
+
+  it('converts booleans in optional args', function()
+    eq({}, exec_lua [[ return vim.api.nvim_exec2("echo 'foobar'", {output=false}) ]])
+    eq({}, exec_lua [[ return vim.api.nvim_exec2("echo 'foobar'", {}) ]]) -- same as {output=nil}
+
+    -- API conventions (not lua conventions): zero is falsy
+    eq({}, exec_lua [[ return vim.api.nvim_exec2("echo 'foobar'", {output=0}) ]])
+
+    eq({output='foobar'}, exec_lua [[ return vim.api.nvim_exec2("echo 'foobar'", {output=true}) ]])
+    eq({output='foobar'}, exec_lua [[ return vim.api.nvim_exec2("echo 'foobar'", {output=1}) ]])
+    eq([[Invalid 'output': not a boolean]], pcall_err(exec_lua, [[ return vim.api.nvim_exec2("echo 'foobar'", {output={}}) ]]))
+  end)
+
   it('errors out correctly when working with API', function()
     -- Conversion errors
     eq([[Vim(call):E5108: Error executing lua [string "luaeval()"]:1: Invalid 'obj': Cannot convert given Lua table]],