fix(api): use text_locked() to check textlock

Problem: some API functions that check textlock (usually those that can change
curwin or curbuf) can break the cmdwin.

Solution: make FUNC_API_CHECK_TEXTLOCK call text_locked() instead, which already
checks for textlock, cmdwin and `<expr>` status.

Add FUNC_API_TEXTLOCK_ALLOW_CMDWIN to allow such functions to be usable in the
cmdwin if they can work properly there; the opt-in nature of this attribute
should hopefully help mitigate future bugs.

Also fix a regression in #22634 that made functions checking textlock usable in
`<expr>` mappings, and rename FUNC_API_CHECK_TEXTLOCK to FUNC_API_TEXTLOCK.

2 files changed, 22 insertions, 6 deletions

diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
index d83658e977..f33da452ff 100644
--- a/src/nvim/generators/c_grammar.lua
+++ b/src/nvim/generators/c_grammar.lua
@@ -47,7 +47,8 @@ local c_proto = Ct(
   (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) *
   (fill * Cg((P('FUNC_API_REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) *
   (fill * Cg((P('FUNC_API_LUA_ONLY') * Cc(true)), 'lua_only') ^ -1) *
-  (fill * Cg((P('FUNC_API_CHECK_TEXTLOCK') * Cc(true)), 'check_textlock') ^ -1) *
+  (fill * (Cg(P('FUNC_API_TEXTLOCK_ALLOW_CMDWIN') * Cc(true), 'textlock_allow_cmdwin') +
+           Cg(P('FUNC_API_TEXTLOCK') * Cc(true), 'textlock')) ^ -1) *
   (fill * Cg((P('FUNC_API_REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) *
   (fill * Cg((P('FUNC_API_COMPOSITOR_IMPL') * Cc(true)), 'compositor_impl') ^ -1) *
   (fill * Cg((P('FUNC_API_CLIENT_IMPL') * Cc(true)), 'client_impl') ^ -1) *
diff --git a/src/nvim/generators/gen_api_dispatch.lua b/src/nvim/generators/gen_api_dispatch.lua
index 244921de31..bf211f5273 100644
--- a/src/nvim/generators/gen_api_dispatch.lua
+++ b/src/nvim/generators/gen_api_dispatch.lua
@@ -210,10 +210,11 @@ local keysets_defs = io.open(keysets_outputf, 'wb')
 --  so that the dispatcher can find the C functions that you are creating!
 -- ===========================================================================
 output:write([[
+#include "nvim/ex_docmd.h"
+#include "nvim/ex_getln.h"
 #include "nvim/log.h"
 #include "nvim/map.h"
 #include "nvim/msgpack_rpc/helpers.h"
-#include "nvim/vim.h"
 
 #include "nvim/api/autocmd.h"
 #include "nvim/api/buffer.h"
@@ -389,8 +390,13 @@ for i = 1, #functions do
       args[#args + 1] = converted
     end
 
-    if fn.check_textlock then
-      output:write('\n  if (textlock != 0) {')
+    if fn.textlock then
+      output:write('\n  if (text_locked()) {')
+      output:write('\n    api_set_error(error, kErrorTypeException, "%s", get_text_locked_msg());')
+      output:write('\n    goto cleanup;')
+      output:write('\n  }\n')
+    elseif fn.textlock_allow_cmdwin then
+      output:write('\n  if (textlock != 0 || expr_map_locked()) {')
       output:write('\n    api_set_error(error, kErrorTypeException, "%s", e_textlock);')
       output:write('\n    goto cleanup;')
       output:write('\n  }\n')
@@ -525,6 +531,8 @@ output:write([[
 #include <lualib.h>
 #include <lauxlib.h>
 
+#include "nvim/ex_docmd.h"
+#include "nvim/ex_getln.h"
 #include "nvim/func_attr.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
@@ -565,9 +573,16 @@ local function process_function(fn)
     ]], fn.name))
   end
 
-  if fn.check_textlock then
+  if fn.textlock then
+    write_shifted_output(output, [[
+    if (text_locked()) {
+      api_set_error(&err, kErrorTypeException, "%s", get_text_locked_msg());
+      goto exit_0;
+    }
+    ]])
+  elseif fn.textlock_allow_cmdwin then
     write_shifted_output(output, [[
-    if (textlock != 0) {
+    if (textlock != 0 || expr_map_locked()) {
       api_set_error(&err, kErrorTypeException, "%s", e_textlock);
       goto exit_0;
     }