fix(api): disallow some more functions during textlock

Problem: nvim_buf_set_text(), nvim_open_term() and termopen() all change buffer
text, which is forbidden during textlock. Additionally, nvim_open_term() and
termopen() may be used to convert the cmdwin buffer into a terminal buffer,
which is weird.

Solution: Allow nvim_buf_set_text() and nvim_open_term() in the cmdwin, but
disallow nvim_open_term() from converting the cmdwin buffer into a terminal
buffer. termopen() is not allowed in the cmdwin (as it always operates on
curbuf), so just check text_locked().

Also happens to improve the error in #21055: nvim_buf_set_text() was callable
during textlock, but happened to check textlock indirectly via u_save();
however, this caused the error to be overwritten by an unhelpful "Failed to
save undo information" message when msg_list == NULL (e.g: an `<expr>` mapping
invoked outside of do_cmdline()).

6 files changed, 52 insertions, 1 deletions

diff --git a/runtime/doc/api.txt b/runtime/doc/api.txt
index 203468cc8a..46c5f2ea18 100644
--- a/runtime/doc/api.txt
+++ b/runtime/doc/api.txt
@@ -1200,6 +1200,9 @@ nvim_open_term({buffer}, {opts})                            *nvim_open_term()*
     |nvim_chan_send()| can be called immediately to process sequences in a
     virtual terminal having the intended size.
 
+    Attributes: ~
+        not allowed when |textlock| is active
+
     Parameters: ~
       • {buffer}  the buffer to use (expected to be empty)
       • {opts}    Optional parameters.
@@ -2426,6 +2429,9 @@ nvim_buf_set_text({buffer}, {start_row}, {start_col}, {end_row}, {end_col},
     Prefer |nvim_buf_set_lines()| if you are only adding or deleting entire
     lines.
 
+    Attributes: ~
+        not allowed when |textlock| is active
+
     Parameters: ~
       • {buffer}       Buffer handle, or 0 for current buffer
       • {start_row}    First line index
diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c
index 7cd9686544..02b97b0ae1 100644
--- a/src/nvim/api/buffer.c
+++ b/src/nvim/api/buffer.c
@@ -515,6 +515,7 @@ end:
 void nvim_buf_set_text(uint64_t channel_id, Buffer buffer, Integer start_row, Integer start_col,
                        Integer end_row, Integer end_col, ArrayOf(String) replacement, Error *err)
   FUNC_API_SINCE(7)
+  FUNC_API_TEXTLOCK_ALLOW_CMDWIN
 {
   MAXSIZE_TEMP_ARRAY(scratch, 1);
   if (replacement.size == 0) {
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index eacc833c58..8c40e5ccd7 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -951,12 +951,18 @@ fail:
 /// @return Channel id, or 0 on error
 Integer nvim_open_term(Buffer buffer, DictionaryOf(LuaRef) opts, Error *err)
   FUNC_API_SINCE(7)
+  FUNC_API_TEXTLOCK_ALLOW_CMDWIN
 {
   buf_T *buf = find_buffer_by_handle(buffer, err);
   if (!buf) {
     return 0;
   }
 
+  if (cmdwin_type != 0 && buf == curbuf) {
+    api_set_error(err, kErrorTypeException, "%s", _(e_cmdwin));
+    return 0;
+  }
+
   LuaRef cb = LUA_NOREF;
   for (size_t i = 0; i < opts.size; i++) {
     String k = opts.items[i].key;
diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c
index f348a61075..7c3c5cc274 100644
--- a/src/nvim/eval/funcs.c
+++ b/src/nvim/eval/funcs.c
@@ -8385,7 +8385,10 @@ static void f_termopen(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
   if (check_secure()) {
     return;
   }
-
+  if (text_locked()) {
+    text_locked_msg();
+    return;
+  }
   if (curbuf->b_changed) {
     emsg(_("Can only call this function in an unmodified buffer"));
     return;
diff --git a/test/functional/terminal/buffer_spec.lua b/test/functional/terminal/buffer_spec.lua
index 4ce354b9a9..bd898ba99e 100644
--- a/test/functional/terminal/buffer_spec.lua
+++ b/test/functional/terminal/buffer_spec.lua
@@ -498,3 +498,17 @@ if is_os('win') then
     end)
   end)
 end
+
+describe('termopen()', function()
+  before_each(clear)
+
+  it('disallowed when textlocked and in cmdwin buffer', function()
+    command("autocmd TextYankPost <buffer> ++once call termopen('foo')")
+    matches("Vim%(call%):E565: Not allowed to change text or change window$",
+      pcall_err(command, "normal! yy"))
+
+    feed("q:")
+    eq("Vim:E11: Invalid in command-line window; <CR> executes, CTRL-C quits",
+      pcall_err(funcs.termopen, "bar"))
+  end)
+end)
diff --git a/test/functional/vimscript/api_functions_spec.lua b/test/functional/vimscript/api_functions_spec.lua
index 683ae2446c..6548548a9e 100644
--- a/test/functional/vimscript/api_functions_spec.lua
+++ b/test/functional/vimscript/api_functions_spec.lua
@@ -54,11 +54,23 @@ describe('eval-API', function()
     matches('Vim%(call%):E5555: API call: E565: Not allowed to change text or change window$',
        pcall_err(command, "normal! yy"))
 
+    command("autocmd TextYankPost <buffer> ++once call nvim_open_term(0, {})")
+    matches('Vim%(call%):E5555: API call: E565: Not allowed to change text or change window$',
+       pcall_err(command, "normal! yy"))
+
     -- Functions checking textlock should also not be usable from <expr> mappings.
     command("inoremap <expr> <f2> nvim_win_close(0, 1)")
     eq('Vim(normal):E5555: API call: E565: Not allowed to change text or change window',
        pcall_err(command, [[execute "normal i\<f2>"]]))
 
+    -- Text-changing functions gave a "Failed to save undo information" error when called from an
+    -- <expr> mapping outside do_cmdline() (msg_list == NULL), so use feed() to test this.
+    command("inoremap <expr> <f2> nvim_buf_set_text(0, 0, 0, 0, 0, ['hi'])")
+    meths.set_vvar('errmsg', '')
+    feed("i<f2><esc>")
+    eq('E5555: API call: E565: Not allowed to change text or change window',
+       meths.get_vvar('errmsg'))
+
     -- Some functions checking textlock (usually those that may change the current window or buffer)
     -- also ought to not be usable in the cmdwin.
     feed("q:")
@@ -68,6 +80,15 @@ describe('eval-API', function()
     -- But others, like nvim_buf_set_lines(), which just changes text, is OK.
     curbufmeths.set_lines(0, -1, 1, {"wow!"})
     eq({'wow!'}, curbufmeths.get_lines(0, -1, 1))
+
+    -- Turning the cmdwin buffer into a terminal buffer would be pretty weird.
+    eq('E11: Invalid in command-line window; <CR> executes, CTRL-C quits',
+       pcall_err(meths.open_term, 0, {}))
+
+    -- But turning a different buffer into a terminal from the cmdwin is OK.
+    local term_buf = meths.create_buf(false, true)
+    meths.open_term(term_buf, {})
+    eq('terminal', meths.get_option_value("buftype", {buf = term_buf}))
   end)
 
   it("use buffer numbers and windows ids as handles", function()