feat(api): allow open_win/win_set_buf in the cmdwin in some cases

Problem: As discussed on Matrix, there was some interest in having
`nvim_open_win` again be able to open floats in the cmdwin (e.g: displaying a
hover doc related to what's in the cmdwin). After #23228, this was disallowed.

Solution: Allow `nvim_open_win` in the cmdwin as long as `!enter` and
`buffer != curbuf` (the former can cause all sorts of issues, and the latter
can crash Nvim after closing cmdwin). Also allow `nvim_win_set_buf` in a similar
fashion.

Note that we're not *entirely* sure if this is 100% safe (cmdwin is a
global-state-using-main-loop-calling beast), but this seems to work OK..?

Also:
  - Check the buffer argument of `nvim_open_win` earlier, and abort if it's
    invalid (it used to still open a window in this case).

  - Untranslate `e_cmdwin` errors in the API (other errors in the API are not
    translated: although not detailed in the API contract yet, errors are
    supposed to be stable).

4 files changed, 28 insertions, 16 deletions

diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 8738b3e38e..b4a6fa718b 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -967,7 +967,7 @@ Integer nvim_open_term(Buffer buffer, DictionaryOf(LuaRef) opts, Error *err)
   }
 
   if (cmdwin_type != 0 && buf == curbuf) {
-    api_set_error(err, kErrorTypeException, "%s", _(e_cmdwin));
+    api_set_error(err, kErrorTypeException, "%s", e_cmdwin);
     return 0;
   }
 
diff --git a/src/nvim/api/win_config.c b/src/nvim/api/win_config.c
index 6ca36a0daf..81a239d913 100644
--- a/src/nvim/api/win_config.c
+++ b/src/nvim/api/win_config.c
@@ -158,8 +158,17 @@
 /// @return Window handle, or 0 on error
 Window nvim_open_win(Buffer buffer, Boolean enter, Dict(float_config) *config, Error *err)
   FUNC_API_SINCE(6)
-  FUNC_API_TEXTLOCK
+  FUNC_API_TEXTLOCK_ALLOW_CMDWIN
 {
+  buf_T *buf = find_buffer_by_handle(buffer, err);
+  if (!buf) {
+    return 0;
+  }
+  if (cmdwin_type != 0 && (enter || buf == curbuf)) {
+    api_set_error(err, kErrorTypeException, "%s", e_cmdwin);
+    return 0;
+  }
+
   FloatConfig fconfig = FLOAT_CONFIG_INIT;
   if (!parse_float_config(config, &fconfig, false, true, err)) {
     return 0;
@@ -173,7 +182,7 @@ Window nvim_open_win(Buffer buffer, Boolean enter, Dict(float_config) *config, E
   }
   // autocmds in win_enter or win_set_buf below may close the window
   if (win_valid(wp) && buffer > 0) {
-    win_set_buf(wp->handle, buffer, fconfig.noautocmd, err);
+    win_set_buf(wp, buf, fconfig.noautocmd, err);
   }
   if (!win_valid(wp)) {
     api_set_error(err, kErrorTypeException, "Window was closed immediately");
diff --git a/src/nvim/api/window.c b/src/nvim/api/window.c
index 5480584aa5..666f09e890 100644
--- a/src/nvim/api/window.c
+++ b/src/nvim/api/window.c
@@ -50,9 +50,18 @@ Buffer nvim_win_get_buf(Window window, Error *err)
 /// @param[out] err Error details, if any
 void nvim_win_set_buf(Window window, Buffer buffer, Error *err)
   FUNC_API_SINCE(5)
-  FUNC_API_TEXTLOCK
+  FUNC_API_TEXTLOCK_ALLOW_CMDWIN
 {
-  win_set_buf(window, buffer, false, err);
+  win_T *win = find_window_by_handle(window, err);
+  buf_T *buf = find_buffer_by_handle(buffer, err);
+  if (!win || !buf) {
+    return;
+  }
+  if (cmdwin_type != 0 && (win == curwin || buf == curbuf)) {
+    api_set_error(err, kErrorTypeException, "%s", e_cmdwin);
+    return;
+  }
+  win_set_buf(win, buf, false, err);
 }
 
 /// Gets the (1,0)-indexed, buffer-relative cursor position for a given window
@@ -396,7 +405,7 @@ void nvim_win_close(Window window, Boolean force, Error *err)
     if (win == curwin) {
       cmdwin_result = Ctrl_C;
     } else {
-      api_set_error(err, kErrorTypeException, "%s", _(e_cmdwin));
+      api_set_error(err, kErrorTypeException, "%s", e_cmdwin);
     }
     return;
   }
diff --git a/src/nvim/window.c b/src/nvim/window.c
index e230bde95c..fa7ca53b08 100644
--- a/src/nvim/window.c
+++ b/src/nvim/window.c
@@ -698,15 +698,9 @@ static void cmd_with_count(char *cmd, char *bufp, size_t bufsize, int64_t Prenum
   }
 }
 
-void win_set_buf(Window window, Buffer buffer, bool noautocmd, Error *err)
+void win_set_buf(win_T *win, buf_T *buf, bool noautocmd, Error *err)
+  FUNC_ATTR_NONNULL_ALL
 {
-  win_T *win = find_window_by_handle(window, err);
-  buf_T *buf = find_buffer_by_handle(buffer, err);
-
-  if (!win || !buf) {
-    return;
-  }
-
   tabpage_T *tab = win_find_tabpage(win);
 
   // no redrawing and don't set the window title
@@ -720,7 +714,7 @@ void win_set_buf(Window window, Buffer buffer, bool noautocmd, Error *err)
     api_set_error(err,
                   kErrorTypeException,
                   "Failed to switch to window %d",
-                  window);
+                  win->handle);
   }
 
   try_start();
@@ -729,7 +723,7 @@ void win_set_buf(Window window, Buffer buffer, bool noautocmd, Error *err)
     api_set_error(err,
                   kErrorTypeException,
                   "Failed to set buffer %d",
-                  buffer);
+                  buf->handle);
   }
 
   // If window is not current, state logic will not validate its cursor.