fix(api): make open_win/win_set_config check if splitting allowed

Problem: splitting is disallowed in some cases to prevent the window layout
changes while a window is closing, but it's not checked for.

Solution: check for this, and set the API error message directly.

(Also sneak in a change to tui.c that got lost from #27352; it's a char* buf,
and the memset is assuming one byte each anyway)

3 files changed, 35 insertions, 11 deletions

diff --git a/src/nvim/api/win_config.c b/src/nvim/api/win_config.c
index 3959e74af9..557c2f37f9 100644
--- a/src/nvim/api/win_config.c
+++ b/src/nvim/api/win_config.c
@@ -246,6 +246,10 @@ Window nvim_open_win(Buffer buffer, Boolean enter, Dict(win_config) *config, Err
       }
     }
 
+    if (!check_split_disallowed_err(parent ? parent : curwin, err)) {
+      return 0;  // error already set
+    }
+
     if (HAS_KEY_X(config, vertical) && !HAS_KEY_X(config, split)) {
       if (config->vertical) {
         fconfig.split = p_spr ? kWinSplitRight : kWinSplitLeft;
@@ -440,6 +444,10 @@ void nvim_win_set_config(Window window, Dict(win_config) *config, Error *err)
       return;
     }
 
+    if (!check_split_disallowed_err(win, err)) {
+      return;  // error already set
+    }
+
     if (was_split) {
       win_T *new_curwin = NULL;
 
diff --git a/src/nvim/tui/tui.c b/src/nvim/tui/tui.c
index 7fae34d33f..c332c17e43 100644
--- a/src/nvim/tui/tui.c
+++ b/src/nvim/tui/tui.c
@@ -1643,7 +1643,7 @@ static void invalidate(TUIData *tui, int top, int bot, int left, int right)
 static void ensure_space_buf_size(TUIData *tui, size_t len)
 {
   if (len > tui->space_buf_len) {
-    tui->space_buf = xrealloc(tui->space_buf, len * sizeof *tui->space_buf);
+    tui->space_buf = xrealloc(tui->space_buf, len);
     memset(tui->space_buf + tui->space_buf_len, ' ', len - tui->space_buf_len);
     tui->space_buf_len = len;
   }
diff --git a/src/nvim/window.c b/src/nvim/window.c
index d5a6e347e7..81f8304b22 100644
--- a/src/nvim/window.c
+++ b/src/nvim/window.c
@@ -905,19 +905,35 @@ void ui_ext_win_viewport(win_T *wp)
   }
 }
 
-/// If "split_disallowed" is set give an error and return FAIL.
+/// If "split_disallowed" is set or "wp"s buffer is closing, give an error and return FAIL.
 /// Otherwise return OK.
-static int check_split_disallowed(void)
+static int check_split_disallowed(const win_T *wp)
+  FUNC_ATTR_NONNULL_ALL
+{
+  Error err = ERROR_INIT;
+  const bool ok = check_split_disallowed_err(wp, &err);
+  if (ERROR_SET(&err)) {
+    emsg(_(err.msg));
+    api_clear_error(&err);
+  }
+  return ok ? OK : FAIL;
+}
+
+/// Like `check_split_disallowed`, but set `err` to the (untranslated) error message on failure and
+/// return false. Otherwise return true.
+/// @see check_split_disallowed
+bool check_split_disallowed_err(const win_T *wp, Error *err)
+  FUNC_ATTR_NONNULL_ALL
 {
   if (split_disallowed > 0) {
-    emsg(_("E242: Can't split a window while closing another"));
-    return FAIL;
+    api_set_error(err, kErrorTypeException, "E242: Can't split a window while closing another");
+    return false;
   }
-  if (curwin->w_buffer->b_locked_split) {
-    emsg(_(e_cannot_split_window_when_closing_buffer));
-    return FAIL;
+  if (wp->w_buffer->b_locked_split) {
+    api_set_error(err, kErrorTypeException, "%s", e_cannot_split_window_when_closing_buffer);
+    return false;
   }
-  return OK;
+  return true;
 }
 
 // split the current window, implements CTRL-W s and :split
@@ -936,7 +952,7 @@ static int check_split_disallowed(void)
 // return FAIL for failure, OK otherwise
 int win_split(int size, int flags)
 {
-  if (check_split_disallowed() == FAIL) {
+  if (check_split_disallowed(curwin) == FAIL) {
     return FAIL;
   }
 
@@ -1871,7 +1887,7 @@ static void win_totop(int size, int flags)
   if (is_aucmd_win(curwin)) {
     return;
   }
-  if (check_split_disallowed() == FAIL) {
+  if (check_split_disallowed(curwin) == FAIL) {
     return;
   }