fix(api): make open_win/win_set_config check if splitting allowed

Problem: splitting is disallowed in some cases to prevent the window layout
changes while a window is closing, but it's not checked for.

Solution: check for this, and set the API error message directly.

(Also sneak in a change to tui.c that got lost from #27352; it's a char* buf,
and the memset is assuming one byte each anyway)

4 files changed, 82 insertions, 11 deletions

diff --git a/src/nvim/api/win_config.c b/src/nvim/api/win_config.c
index 3959e74af9..557c2f37f9 100644
--- a/src/nvim/api/win_config.c
+++ b/src/nvim/api/win_config.c
@@ -246,6 +246,10 @@ Window nvim_open_win(Buffer buffer, Boolean enter, Dict(win_config) *config, Err
       }
     }
 
+    if (!check_split_disallowed_err(parent ? parent : curwin, err)) {
+      return 0;  // error already set
+    }
+
     if (HAS_KEY_X(config, vertical) && !HAS_KEY_X(config, split)) {
       if (config->vertical) {
         fconfig.split = p_spr ? kWinSplitRight : kWinSplitLeft;
@@ -440,6 +444,10 @@ void nvim_win_set_config(Window window, Dict(win_config) *config, Error *err)
       return;
     }
 
+    if (!check_split_disallowed_err(win, err)) {
+      return;  // error already set
+    }
+
     if (was_split) {
       win_T *new_curwin = NULL;
 
diff --git a/src/nvim/tui/tui.c b/src/nvim/tui/tui.c
index 7fae34d33f..c332c17e43 100644
--- a/src/nvim/tui/tui.c
+++ b/src/nvim/tui/tui.c
@@ -1643,7 +1643,7 @@ static void invalidate(TUIData *tui, int top, int bot, int left, int right)
 static void ensure_space_buf_size(TUIData *tui, size_t len)
 {
   if (len > tui->space_buf_len) {
-    tui->space_buf = xrealloc(tui->space_buf, len * sizeof *tui->space_buf);
+    tui->space_buf = xrealloc(tui->space_buf, len);
     memset(tui->space_buf + tui->space_buf_len, ' ', len - tui->space_buf_len);
     tui->space_buf_len = len;
   }
diff --git a/src/nvim/window.c b/src/nvim/window.c
index d5a6e347e7..81f8304b22 100644
--- a/src/nvim/window.c
+++ b/src/nvim/window.c
@@ -905,19 +905,35 @@ void ui_ext_win_viewport(win_T *wp)
   }
 }
 
-/// If "split_disallowed" is set give an error and return FAIL.
+/// If "split_disallowed" is set or "wp"s buffer is closing, give an error and return FAIL.
 /// Otherwise return OK.
-static int check_split_disallowed(void)
+static int check_split_disallowed(const win_T *wp)
+  FUNC_ATTR_NONNULL_ALL
+{
+  Error err = ERROR_INIT;
+  const bool ok = check_split_disallowed_err(wp, &err);
+  if (ERROR_SET(&err)) {
+    emsg(_(err.msg));
+    api_clear_error(&err);
+  }
+  return ok ? OK : FAIL;
+}
+
+/// Like `check_split_disallowed`, but set `err` to the (untranslated) error message on failure and
+/// return false. Otherwise return true.
+/// @see check_split_disallowed
+bool check_split_disallowed_err(const win_T *wp, Error *err)
+  FUNC_ATTR_NONNULL_ALL
 {
   if (split_disallowed > 0) {
-    emsg(_("E242: Can't split a window while closing another"));
-    return FAIL;
+    api_set_error(err, kErrorTypeException, "E242: Can't split a window while closing another");
+    return false;
   }
-  if (curwin->w_buffer->b_locked_split) {
-    emsg(_(e_cannot_split_window_when_closing_buffer));
-    return FAIL;
+  if (wp->w_buffer->b_locked_split) {
+    api_set_error(err, kErrorTypeException, "%s", e_cannot_split_window_when_closing_buffer);
+    return false;
   }
-  return OK;
+  return true;
 }
 
 // split the current window, implements CTRL-W s and :split
@@ -936,7 +952,7 @@ static int check_split_disallowed(void)
 // return FAIL for failure, OK otherwise
 int win_split(int size, int flags)
 {
-  if (check_split_disallowed() == FAIL) {
+  if (check_split_disallowed(curwin) == FAIL) {
     return FAIL;
   }
 
@@ -1871,7 +1887,7 @@ static void win_totop(int size, int flags)
   if (is_aucmd_win(curwin)) {
     return;
   }
-  if (check_split_disallowed() == FAIL) {
+  if (check_split_disallowed(curwin) == FAIL) {
     return;
   }
 
diff --git a/test/functional/api/window_spec.lua b/test/functional/api/window_spec.lua
index 2f6a02b5d5..30235d6b84 100644
--- a/test/functional/api/window_spec.lua
+++ b/test/functional/api/window_spec.lua
@@ -1589,6 +1589,37 @@ describe('API/win', function()
       api.nvim_open_win(new_buf, true, { split = 'left' })
       eq(cur_buf, api.nvim_get_current_buf())
     end)
+
+    it('checks if splitting disallowed', function()
+      command('split | autocmd WinEnter * ++once call nvim_open_win(0, 0, #{split: "right"})')
+      matches("E242: Can't split a window while closing another$", pcall_err(command, 'quit'))
+
+      command('only | autocmd BufHidden * ++once call nvim_open_win(0, 0, #{split: "left"})')
+      matches(
+        'E1159: Cannot split a window when closing the buffer$',
+        pcall_err(command, 'new | quit')
+      )
+
+      local w = api.nvim_get_current_win()
+      command(
+        'only | new | autocmd BufHidden * ++once call nvim_open_win(0, 0, #{split: "left", win: '
+          .. w
+          .. '})'
+      )
+      matches(
+        'E1159: Cannot split a window when closing the buffer$',
+        pcall_err(api.nvim_win_close, w, true)
+      )
+
+      -- OK when using window to different buffer than `win`s.
+      w = api.nvim_get_current_win()
+      command(
+        'only | autocmd BufHidden * ++once call nvim_open_win(0, 0, #{split: "left", win: '
+          .. w
+          .. '})'
+      )
+      command('new | quit')
+    end)
   end)
 
   describe('set_config', function()
@@ -1941,6 +1972,22 @@ describe('API/win', function()
       -- Shouldn't see any of those events, as we remain in the same window.
       eq({}, eval('result'))
     end)
+
+    it('checks if splitting disallowed', function()
+      command('split | autocmd WinEnter * ++once call nvim_win_set_config(0, #{split: "right"})')
+      matches("E242: Can't split a window while closing another$", pcall_err(command, 'quit'))
+
+      command('autocmd BufHidden * ++once call nvim_win_set_config(0, #{split: "left"})')
+      matches(
+        'E1159: Cannot split a window when closing the buffer$',
+        pcall_err(command, 'new | quit')
+      )
+
+      -- OK when using window to different buffer.
+      local w = api.nvim_get_current_win()
+      command('autocmd BufHidden * ++once call nvim_win_set_config(' .. w .. ', #{split: "left"})')
+      command('new | quit')
+    end)
   end)
 
   describe('get_config', function()