fix(api): don't use 'winborder' when reconfiguring float (#32984)

Problem: Reconfiguring a float window applies the global 'winborder'.
Solution:
- Ignore 'winborder' when reconfiguring a float window.
- Still apply 'winborder' when converting a split to a float window.

3 files changed, 39 insertions, 2 deletions

diff --git a/src/nvim/api/win_config.c b/src/nvim/api/win_config.c
index 4b5b02e507..ee6571fd95 100644
--- a/src/nvim/api/win_config.c
+++ b/src/nvim/api/win_config.c
@@ -1280,7 +1280,7 @@ static bool parse_win_config(win_T *wp, Dict(win_config) *config, WinConfig *fco
       goto fail;
     }
     border_style = config->border;
-  } else if (*p_winborder != NUL) {
+  } else if (*p_winborder != NUL && (wp == NULL || !wp->w_floating)) {
     border_style = CSTR_AS_OBJ(p_winborder);
   }
   if (border_style.type != kObjectTypeNil) {
diff --git a/test/functional/api/window_spec.lua b/test/functional/api/window_spec.lua
index 119c185b72..34b3b2dbe0 100644
--- a/test/functional/api/window_spec.lua
+++ b/test/functional/api/window_spec.lua
@@ -1907,6 +1907,38 @@ describe('API/win', function()
   end)
 
   describe('set_config', function()
+    it("uses 'winborder' when converting a split to a floating window", function()
+      api.nvim_set_option_value('winborder', 'single', {})
+      command('split')
+      local winid = api.nvim_get_current_win()
+      -- Convert split to float without specifying border
+      api.nvim_win_set_config(winid, {
+        relative = 'editor',
+        row = 2,
+        col = 2,
+        width = 10,
+        height = 5,
+      })
+      local config = api.nvim_win_get_config(winid)
+      eq('┌', config.border[1])
+    end)
+
+    it('erases border of a floating window when converting to split window', function()
+      api.nvim_set_option_value('winborder', 'single', {})
+      local winid = api.nvim_open_win(api.nvim_create_buf(false, false), false, {
+        relative = 'editor',
+        row = 2,
+        col = 2,
+        width = 10,
+        height = 5,
+      })
+      local config = api.nvim_win_get_config(winid)
+      eq('┌', config.border[1])
+      api.nvim_win_set_config(winid, { split = 'right', win = 0 })
+      config = api.nvim_win_get_config(winid)
+      eq(nil, config.border)
+    end)
+
     it('moves a split into a float', function()
       local win = api.nvim_open_win(0, true, {
         vertical = false,
diff --git a/test/functional/ui/float_spec.lua b/test/functional/ui/float_spec.lua
index fa4937cec3..d9f29b9496 100644
--- a/test/functional/ui/float_spec.lua
+++ b/test/functional/ui/float_spec.lua
@@ -10088,7 +10088,7 @@ describe('float window', function()
       -- respect config.border
       command('set winborder=rounded')
       config.border = 'single'
-      api.nvim_open_win(buf, false, config)
+      local winid = api.nvim_open_win(buf, false, config)
       if multigrid then
         screen:expect({
           grid = [[
@@ -10153,6 +10153,11 @@ describe('float window', function()
         ]])
       end
 
+      -- don't use winborder when reconfig a floating window
+      config.border = nil
+      api.nvim_win_set_config(winid, config)
+      screen:expect_unchanged()
+      command('fclose!')
       -- it is currently not supported.
       eq('Vim(set):E474: Invalid argument: winborder=custom', pcall_err(command, 'set winborder=custom'))
     end)