terminal: Fix potential invalid local 'scrollback' (#9605)

TermOpen autocmd may set local 'scrollback' to -1, this needs to be
adjusted as in on_scrollback_option_changed().

fixes #9588 (OOM, out of memory)

1 files changed, 34 insertions, 7 deletions

diff --git a/test/functional/terminal/scrollback_spec.lua b/test/functional/terminal/scrollback_spec.lua
index 7c74e82d17..75bb89a1ab 100644
--- a/test/functional/terminal/scrollback_spec.lua
+++ b/test/functional/terminal/scrollback_spec.lua
@@ -10,6 +10,7 @@ local wait = helpers.wait
 local retry = helpers.retry
 local curbufmeths = helpers.curbufmeths
 local nvim = helpers.nvim
+local expect_err = helpers.expect_err
 local feed_data = thelpers.feed_data
 
 describe(':terminal scrollback', function()
@@ -467,13 +468,8 @@ describe("'scrollback' option", function()
   end)
 
   it('error if set to invalid value', function()
-    local status, rv = pcall(command, 'set scrollback=-2')
-    eq(false, status)  -- assert failure
-    eq('E474:', string.match(rv, "E%d*:"))
-
-    status, rv = pcall(command, 'set scrollback=100001')
-    eq(false, status)  -- assert failure
-    eq('E474:', string.match(rv, "E%d*:"))
+    expect_err('E474:', command, 'set scrollback=-2')
+    expect_err('E474:', command, 'set scrollback=100001')
   end)
 
   it('defaults to -1 on normal buffers', function()
@@ -481,6 +477,37 @@ describe("'scrollback' option", function()
     eq(-1, curbufmeths.get_option('scrollback'))
   end)
 
+  it(':setlocal in a :terminal buffer', function()
+    set_fake_shell()
+
+    -- _Global_ scrollback=-1 defaults :terminal to 10_000.
+    command('setglobal scrollback=-1')
+    command('terminal')
+    eq(10000, curbufmeths.get_option('scrollback'))
+
+    -- _Local_ scrollback=-1 in :terminal forces the _maximum_.
+    command('setlocal scrollback=-1')
+    retry(nil, nil, function()  -- Fixup happens on refresh, not immediately.
+      eq(100000, curbufmeths.get_option('scrollback'))
+    end)
+
+    -- _Local_ scrollback=-1 during TermOpen forces the maximum. #9605
+    command('setglobal scrollback=-1')
+    command('autocmd TermOpen * setlocal scrollback=-1')
+    command('terminal')
+    eq(100000, curbufmeths.get_option('scrollback'))
+  end)
+
+  it(':setlocal in a normal buffer', function()
+    command('new')
+    -- :setlocal to -1.
+    command('setlocal scrollback=-1')
+    eq(-1, curbufmeths.get_option('scrollback'))
+    -- :setlocal to anything except -1. Currently, this just has no effect.
+    command('setlocal scrollback=42')
+    eq(42, curbufmeths.get_option('scrollback'))
+  end)
+
   it(':set updates local value and global default', function()
     set_fake_shell()
     command('set scrollback=42')                  -- set global value