vim-patch:9.0.0595: extra newline in messages after a verbose shell message (#20359)

Problem:    Extra newline in messages after a verbose shell message.
Solution:   Output the newline with msg_putchar_attr(). (closes vim/vim#11233)
            Make it possible to filter a screendump before comparing it.
https://github.com/vim/vim/commit/1190139ed01c27539615beea9559a88b2551daf3

Cherry-pick Test_message_more_scrolledback() from patch 9.0.0592 because
Nvim already behaves as intended.

3 files changed, 197 insertions, 17 deletions

diff --git a/src/nvim/message.c b/src/nvim/message.c
index e0b0dfb0bc..fd20d9dd81 100644
--- a/src/nvim/message.c
+++ b/src/nvim/message.c
@@ -2315,7 +2315,7 @@ static void msg_puts_display(const char *str, int maxlen, int attr, int recurse)
   if (t_col > 0) {
     t_puts(&t_col, t_s, s, attr);
   }
-  if (p_more && !recurse) {
+  if (p_more && !recurse && !(s == sb_str + 1 && *sb_str == '\n')) {
     store_sb_text((char **)&sb_str, (char *)s, attr, &sb_col, false);
   }
 
diff --git a/src/nvim/testdir/test_messages.vim b/src/nvim/testdir/test_messages.vim
index 8e66a2c529..42a1fdcfe2 100644
--- a/src/nvim/testdir/test_messages.vim
+++ b/src/nvim/testdir/test_messages.vim
@@ -316,6 +316,66 @@ func Test_message_more()
   call StopVimInTerminal(buf)
 endfunc
 
+" Test more-prompt scrollback
+func Test_message_more_scrollback()
+  CheckRunVimInTerminal
+
+  let lines =<< trim END
+      set t_ut=
+      hi Normal ctermfg=15 ctermbg=0
+      for i in range(100)
+          echo i
+      endfor
+  END
+  call writefile(lines, 'XmoreScrollback', 'D')
+  let buf = RunVimInTerminal('-S XmoreScrollback', {'rows': 10})
+  call VerifyScreenDump(buf, 'Test_more_scrollback_1', {})
+
+  call term_sendkeys(buf, 'f')
+  call TermWait(buf)
+  call term_sendkeys(buf, 'b')
+  call VerifyScreenDump(buf, 'Test_more_scrollback_2', {})
+
+  call term_sendkeys(buf, 'q')
+  call TermWait(buf)
+  call StopVimInTerminal(buf)
+endfunc
+
+" Test verbose message before echo command
+func Test_echo_verbose_system()
+  CheckRunVimInTerminal
+  CheckUnix
+
+  let buf = RunVimInTerminal('', {'rows': 10})
+  call term_sendkeys(buf, ":4 verbose echo system('seq 20')\<CR>")
+  " Note that the screendump is filtered to remove the name of the temp file
+  call VerifyScreenDump(buf, 'Test_verbose_system_1', {})
+
+  " display a page and go back, results in exactly the same view
+  call term_sendkeys(buf, ' ')
+  call TermWait(buf)
+  call term_sendkeys(buf, 'b')
+  call VerifyScreenDump(buf, 'Test_verbose_system_1', {})
+
+  " do the same with 'cmdheight' set to 2
+  call term_sendkeys(buf, 'q')
+  call TermWait(buf)
+  call term_sendkeys(buf, ":set ch=2\<CR>")
+  call TermWait(buf)
+  call term_sendkeys(buf, ":4 verbose echo system('seq 20')\<CR>")
+  call VerifyScreenDump(buf, 'Test_verbose_system_2', {})
+
+  call term_sendkeys(buf, ' ')
+  call TermWait(buf)
+  call term_sendkeys(buf, 'b')
+  call VerifyScreenDump(buf, 'Test_verbose_system_2', {})
+
+  call term_sendkeys(buf, 'q')
+  call TermWait(buf)
+  call StopVimInTerminal(buf)
+endfunc
+
+
 func Test_ask_yesno()
   CheckRunVimInTerminal
   let buf = RunVimInTerminal('', {'rows': 6})
diff --git a/test/functional/legacy/messages_spec.lua b/test/functional/legacy/messages_spec.lua
index 5b671bbfdf..71a53c8381 100644
--- a/test/functional/legacy/messages_spec.lua
+++ b/test/functional/legacy/messages_spec.lua
@@ -4,6 +4,8 @@ local clear = helpers.clear
 local command = helpers.command
 local exec = helpers.exec
 local feed = helpers.feed
+local meths = helpers.meths
+local nvim_dir = helpers.nvim_dir
 
 before_each(clear)
 
@@ -49,19 +51,18 @@ describe('messages', function()
 
   describe('more prompt', function()
     before_each(function()
+      command('set more')
+    end)
+
+    -- oldtest: Test_message_more()
+    it('works', function()
       screen = Screen.new(75, 6)
       screen:set_default_attr_ids({
-        [0] = {bold = true, foreground = Screen.colors.Blue},  -- NonText
         [1] = {bold = true, foreground = Screen.colors.SeaGreen},  -- MoreMsg
         [2] = {foreground = Screen.colors.Brown},  -- LineNr
-        [3] = {foreground = Screen.colors.Blue},  -- SpecialKey
       })
       screen:attach()
-      command('set more')
-    end)
 
-    -- oldtest: Test_message_more()
-    it('works', function()
       command('call setline(1, range(1, 100))')
 
       feed(':%pfoo<C-H><C-H><C-H>#')
@@ -350,20 +351,139 @@ describe('messages', function()
       ]])
     end)
 
+    -- oldtest: Test_echo_verbose_system()
+    it('verbose message before echo command', function()
+      screen = Screen.new(60, 10)
+      screen:set_default_attr_ids({
+        [0] = {bold = true, foreground = Screen.colors.Blue},  -- NonText
+        [1] = {bold = true, foreground = Screen.colors.SeaGreen},  -- MoreMsg
+      })
+      screen:attach()
+
+      command('cd '..nvim_dir)
+      meths.set_option('shell', './shell-test')
+      meths.set_option('shellcmdflag', 'REP 20')
+      meths.set_option('shellxquote', '')  -- win: avoid extra quotes
+
+      -- display a page and go back, results in exactly the same view
+      feed([[:4 verbose echo system('foo')<CR>]])
+      screen:expect([[
+        Executing command: "'./shell-test' 'REP' '20' 'foo'"        |
+                                                                    |
+        0: foo                                                      |
+        1: foo                                                      |
+        2: foo                                                      |
+        3: foo                                                      |
+        4: foo                                                      |
+        5: foo                                                      |
+        6: foo                                                      |
+        {1:-- More --}^                                                  |
+      ]])
+      feed('<Space>')
+      screen:expect([[
+        7: foo                                                      |
+        8: foo                                                      |
+        9: foo                                                      |
+        10: foo                                                     |
+        11: foo                                                     |
+        12: foo                                                     |
+        13: foo                                                     |
+        14: foo                                                     |
+        15: foo                                                     |
+        {1:-- More --}^                                                  |
+      ]])
+      feed('b')
+      screen:expect([[
+        Executing command: "'./shell-test' 'REP' '20' 'foo'"        |
+                                                                    |
+        0: foo                                                      |
+        1: foo                                                      |
+        2: foo                                                      |
+        3: foo                                                      |
+        4: foo                                                      |
+        5: foo                                                      |
+        6: foo                                                      |
+        {1:-- More --}^                                                  |
+      ]])
+
+      -- do the same with 'cmdheight' set to 2
+      feed('q')
+      command('set ch=2')
+      command('mode')  -- FIXME: bottom is invalid after scrolling
+      screen:expect([[
+        ^                                                            |
+        {0:~                                                           }|
+        {0:~                                                           }|
+        {0:~                                                           }|
+        {0:~                                                           }|
+        {0:~                                                           }|
+        {0:~                                                           }|
+        {0:~                                                           }|
+                                                                    |
+                                                                    |
+      ]])
+      feed([[:4 verbose echo system('foo')<CR>]])
+      screen:expect([[
+        Executing command: "'./shell-test' 'REP' '20' 'foo'"        |
+                                                                    |
+        0: foo                                                      |
+        1: foo                                                      |
+        2: foo                                                      |
+        3: foo                                                      |
+        4: foo                                                      |
+        5: foo                                                      |
+        6: foo                                                      |
+        {1:-- More --}^                                                  |
+      ]])
+      feed('<Space>')
+      screen:expect([[
+        7: foo                                                      |
+        8: foo                                                      |
+        9: foo                                                      |
+        10: foo                                                     |
+        11: foo                                                     |
+        12: foo                                                     |
+        13: foo                                                     |
+        14: foo                                                     |
+        15: foo                                                     |
+        {1:-- More --}^                                                  |
+      ]])
+      feed('b')
+      screen:expect([[
+        Executing command: "'./shell-test' 'REP' '20' 'foo'"        |
+                                                                    |
+        0: foo                                                      |
+        1: foo                                                      |
+        2: foo                                                      |
+        3: foo                                                      |
+        4: foo                                                      |
+        5: foo                                                      |
+        6: foo                                                      |
+        {1:-- More --}^                                                  |
+      ]])
+    end)
+
     -- oldtest: Test_quit_long_message()
     it('with control characters can be quit vim-patch:8.2.1844', function()
-      screen:try_resize(40, 10)
+      screen = Screen.new(40, 10)
+      screen:set_default_attr_ids({
+        [0] = {bold = true, foreground = Screen.colors.Blue},  -- NonText
+        [1] = {bold = true, foreground = Screen.colors.SeaGreen},  -- MoreMsg
+        [2] = {foreground = Screen.colors.Blue},  -- SpecialKey
+      })
+      screen:attach()
+
       feed([[:echom range(9999)->join("\x01")<CR>]])
       screen:expect([[
-        0{3:^A}1{3:^A}2{3:^A}3{3:^A}4{3:^A}5{3:^A}6{3:^A}7{3:^A}8{3:^A}9{3:^A}10{3:^A}11{3:^A}12|
-        {3:^A}13{3:^A}14{3:^A}15{3:^A}16{3:^A}17{3:^A}18{3:^A}19{3:^A}20{3:^A}21{3:^A}22|
-        {3:^A}23{3:^A}24{3:^A}25{3:^A}26{3:^A}27{3:^A}28{3:^A}29{3:^A}30{3:^A}31{3:^A}32|
-        {3:^A}33{3:^A}34{3:^A}35{3:^A}36{3:^A}37{3:^A}38{3:^A}39{3:^A}40{3:^A}41{3:^A}42|
-        {3:^A}43{3:^A}44{3:^A}45{3:^A}46{3:^A}47{3:^A}48{3:^A}49{3:^A}50{3:^A}51{3:^A}52|
-        {3:^A}53{3:^A}54{3:^A}55{3:^A}56{3:^A}57{3:^A}58{3:^A}59{3:^A}60{3:^A}61{3:^A}62|
-        {3:^A}63{3:^A}64{3:^A}65{3:^A}66{3:^A}67{3:^A}68{3:^A}69{3:^A}70{3:^A}71{3:^A}72|
-        {3:^A}73{3:^A}74{3:^A}75{3:^A}76{3:^A}77{3:^A}78{3:^A}79{3:^A}80{3:^A}81{3:^A}82|
-        {3:^A}83{3:^A}84{3:^A}85{3:^A}86{3:^A}87{3:^A}88{3:^A}89{3:^A}90{3:^A}91{3:^A}92|
+        0{2:^A}1{2:^A}2{2:^A}3{2:^A}4{2:^A}5{2:^A}6{2:^A}7{2:^A}8{2:^A}9{2:^A}10{2:^A}11{2:^A}12|
+        {2:^A}13{2:^A}14{2:^A}15{2:^A}16{2:^A}17{2:^A}18{2:^A}19{2:^A}20{2:^A}21{2:^A}22|
+        {2:^A}23{2:^A}24{2:^A}25{2:^A}26{2:^A}27{2:^A}28{2:^A}29{2:^A}30{2:^A}31{2:^A}32|
+        {2:^A}33{2:^A}34{2:^A}35{2:^A}36{2:^A}37{2:^A}38{2:^A}39{2:^A}40{2:^A}41{2:^A}42|
+        {2:^A}43{2:^A}44{2:^A}45{2:^A}46{2:^A}47{2:^A}48{2:^A}49{2:^A}50{2:^A}51{2:^A}52|
+        {2:^A}53{2:^A}54{2:^A}55{2:^A}56{2:^A}57{2:^A}58{2:^A}59{2:^A}60{2:^A}61{2:^A}62|
+        {2:^A}63{2:^A}64{2:^A}65{2:^A}66{2:^A}67{2:^A}68{2:^A}69{2:^A}70{2:^A}71{2:^A}72|
+        {2:^A}73{2:^A}74{2:^A}75{2:^A}76{2:^A}77{2:^A}78{2:^A}79{2:^A}80{2:^A}81{2:^A}82|
+        {2:^A}83{2:^A}84{2:^A}85{2:^A}86{2:^A}87{2:^A}88{2:^A}89{2:^A}90{2:^A}91{2:^A}92|
         {1:-- More --}^                              |
       ]])
       feed('q')