os/shell: Throttle :! output, pulse "..." message.

Periodically skip :! spam. This is a "cheat" that works for all UIs and greatly
improves responsiveness when :! spams MB or GB of output:
    :!yes
    :!while true; do date; done
    :!git grep ''
    :grep -r '' *

After ~10KB of data is seen from a single :! invocation, output will be skipped
for ~1s and three dots "..." will pulse in the bottom-left. Thereafter the
behavior alternates at every:
    * 10KB received
    * ~1s throttled

This also avoids out-of-memory which could happen with large :! outputs.

Note: This commit does not change the behavior of execute(':!foo').
      execute(':!foo') returns the string ':!foo^M', it captures *only* Vim
      messages, *not* shell command output. Vim behaves the same way.
      Use system('foo') for capturing shell command output.

Closes #1234

Helped-by: oni-link <knil.ino@gmail.com>

4 files changed, 54 insertions, 19 deletions

diff --git a/test/functional/eval/execute_spec.lua b/test/functional/eval/execute_spec.lua
index b5b481435a..fc13c0a72b 100644
--- a/test/functional/eval/execute_spec.lua
+++ b/test/functional/eval/execute_spec.lua
@@ -12,16 +12,16 @@ local feed = helpers.feed
 describe('execute()', function()
   before_each(clear)
 
-  it('returns the same result with :redir', function()
+  it('captures the same result as :redir', function()
     eq(redir_exec('messages'), funcs.execute('messages'))
   end)
 
-  it('returns the output of the commands if the argument is List', function()
+  it('captures the concatenated outputs of a List of commands', function()
     eq("foobar", funcs.execute({'echon "foo"', 'echon "bar"'}))
     eq("\nfoo\nbar", funcs.execute({'echo "foo"', 'echo "bar"'}))
   end)
 
-  it('supports the nested redirection', function()
+  it('supports nested redirection', function()
     source([[
     function! g:Foo()
       let a = ''
@@ -43,17 +43,17 @@ describe('execute()', function()
     eq('42', funcs.execute([[echon execute("echon execute('echon 42')")]]))
   end)
 
-  it('returns the transformed string', function()
+  it('captures a transformed string', function()
     eq('^A', funcs.execute('echon "\\<C-a>"'))
   end)
 
-  it('returns the empty string if the argument list is empty', function()
+  it('returns empty string if the argument list is empty', function()
     eq('', funcs.execute({}))
     eq(0, exc_exec('let g:ret = execute(v:_null_list)'))
     eq('', eval('g:ret'))
   end)
 
-  it('returns the errors', function()
+  it('captures errors', function()
     local ret
     ret = exc_exec('call execute(0.0)')
     eq('Vim(call):E806: using Float as a String', ret)
@@ -69,6 +69,11 @@ describe('execute()', function()
     eq('Vim(call):E729: using Funcref as a String', ret)
   end)
 
+  -- This matches Vim behavior.
+  it('does not capture shell-command output', function()
+    eq('\n:!echo "foo"\13\n', funcs.execute('!echo "foo"'))
+  end)
+
   it('silences command run inside', function()
     local screen = Screen.new(40, 5)
     screen:attach()
diff --git a/test/functional/terminal/helpers.lua b/test/functional/terminal/helpers.lua
index aacf109f2f..ae5e6d4b1f 100644
--- a/test/functional/terminal/helpers.lua
+++ b/test/functional/terminal/helpers.lua
@@ -51,6 +51,7 @@ local function screen_setup(extra_height, command)
     [7] = {foreground = 130},
     [8] = {foreground = 15, background = 1}, -- error message
     [9] = {foreground = 4},
+    [10] = {foreground = 2},  -- "Press ENTER" in embedded :terminal session.
   })
 
   screen:attach({rgb=false})
diff --git a/test/functional/ui/output_spec.lua b/test/functional/ui/output_spec.lua
index c58bbe9147..4aae2edfaa 100644
--- a/test/functional/ui/output_spec.lua
+++ b/test/functional/ui/output_spec.lua
@@ -1,5 +1,6 @@
 local session = require('test.functional.helpers')(after_each)
 local child_session = require('test.functional.terminal.helpers')
+local Screen = require('test.functional.ui.screen')
 
 if session.pending_win32(pending) then return end
 
@@ -39,4 +40,11 @@ describe("shell command :!", function()
       {3:-- TERMINAL --}                                    |
     ]])
   end)
+
+  it("throttles shell-command output greater than ~20KB", function()
+    child_session.feed_data(
+      ":!for i in $(seq 2 3000); do echo XXXXXXXXXX; done\n")
+    -- If a line with only a dot "." appears, then throttling was triggered.
+    screen:expect("\n.", nil, nil, nil, true)
+  end)
 end)
diff --git a/test/functional/ui/screen.lua b/test/functional/ui/screen.lua
index ebe8af35eb..2581b36711 100644
--- a/test/functional/ui/screen.lua
+++ b/test/functional/ui/screen.lua
@@ -126,7 +126,7 @@ end
 do
   local spawn, nvim_prog = helpers.spawn, helpers.nvim_prog
   local session = spawn({nvim_prog, '-u', 'NONE', '-i', 'NONE', '-N', '--embed'})
-  local status, rv = session:request('vim_get_color_map')
+  local status, rv = session:request('nvim_get_color_map')
   if not status then
     print('failed to get color map')
     os.exit(1)
@@ -207,7 +207,15 @@ function Screen:try_resize(columns, rows)
   uimeths.try_resize(columns, rows)
 end
 
-function Screen:expect(expected, attr_ids, attr_ignore, condition)
+-- Asserts that `expected` eventually matches the screen state.
+--
+-- expected:    Expected screen state (string).
+-- attr_ids:    Text attribute definitions.
+-- attr_ignore: Ignored text attributes.
+-- condition:   Function asserting some arbitrary condition.
+-- any:         true: Succeed if `expected` matches ANY screen line(s).
+--              false (default): `expected` must match screen exactly.
+function Screen:expect(expected, attr_ids, attr_ignore, condition, any)
   -- remove the last line and dedent
   expected = dedent(expected:gsub('\n[ ]+$', ''))
   local expected_rows = {}
@@ -229,21 +237,34 @@ function Screen:expect(expected, attr_ids, attr_ignore, condition)
     for i = 1, self._height do
       actual_rows[i] = self:_row_repr(self._rows[i], ids, ignore)
     end
-    for i = 1, self._height do
-      if expected_rows[i] ~= actual_rows[i] then
-        local msg_expected_rows = {}
-        for j = 1, #expected_rows do
-          msg_expected_rows[j] = expected_rows[j]
-        end
-        msg_expected_rows[i] = '*' .. msg_expected_rows[i]
-        actual_rows[i] = '*' .. actual_rows[i]
+
+    if any then
+      -- Search for `expected` anywhere in the screen lines.
+      local actual_screen_str = table.concat(actual_rows, '\n')
+      if nil == string.find(actual_screen_str, expected) then
         return (
-          'Row ' .. tostring(i) .. ' didn\'t match.\n'
-          .. 'Expected:\n|' .. table.concat(msg_expected_rows, '|\n|') .. '|\n'
-          .. 'Actual:\n|' .. table.concat(actual_rows, '|\n|') .. '|\n\n' .. [[
+          'Failed to match any screen lines.\n'
+          .. 'Expected (anywhere): "' .. expected .. '"\n'
+          .. 'Actual:\n  |' .. table.concat(actual_rows, '|\n  |') .. '|\n\n')
+      end
+    else
+      -- `expected` must match the screen lines exactly.
+      for i = 1, self._height do
+        if expected_rows[i] ~= actual_rows[i] then
+          local msg_expected_rows = {}
+          for j = 1, #expected_rows do
+            msg_expected_rows[j] = expected_rows[j]
+          end
+          msg_expected_rows[i] = '*' .. msg_expected_rows[i]
+          actual_rows[i] = '*' .. actual_rows[i]
+          return (
+            'Row ' .. tostring(i) .. ' did not match.\n'
+            ..'Expected:\n  |'..table.concat(msg_expected_rows, '|\n  |')..'|\n'
+            ..'Actual:\n  |'..table.concat(actual_rows, '|\n  |')..'|\n\n'..[[
 To print the expect() call that would assert the current screen state, use
 screen:snaphot_util(). In case of non-deterministic failures, use
 screen:redraw_debug() to show all intermediate screen states.  ]])
+        end
       end
     end
   end)