Merge pull request #7979 from bfredl/shellbell

Shell: support bell and buffer incomplete UTF-8 sequences

5 files changed, 165 insertions, 94 deletions

diff --git a/src/nvim/os/shell.c b/src/nvim/os/shell.c
index 5b3cb64a4d..f650a51fe7 100644
--- a/src/nvim/os/shell.c
+++ b/src/nvim/os/shell.c
@@ -422,7 +422,7 @@ static void out_data_ring(char *output, size_t size)
   }
 
   if (output == NULL && size == SIZE_MAX) {   // Print mode
-    out_data_append_to_screen(last_skipped, last_skipped_len, true);
+    out_data_append_to_screen(last_skipped, &last_skipped_len, true);
     return;
   }
 
@@ -450,30 +450,40 @@ static void out_data_ring(char *output, size_t size)
 /// @param output       Data to append to screen lines.
 /// @param remaining    Size of data.
 /// @param new_line     If true, next data output will be on a new line.
-static void out_data_append_to_screen(char *output, size_t remaining,
-                                      bool new_line)
+static void out_data_append_to_screen(char *output, size_t *count,
+                                      bool eof)
 {
-  char *p = output, *end = output + remaining;
+  char *p = output, *end = output + *count;
   while (p < end) {
-    if (*p == '\n' || *p == '\r' || *p == TAB) {
+    if (*p == '\n' || *p == '\r' || *p == TAB || *p == BELL) {
       msg_putchar_attr((uint8_t)(*p), 0);
       p++;
     } else {
+      // Note: this is not 100% precise:
+      // 1. we don't check if received continuation bytes are already invalid
+      //    and we thus do some buffering that could be avoided
+      // 2. we don't compose chars over buffer boundaries, even if we see an
+      //    incomplete UTF-8 sequence that could be composing with the last
+      //    complete sequence.
+      // This will be corrected when we switch to vterm based implementation
       int i = *p ? mb_ptr2len_len((char_u *)p, (int)(end-p)) : 1;
+      if (!eof && i == 1 && utf8len_tab_zero[*(uint8_t *)p] > (end-p)) {
+        *count = (size_t)(p - output);
+        goto end;
+      }
 
       (void)msg_outtrans_len_attr((char_u *)p, i, 0);
       p += i;
     }
   }
 
+end:
   ui_flush();
 }
 
 static void out_data_cb(Stream *stream, RBuffer *buf, size_t count, void *data,
     bool eof)
 {
-  // We always output the whole buffer, so the buffer can never
-  // wrap around.
   size_t cnt;
   char *ptr = rbuffer_read_ptr(buf, &cnt);
 
@@ -482,12 +492,16 @@ static void out_data_cb(Stream *stream, RBuffer *buf, size_t count, void *data,
     // Save the skipped output. If it is the final chunk, we display it later.
     out_data_ring(ptr, cnt);
   } else {
-    out_data_append_to_screen(ptr, cnt, eof);
+    out_data_append_to_screen(ptr, &cnt, eof);
   }
 
   if (cnt) {
     rbuffer_consumed(buf, cnt);
   }
+
+  // Move remaining data to start of buffer, so the buffer can never
+  // wrap around.
+  rbuffer_reset(buf);
 }
 
 /// Parses a command string into a sequence of words, taking quotes into
diff --git a/test/functional/eval/backtick_expansion_spec.lua b/test/functional/eval/backtick_expansion_spec.lua
index 81e8e295fa..b1b44cfa8b 100644
--- a/test/functional/eval/backtick_expansion_spec.lua
+++ b/test/functional/eval/backtick_expansion_spec.lua
@@ -21,11 +21,19 @@ describe("backtick expansion", function()
   end)
 
   it("with default 'shell'", function()
-    if helpers.pending_win32(pending) then return end  -- Need win32 shell fixes
-    command(":silent args `echo ***2`")
+    if helpers.iswin() then
+      command(":silent args `dir /b *2`")
+    else
+      command(":silent args `echo ***2`")
+    end
     eq({ "file2", }, eval("argv()"))
-    command(":silent args `echo */*4`")
-    eq({ "subdir/file4", }, eval("argv()"))
+    if helpers.iswin() then
+      command(":silent args `dir /s/b *4`")
+      eq({ "subdir\\file4", }, eval("map(argv(), 'fnamemodify(v:val, \":.\")')"))
+    else
+      command(":silent args `echo */*4`")
+      eq({ "subdir/file4", }, eval("argv()"))
+    end
   end)
 
   it("with shell=fish", function()
diff --git a/test/functional/ex_cmds/bang_filter_spec.lua b/test/functional/ex_cmds/bang_filter_spec.lua
deleted file mode 100644
index 636d732161..0000000000
--- a/test/functional/ex_cmds/bang_filter_spec.lua
+++ /dev/null
@@ -1,69 +0,0 @@
--- Specs for bang/filter commands
-
-local helpers = require('test.functional.helpers')(after_each)
-local feed, command, clear = helpers.feed, helpers.command, helpers.clear
-local mkdir, write_file, rmdir = helpers.mkdir, helpers.write_file, helpers.rmdir
-local feed_command = helpers.feed_command
-
-if helpers.pending_win32(pending) then return end
-
-local Screen = require('test.functional.ui.screen')
-
-
-describe(':! command', function()
-  local screen
-
-  before_each(function()
-    clear()
-    rmdir('bang_filter_spec')
-    mkdir('bang_filter_spec')
-    write_file('bang_filter_spec/f1', 'f1')
-    write_file('bang_filter_spec/f2', 'f2')
-    write_file('bang_filter_spec/f3', 'f3')
-    screen = Screen.new(53,10)
-    screen:set_default_attr_ids({
-      [1] = {bold = true, foreground = Screen.colors.Blue1},
-      [2] = {foreground = Screen.colors.Blue1},
-      [3] = {bold = true, foreground = Screen.colors.SeaGreen4},
-    })
-    screen:attach()
-  end)
-
-  after_each(function()
-    rmdir('bang_filter_spec')
-  end)
-
-  it("doesn't truncate Last line of shell output #3269", function()
-    command([[nnoremap <silent>\l :!ls bang_filter_spec<cr>]])
-    feed([[\l]])
-    screen:expect([[
-      {1:~                                                    }|
-      {1:~                                                    }|
-      {1:~                                                    }|
-      {1:~                                                    }|
-      :!ls bang_filter_spec                                |
-      f1                                                   |
-      f2                                                   |
-      f3                                                   |
-                                                           |
-      {3:Press ENTER or type command to continue}^              |
-    ]])
-  end)
-
-  it('handles binary and multibyte data', function()
-    feed_command('!cat test/functional/fixtures/shell_data.txt')
-    screen:expect([[
-      {1:~                                                    }|
-      {1:~                                                    }|
-      {1:~                                                    }|
-      :!cat test/functional/fixtures/shell_data.txt        |
-      {2:^@^A^B^C^D^E^F^G^H}                                   |
-      {2:^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_}                 |
-      ö 한글 {2:<a5><c3>}                                      |
-      t       {2:<ff>}                                         |
-                                                           |
-      {3:Press ENTER or type command to continue}^              |
-  ]])
-  end)
-
-end)
diff --git a/test/functional/fixtures/shell-test.c b/test/functional/fixtures/shell-test.c
index 8dbec2aaee..38695ce76b 100644
--- a/test/functional/fixtures/shell-test.c
+++ b/test/functional/fixtures/shell-test.c
@@ -4,6 +4,13 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdint.h>
+#include <unistd.h>
+
+static void wait(void)
+{
+  fflush(stdout);
+  usleep(10*1000);
+}
 
 static void help(void)
 {
@@ -61,6 +68,22 @@ int main(int argc, char **argv)
       for (uint8_t i = 0; i < number; i++) {
         printf("%d: %s\n", (int) i, argv[3]);
       }
+    } else if (strcmp(argv[1], "UTF-8") == 0) {
+      // test split-up UTF-8 sequence
+      printf("\xc3"); wait();
+      printf("\xa5\n"); wait();
+
+      // split up a 2+2 grapheme clusters all possible ways
+      printf("ref: \xc3\xa5\xcc\xb2\n"); wait();
+
+      printf("1: \xc3"); wait();
+      printf("\xa5\xcc\xb2\n"); wait();
+
+      printf("2: \xc3\xa5"); wait();
+      printf("\xcc\xb2\n"); wait();
+
+      printf("3: \xc3\xa5\xcc"); wait();
+      printf("\xb2\n"); wait();
     } else {
       fprintf(stderr, "Unknown first argument\n");
       return 3;
diff --git a/test/functional/ui/output_spec.lua b/test/functional/ui/output_spec.lua
index c42c0b26c6..4246020fab 100644
--- a/test/functional/ui/output_spec.lua
+++ b/test/functional/ui/output_spec.lua
@@ -1,19 +1,24 @@
 local Screen = require('test.functional.ui.screen')
-local session = require('test.functional.helpers')(after_each)
+local helpers = require('test.functional.helpers')(after_each)
 local child_session = require('test.functional.terminal.helpers')
-local eq = session.eq
-local eval = session.eval
-local feed = session.feed
-local iswin = session.iswin
+local mkdir, write_file, rmdir = helpers.mkdir, helpers.write_file, helpers.rmdir
+local eq = helpers.eq
+local eval = helpers.eval
+local feed = helpers.feed
+local feed_command = helpers.feed_command
+local iswin = helpers.iswin
+local clear = helpers.clear
+local command = helpers.command
+local nvim_dir = helpers.nvim_dir
 
 describe("shell command :!", function()
-  if session.pending_win32(pending) then return end
+  if helpers.pending_win32(pending) then return end
 
   local screen
   before_each(function()
-    session.clear()
-    screen = child_session.screen_setup(0, '["'..session.nvim_prog..
-      '", "-u", "NONE", "-i", "NONE", "--cmd", "'..session.nvim_set..'"]')
+    clear()
+    screen = child_session.screen_setup(0, '["'..helpers.nvim_prog..
+      '", "-u", "NONE", "-i", "NONE", "--cmd", "'..helpers.nvim_set..'"]')
     screen:expect([[
       {1: }                                                 |
       {4:~                                                 }|
@@ -46,7 +51,7 @@ describe("shell command :!", function()
   end)
 
   it("throttles shell-command output greater than ~10KB", function()
-    if os.getenv("TRAVIS") and session.os_name() == "osx" then
+    if os.getenv("TRAVIS") and helpers.os_name() == "osx" then
       pending("[Unreliable on Travis macOS.]", function() end)
       return
     end
@@ -74,7 +79,7 @@ end)
 
 describe("shell command :!", function()
   before_each(function()
-    session.clear()
+    clear()
   end)
 
   it("cat a binary file #4142", function()
@@ -104,13 +109,16 @@ describe("shell command :!", function()
     ]])
     feed([[<CR>]])
     -- Print BELL control code. #4338
+    screen.bell = false
     feed([[:!printf '\x07\x07\x07\x07text'<CR>]])
     screen:expect([[
       ~                                                 |
       :!printf '\x07\x07\x07\x07text'                   |
-      ^G^G^G^Gtext                                      |
+      text                                              |
       Press ENTER or type command to continue^           |
-    ]])
+    ]], nil, nil, function()
+      eq(true, screen.bell)
+    end)
     feed([[<CR>]])
     -- Print BS control code.
     feed([[:echo system('printf ''\x08\n''')<CR>]])
@@ -131,4 +139,91 @@ describe("shell command :!", function()
     ]])
     feed([[<CR>]])
   end)
+
+  describe('', function()
+    local screen
+    before_each(function()
+      rmdir('bang_filter_spec')
+      mkdir('bang_filter_spec')
+      write_file('bang_filter_spec/f1', 'f1')
+      write_file('bang_filter_spec/f2', 'f2')
+      write_file('bang_filter_spec/f3', 'f3')
+      screen = Screen.new(53,10)
+      screen:set_default_attr_ids({
+        [1] = {bold = true, foreground = Screen.colors.Blue1},
+        [2] = {foreground = Screen.colors.Blue1},
+        [3] = {bold = true, foreground = Screen.colors.SeaGreen4},
+      })
+      screen:attach()
+    end)
+
+    after_each(function()
+      rmdir('bang_filter_spec')
+    end)
+
+    it("doesn't truncate Last line of shell output #3269", function()
+      command(helpers.iswin()
+        and [[nnoremap <silent>\l :!dir /b bang_filter_spec<cr>]]
+        or  [[nnoremap <silent>\l :!ls bang_filter_spec<cr>]])
+      local result = (helpers.iswin()
+        and [[:!dir /b bang_filter_spec]]
+        or  [[:!ls bang_filter_spec    ]])
+      feed([[\l]])
+      screen:expect([[
+        {1:~                                                    }|
+        {1:~                                                    }|
+        {1:~                                                    }|
+        {1:~                                                    }|
+        ]]..result..[[                            |
+        f1                                                   |
+        f2                                                   |
+        f3                                                   |
+                                                             |
+        {3:Press ENTER or type command to continue}^              |
+      ]])
+    end)
+
+    it('handles binary and multibyte data', function()
+      feed_command('!cat test/functional/fixtures/shell_data.txt')
+      screen.bell = false
+      screen:expect([[
+        {1:~                                                    }|
+        {1:~                                                    }|
+        {1:~                                                    }|
+        :!cat test/functional/fixtures/shell_data.txt        |
+        {2:^@^A^B^C^D^E^F^H}                                     |
+        {2:^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_}                 |
+        ö 한글 {2:<a5><c3>}                                      |
+        t       {2:<ff>}                                         |
+                                                             |
+        {3:Press ENTER or type command to continue}^              |
+    ]], nil, nil, function()
+        eq(true, screen.bell)
+      end)
+    end)
+
+    it('handles multibyte sequences split over buffer boundaries', function()
+      command('cd '..nvim_dir)
+      local cmd
+      if iswin() then
+        cmd = '!shell-test UTF-8  '
+      else
+        cmd = '!./shell-test UTF-8'
+      end
+      feed_command(cmd)
+      -- Note: only the first example of split composed char works
+      screen:expect([[
+        {1:~                                                    }|
+        {1:~                                                    }|
+        :]]..cmd..[[                                 |
+        å                                                    |
+        ref: å̲                                               |
+        1: å̲                                                 |
+        2: å ̲                                               |
+        3: å ̲                                               |
+                                                             |
+        {3:Press ENTER or type command to continue}^              |
+      ]])
+    end)
+  end)
 end)