vim-patch:8.1.1476: no statistics displayed after running tests

Problem:    No statistics displayed after running tests.
Solution:   Summarize the test results. (Christian Brabandt, closes vim/vim#4391)
            Also make it possible to report a skipped file.
https://github.com/vim/vim/commit/9c0cec65f891492314caadeef87a50251a21e630

Removes our custom no-inits from `$(RUN_VIMTEST)`, since we have
`$(NO_INITS)` now also.

5 files changed, 74 insertions, 11 deletions

diff --git a/src/nvim/testdir/Makefile b/src/nvim/testdir/Makefile
index 90381d9116..ef3dce5932 100644
--- a/src/nvim/testdir/Makefile
+++ b/src/nvim/testdir/Makefile
@@ -86,15 +86,14 @@ nongui: nolog $(FIXFF) $(SCRIPTS) newtests report
 	@echo 'set $$_exitcode = -1\nrun\nif $$_exitcode != -1\n  quit\nend' > .gdbinit
 
 report:
+	$(RUN_VIMTEST) $(NO_INITS) -S summarize.vim messages
 	@echo
 	@echo 'Test results:'
-	@/bin/sh -c "if test -f test.log; then \
-	                 cat test.log;         \
-	                 echo TEST FAILURE;    \
-	                 exit 1;               \
-	             else                      \
-	                 echo ALL DONE;        \
-	             fi"
+	@cat test_result.log
+	@/bin/sh -c "if test -f test.log; \
+		then echo TEST FAILURE; exit 1; \
+		else echo ALL DONE; \
+		fi"
 
 test1.out: $(NVIM_PRG)
 
@@ -124,6 +123,7 @@ CLEAN_FILES := *.out \
   *.orig            \
   *.tlog            \
   test.log          \
+  test_result.log   \
   messages          \
   $(RM_ON_RUN)      \
   $(RM_ON_START)    \
@@ -163,7 +163,7 @@ nolog:
 # New style of tests uses Vim script with assert calls.  These are easier
 # to write and a lot easier to read and debug.
 # Limitation: Only works with the +eval feature.
-RUN_VIMTEST = $(TOOL) $(NVIM_PRG) -u unix.vim -U NONE -i viminfo --headless --noplugin
+RUN_VIMTEST = $(TOOL) $(NVIM_PRG) -u unix.vim
 
 newtests: newtestssilent
 	@/bin/sh -c "if test -f messages && grep -q 'FAILED' messages; then \
@@ -176,4 +176,4 @@ newtestssilent: $(NEW_TESTS)
 	@echo "[OLDTEST] Running" $*
 	@rm -rf $*.failed test.ok $(RM_ON_RUN)
 	@mkdir -p $(TMPDIR)
-	@/bin/sh runnvim.sh $(ROOT) $(NVIM_PRG) $* $(RUN_VIMTEST) -u NONE -S runtest.vim $*.vim
+	@/bin/sh runnvim.sh $(ROOT) $(NVIM_PRG) $* $(RUN_VIMTEST) $(NO_INITS) -u NONE -S runtest.vim $*.vim
diff --git a/src/nvim/testdir/runtest.vim b/src/nvim/testdir/runtest.vim
index ef4676a770..fa25740994 100644
--- a/src/nvim/testdir/runtest.vim
+++ b/src/nvim/testdir/runtest.vim
@@ -268,6 +268,9 @@ if expand('%') =~ 'test_vimscript.vim'
 else
   try
     source %
+  catch /^\cskipped/
+    call add(s:messages, '    Skipped')
+    call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '',  ''))
   catch
     let s:fail += 1
     call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)
diff --git a/src/nvim/testdir/summarize.vim b/src/nvim/testdir/summarize.vim
new file mode 100644
index 0000000000..43dbce0f44
--- /dev/null
+++ b/src/nvim/testdir/summarize.vim
@@ -0,0 +1,60 @@
+if 1
+  " This is executed with the eval feature
+  set nocp
+  func Count(match, type)
+    if a:type ==# 'executed'
+      let g:executed += (a:match+0)
+    elseif a:type ==# 'failed'
+      let g:failed = a:match+0
+    elseif a:type ==# 'skipped'
+      let g:skipped += 1
+      call extend(g:skipped_output, ["\t".a:match])
+    endif
+  endfunc
+
+  let g:executed = 0
+  let g:skipped = 0
+  let g:failed = 0
+  let g:skipped_output = []
+  let g:failed_output = []
+  let output = [""]
+
+  try
+    " This uses the :s command to just fetch and process the output of the
+    " tests, it doesn't acutally replay anything
+    %s/^Executed\s\+\zs\d\+\ze\s\+tests/\=Count(submatch(0),'executed')/egn
+    %s/^SKIPPED \zs.*/\=Count(submatch(0), 'skipped')/egn
+    %s/^\(\d\+\)\s\+FAILED:/\=Count(submatch(1), 'failed')/egn
+
+    call extend(output, ["Skipped:"]) 
+    call extend(output, skipped_output)
+
+    call extend(output, [
+          \ "",
+          \ "-------------------------------",
+          \ printf("Executed: %5d Tests", g:executed),
+          \ printf(" Skipped: %5d Tests", g:skipped),
+          \ printf("  %s: %5d Tests", g:failed == 0 ? 'Failed' : 'FAILED', g:failed),
+          \ "",
+          \ ]) 
+    if filereadable('test.log')
+      " outputs and indents the failed test result
+      call extend(output, ["", "Failures: "])
+      let failed_output = filter(readfile('test.log'), { v,k -> !empty(k)})
+      call extend(output, map(failed_output, { v,k -> "\t".k}))
+      " Add a final newline
+      call extend(output, [""])
+    endif
+
+  catch  " Catch-all
+  finally
+    call writefile(output, 'test_result.log')  " overwrites an existing file
+    q!
+  endtry
+endif
+
+" This is executed without the eval feature
+%d
+r test.log
+w test_result.log
+q!
diff --git a/src/nvim/testdir/test_arabic.vim b/src/nvim/testdir/test_arabic.vim
index d0211fee22..450c6f98f5 100644
--- a/src/nvim/testdir/test_arabic.vim
+++ b/src/nvim/testdir/test_arabic.vim
@@ -3,7 +3,7 @@
 " functional tests that check the shaping works with real text.
 
 if !has('arabic')
-  finish
+  throw 'Skipped: arabic feature missing'
 endif
 
 source view_util.vim
diff --git a/src/nvim/testdir/test_autochdir.vim b/src/nvim/testdir/test_autochdir.vim
index 05d69631c4..67c537b407 100644
--- a/src/nvim/testdir/test_autochdir.vim
+++ b/src/nvim/testdir/test_autochdir.vim
@@ -1,7 +1,7 @@
 " Test 'autochdir' behavior
 
 if !exists("+autochdir")
-  finish
+  throw 'Skipped: autochdir feature missing'
 endif
 
 func Test_set_filename()