Merge branch 'master' into luaviml'/lua

6 files changed, 160 insertions, 83 deletions

diff --git a/ci/build.bat b/ci/build.bat
index 9071c0864e..c871c6b849 100644
--- a/ci/build.bat
+++ b/ci/build.bat
@@ -17,7 +17,7 @@ set PATH=C:\Program Files (x86)\CMake\bin\cpack.exe;%PATH%
 
 :: Build third-party dependencies
 C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm -Su" || goto :error
-C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm --needed -S mingw-w64-%ARCH%-cmake mingw-w64-%ARCH%-perl mingw-w64-%ARCH%-diffutils gperf" || goto :error
+C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm --needed -S mingw-w64-%ARCH%-cmake mingw-w64-%ARCH%-perl mingw-w64-%ARCH%-diffutils mingw-w64-%ARCH%-unibilium gperf" || goto :error
 
 :: Setup python (use AppVeyor system python)
 C:\Python27\python.exe -m pip install neovim || goto :error
@@ -31,14 +31,14 @@ python3 -c "import neovim; print(str(neovim))" || goto :error
 
 mkdir .deps
 cd .deps
-cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release ..\third-party\ || goto :error
+cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=RelWithDebInfo ..\third-party\ || goto :error
 mingw32-make VERBOSE=1 || goto :error
 cd ..
 
 :: Build Neovim
 mkdir build
 cd build
-cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUSTED_OUTPUT_TYPE=nvim -DGPERF_PRG="C:\msys64\usr\bin\gperf.exe" .. || goto :error
+cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUSTED_OUTPUT_TYPE=nvim -DGPERF_PRG="C:\msys64\usr\bin\gperf.exe" .. || goto :error
 mingw32-make VERBOSE=1 || goto :error
 bin\nvim --version || goto :error
 
@@ -46,8 +46,8 @@ bin\nvim --version || goto :error
 mingw32-make functionaltest VERBOSE=1 || goto :error
 
 :: Build artifacts
-cpack -G ZIP -C Release
-if defined APPVEYOR_REPO_TAG_NAME cpack -G NSIS -C Release
+cpack -G ZIP -C RelWithDebInfo
+if defined APPVEYOR_REPO_TAG_NAME cpack -G NSIS -C RelWithDebInfo
 
 goto :EOF
 :error
diff --git a/ci/common/build.sh b/ci/common/build.sh
index d5211405ca..f398a1a1cc 100644
--- a/ci/common/build.sh
+++ b/ci/common/build.sh
@@ -7,11 +7,11 @@ build_make() {
 }
 
 build_deps() {
-  if [[ "${BUILD_32BIT}" == ON ]]; then
+  if test "${BUILD_32BIT}" = ON ; then
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
   fi
-  if [[ "${FUNCTIONALTEST}" == "functionaltest-lua" ]] \
-     || [[ "${CLANG_SANITIZER}" == "ASAN_UBSAN" ]]; then
+  if test "${FUNCTIONALTEST}" = "functionaltest-lua" \
+     || test "${CLANG_SANITIZER}" = "ASAN_UBSAN" ; then
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} -DUSE_BUNDLED_LUA=ON"
   fi
 
@@ -19,16 +19,15 @@ build_deps() {
 
   # If there is a valid cache and we're not forced to recompile,
   # use cached third-party dependencies.
-  if [[ -f "${CACHE_MARKER}" ]] && [[ "${BUILD_NVIM_DEPS}" != true ]]; then
-    if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
-      local statcmd="stat -f '%Sm'"
-    else
-      local statcmd="stat -c '%y'"
+  if test -f "${CACHE_MARKER}" && test "${BUILD_NVIM_DEPS}" != "true" ; then
+    local statcmd="stat -c '%y'"
+    if test "${TRAVIS_OS_NAME}" = osx ; then
+      statcmd="stat -f '%Sm'"
     fi
     echo "Using third-party dependencies from Travis's cache (last updated: $(${statcmd} "${CACHE_MARKER}"))."
 
-     mkdir -p "$(dirname "${DEPS_BUILD_DIR}")"
-     mv "${HOME}/.cache/nvim-deps" "${DEPS_BUILD_DIR}"
+    mkdir -p "$(dirname "${DEPS_BUILD_DIR}")"
+    mv "${HOME}/.cache/nvim-deps" "${DEPS_BUILD_DIR}"
   else
     mkdir -p "${DEPS_BUILD_DIR}"
   fi
@@ -47,10 +46,10 @@ build_deps() {
 }
 
 prepare_build() {
-  if [[ -n "${CLANG_SANITIZER}" ]]; then
+  if test -n "${CLANG_SANITIZER}" ; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DCLANG_${CLANG_SANITIZER}=ON"
   fi
-  if [[ "${BUILD_32BIT}" == ON ]]; then
+  if test "${BUILD_32BIT}" = ON ; then
     CMAKE_FLAGS="${CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
   fi
 
@@ -62,24 +61,24 @@ prepare_build() {
 
 build_nvim() {
   echo "Building nvim."
-  if ! top_make nvim; then
+  if ! top_make nvim ; then
     exit 1
   fi
 
-  if [ "$CLANG_SANITIZER" != "TSAN" ]; then
+  if test "$CLANG_SANITIZER" != "TSAN" ; then
     echo "Building libnvim."
-    if ! top_make libnvim; then
+    if ! top_make libnvim ; then
       exit 1
     fi
 
     echo "Building nvim-test."
-    if ! top_make nvim-test; then
+    if ! top_make nvim-test ; then
       exit 1
     fi
   fi
 
   # Invoke nvim to trigger *San early.
-  if ! (bin/nvim --version && bin/nvim -u NONE -e -c ':qall'); then
+  if ! (bin/nvim --version && bin/nvim -u NONE -e -c ':qall') ; then
     asan_check "${LOG_DIR}"
     exit 1
   fi
diff --git a/ci/common/suite.sh b/ci/common/suite.sh
index 568d5d5bee..8feb642547 100644
--- a/ci/common/suite.sh
+++ b/ci/common/suite.sh
@@ -11,20 +11,41 @@ FAIL_SUMMARY=""
 END_MARKER="$BUILD_DIR/.tests_finished"
 FAIL_SUMMARY_FILE="$BUILD_DIR/.test_errors"
 
+ANSI_CLEAR="\033[0K"
+
+travis_fold() {
+  local action="$1"
+  local name="$2"
+  name="$(echo -n "$name" | tr '\n\0' '--' | sed 's/[^A-Za-z0-9]\{1,\}/-/g')"
+  name="$(echo -n "$name" | sed 's/-$//')"
+  echo -en "travis_fold:${action}:${name}\r${ANSI_CLEAR}"
+}
+
+if test "$TRAVIS" != "true" ; then
+  travis_fold() {
+    return 0
+  }
+fi
+
 enter_suite() {
+  set +x
   FAILED=0
   rm -f "${END_MARKER}"
   local suite_name="$1"
   export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name"
+  travis_fold start "${NVIM_TEST_CURRENT_SUITE}"
+  set -x
 }
 
 exit_suite() {
+  set +x
+  travis_fold end "${NVIM_TEST_CURRENT_SUITE}"
   if test $FAILED -ne 0 ; then
     echo "Suite ${NVIM_TEST_CURRENT_SUITE} failed, summary:"
     echo "${FAIL_SUMMARY}"
   fi
   export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}"
-  if test "x$1" != "x--continue" ; then
+  if test "$1" != "--continue" ; then
     exit $FAILED
   else
     local saved_failed=$FAILED
@@ -61,7 +82,7 @@ run_test() {
 
 run_test_wd() {
   local hang_ok=
-  if test "x$1" = "x--allow-hang" ; then
+  if test "$1" = "--allow-hang" ; then
     hang_ok=1
     shift
   fi
@@ -82,21 +103,31 @@ run_test_wd() {
 
   local output_file="$(mktemp)"
   local status_file="$(mktemp)"
+  local sid_file="$(mktemp)"
 
   local restarts=5
   local prev_tmpsize=-1
   while test $restarts -gt 0 ; do
-    : > "${status_file}"
-    (
-      set -o pipefail
-      ret=0
-      if ! eval "$cmd" 2>&1 | tee -a "$output_file" ; then
-        ret=1
-      fi
-      echo "$ret" > "$status_file"
-      exit $ret
-    ) &
-    local pid=$!
+    : > "$status_file"
+    : > "$sid_file"
+    setsid \
+      env \
+        output_file="$output_file" \
+        status_file="$status_file" \
+        sid_file="$sid_file" \
+        cmd="$cmd" \
+        CI_DIR="$CI_DIR" \
+        sh -c '
+          . "${CI_DIR}/common/test.sh"
+          ps -o sid= > "$sid_file"
+          (
+            ret=0
+            if ! eval "$cmd" 2>&1 ; then
+              ret=1
+            fi
+            echo "$ret" > "$status_file"
+          ) | tee -a "$output_file"
+        '
     while test "$(stat -c "%s" "$status_file")" -eq 0 ; do
       prev_tmpsize=$tmpsize
       sleep $timeout
@@ -106,13 +137,23 @@ run_test_wd() {
         break
       fi
     done
-    restarts=$[ restarts - 1 ]
+    restarts=$(( restarts - 1 ))
     if test "$(stat -c "%s" "$status_file")" -eq 0 ; then
-      # status file not updated, assuming hang
-      kill -KILL $pid
+      # Status file not updated, assuming hang
+
+      # SID not known, this should not ever happen
+      if test "$(stat -c "%s" "$sid_file")" -eq 0 ; then
+        fail "$test_name" E "Shell did not run"
+        break
+      fi
+
+      # Kill all processes which belong to one session: should get rid of test
+      # processes as well as sh itself.
+      pkill -KILL -s$(cat "$sid_file")
+
       if test $restarts -eq 0 ; then
-        if test "x$hang_ok" = "x" ; then
-          fail "${test_name}" E "Test hang up"
+        if test -z "$hang_ok" ; then
+          fail "$test_name" E "Test hang up"
         fi
       else
         echo "Test ${test_name} hang up, restarting"
@@ -120,21 +161,25 @@ run_test_wd() {
       fi
     else
       local new_failed="$(cat "$status_file")"
-      if test "x$new_failed" != "x0" ; then
-        fail "${test_name}" F "Test failed in run_test_wd"
+      if test "$new_failed" != "0" ; then
+        fail "$test_name" F "Test failed in run_test_wd"
       fi
-      return 0
+      break
     fi
   done
+
+  rm -f "$output_file"
+  rm -f "$status_file"
+  rm -f "$sid_file"
 }
 
 ended_successfully() {
-  if [[ -f "${FAIL_SUMMARY_FILE}" ]]; then
+  if test -f "${FAIL_SUMMARY_FILE}" ; then
     echo 'Test failed, complete summary:'
     cat "${FAIL_SUMMARY_FILE}"
     return 1
   fi
-  if ! [[ -f "${END_MARKER}" ]] ; then
+  if ! test -f "${END_MARKER}" ; then
     echo 'ended_successfully called before end marker was touched'
     return 1
   fi
diff --git a/ci/common/test.sh b/ci/common/test.sh
index d911d9bc18..55f76ca798 100644
--- a/ci/common/test.sh
+++ b/ci/common/test.sh
@@ -1,5 +1,5 @@
-source "${CI_DIR}/common/build.sh"
-source "${CI_DIR}/common/suite.sh"
+. "${CI_DIR}/common/build.sh"
+. "${CI_DIR}/common/suite.sh"
 
 print_core() {
   local app="$1"
@@ -9,7 +9,7 @@ print_core() {
     return 0
   fi
   echo "======= Core file $core ======="
-  if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  if test "${TRAVIS_OS_NAME}" = osx ; then
     lldb -Q -o "bt all" -f "${app}" -c "${core}"
   else
     gdb -n -batch -ex 'thread apply all bt full' "${app}" -c "${core}"
@@ -23,13 +23,13 @@ check_core_dumps() {
     shift
   fi
   local app="${1:-${BUILD_DIR}/bin/nvim}"
-  if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  if test "${TRAVIS_OS_NAME}" = osx ; then
     local cores="$(find /cores/ -type f -print)"
   else
     local cores="$(find ./ -type f -name 'core.*' -print)"
   fi
 
-  if [ -z "${cores}" ]; then
+  if test -z "${cores}" ; then
     return
   fi
   local core
@@ -61,7 +61,7 @@ check_logs() {
     cat "${log}"
     err=1
   done
-  if [[ -n "${err}" ]]; then
+  if test -n "${err}" ; then
     fail 'logs' E 'Runtime errors detected.'
   fi
 }
@@ -76,7 +76,7 @@ asan_check() {
 
 run_unittests() {(
   enter_suite unittests
-  ulimit -c unlimited
+  ulimit -c unlimited || true
   if ! build_make unittest ; then
     fail 'unittests' F 'Unit tests failed'
   fi
@@ -86,7 +86,7 @@ run_unittests() {(
 
 run_functionaltests() {(
   enter_suite functionaltests
-  ulimit -c unlimited
+  ulimit -c unlimited || true
   if ! build_make ${FUNCTIONALTEST}; then
     fail 'functionaltests' F 'Functional tests failed'
   fi
@@ -98,7 +98,7 @@ run_functionaltests() {(
 
 run_oldtests() {(
   enter_suite oldtests
-  ulimit -c unlimited
+  ulimit -c unlimited || true
   if ! make -C "${TRAVIS_BUILD_DIR}/src/nvim/testdir"; then
     reset
     fail 'oldtests' F 'Legacy tests failed'
@@ -109,6 +109,27 @@ run_oldtests() {(
   exit_suite
 )}
 
+check_runtime_files() {(
+  set +x
+  local test_name="$1" ; shift
+  local message="$1" ; shift
+  local tst="$1" ; shift
+
+  cd runtime
+  for file in $(git ls-files "$@") ; do
+    # Check that test is not trying to work with files with spaces/etc
+    # Prefer failing the build over using more robust construct because files
+    # with IFS are not welcome.
+    if ! test -e "$file" ; then
+      fail "$test_name" E \
+        "It appears that $file is only a part of the file name"
+    fi
+    if ! test "$tst" "$INSTALL_PREFIX/share/nvim/runtime/$file" ; then
+      fail "$test_name" F "$(printf "$message" "$file")"
+    fi
+  done
+)}
+
 install_nvim() {(
   enter_suite 'install_nvim'
   if ! build_make install ; then
@@ -117,34 +138,37 @@ install_nvim() {(
   fi
 
   "${INSTALL_PREFIX}/bin/nvim" --version
-  "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || {
+  if ! "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' ; then
     echo "Running ':help' in the installed nvim failed."
     echo "Maybe the helptags have not been generated properly."
     fail 'help' F 'Failed running :help'
-  }
+  fi
 
-  local genvimsynf=syntax/vim/generated.vim
   # Check that all runtime files were installed
-  for file in doc/tags $genvimsynf $(
-    cd runtime ; git ls-files | grep -e '.vim$' -e '.ps$' -e '.dict$' -e '.py$' -e '.tutor$'
-  ) ; do
-    if ! test -e "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
-      fail 'runtime-install' F "It appears that $file is not installed."
-    fi
-  done
+  check_runtime_files \
+    'runtime-install' \
+    'It appears that %s is not installed.' \
+    -e \
+    '*.vim' '*.ps' '*.dict' '*.py' '*.tutor'
+
+  # Check that some runtime files are installed and are executables
+  check_runtime_files \
+    'not-exe' \
+    'It appears that %s is not installed or is not executable.' \
+    -x \
+    '*.awk' '*.sh' '*.bat'
 
   # Check that generated syntax file has function names, #5060.
+  local genvimsynf=syntax/vim/generated.vim
   local gpat='syn keyword vimFuncName .*eval'
-  if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf"; then
+  if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf" ; then
     fail 'funcnames' F "It appears that $genvimsynf does not contain $gpat."
   fi
 
-  for file in $(
-    cd runtime ; git ls-files | grep -e '.awk$' -e '.sh$' -e '.bat$'
-  ) ; do
-    if ! test -x "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
-      fail 'not-exe' F "It appears that $file is not installed or is not executable."
-    fi
-  done
   exit_suite
 )}
+
+csi_clean() {
+  find "${BUILD_DIR}/bin" -name 'test-includes-*' -delete
+  find "${BUILD_DIR}" -name '*test-include*.o' -delete
+}
diff --git a/ci/run_lint.sh b/ci/run_lint.sh
index 5639b4c3db..73647dacaa 100755
--- a/ci/run_lint.sh
+++ b/ci/run_lint.sh
@@ -8,17 +8,20 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/suite.sh"
 
-enter_suite 'lint'
+enter_suite 'clint'
 
-set -x
+run_test 'top_make clint-full' clint
 
-csi_clean() {
-  find "${BUILD_DIR}/bin" -name 'test-includes-*' -delete
-  find "${BUILD_DIR}" -name '*test-include*.o' -delete
-}
+exit_suite --continue
+
+enter_suite 'testlint'
 
-run_test 'top_make clint-full' clint
 run_test 'top_make testlint' testlint
+
+exit_suite --continue
+
+enter_suite single-includes
+
 CLICOLOR_FORCE=1 run_test_wd \
   --allow-hang \
   10s \
@@ -26,4 +29,6 @@ CLICOLOR_FORCE=1 run_test_wd \
   'csi_clean' \
   single-includes
 
+exit_suite --continue
+
 end_tests
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index 4abc9eea9f..a0bf6e010d 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -8,16 +8,18 @@ source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/test.sh"
 source "${CI_DIR}/common/suite.sh"
 
-set -x
-
-enter_suite tests
+enter_suite build
 
 check_core_dumps --delete quiet
 
 prepare_build
 build_nvim
 
-if [ "$CLANG_SANITIZER" != "TSAN" ]; then
+exit_suite --continue
+
+enter_suite tests
+
+if test "$CLANG_SANITIZER" != "TSAN" ; then
   # Additional threads are only created when the builtin UI starts, which
   # doesn't happen in the unit/functional tests
   run_test run_unittests
@@ -27,4 +29,6 @@ run_test run_oldtests
 
 run_test install_nvim
 
+exit_suite --continue
+
 end_tests