From 2bf9d36ccd59f12e3895c885e8cf17e620bf191b Mon Sep 17 00:00:00 2001
From: ZyX <kp-pav@yandex.ru>
Date: Fri, 31 Mar 2017 16:02:53 +0300
Subject: ci: Refactor CI scripts

1. CI_TARGET now determines which run_${CI_TARGET}.sh script to use. Defaults to
   `tests`.
2. Build no longer halts on the first failing suit: e.g. if functional tests
   failed it will continue with unit tests, etc.
3. All ${MAKE_CMD} occurrences moved to `top_make` function, added `build_make`
   as an alias to `make -C build` (`"${MAKE_CMD}" -C "${BUILD_DIR}"`) which is
   too verbose.

`suite.sh` was copied from powerline (tests/common.sh file), assumes running
with POSIX shells (and actually uses dash in powerline). Then some convenience
functions were added (run_test and below).
---
 ci/common/build.sh | 16 ++++++++++++----
 ci/common/suite.sh | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 ci/common/test.sh  |  8 +++++---
 ci/run_lint.sh     | 17 +++++++++++++++++
 ci/run_tests.sh    | 17 ++++++++++++-----
 ci/script.sh       |  9 ++-------
 6 files changed, 103 insertions(+), 19 deletions(-)
 create mode 100644 ci/common/suite.sh
 create mode 100755 ci/run_lint.sh

(limited to 'ci')

diff --git a/ci/common/build.sh b/ci/common/build.sh
index 44087513ee..19c23bdda8 100644
--- a/ci/common/build.sh
+++ b/ci/common/build.sh
@@ -1,3 +1,11 @@
+top_make() {
+  "${MAKE_CMD}" "$@"
+}
+
+build_make() {
+  top_make -C "${BUILD_DIR}" "$@"
+}
+
 build_deps() {
   if [[ "${BUILD_32BIT}" == ON ]]; then
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
@@ -30,7 +38,7 @@ build_deps() {
   echo "Configuring with '${DEPS_CMAKE_FLAGS}'."
   CC= cmake ${DEPS_CMAKE_FLAGS} "${TRAVIS_BUILD_DIR}/third-party/"
 
-  if ! ${MAKE_CMD}; then
+  if ! top_make; then
     exit 1
   fi
 
@@ -53,18 +61,18 @@ prepare_build() {
 
 build_nvim() {
   echo "Building nvim."
-  if ! ${MAKE_CMD} nvim; then
+  if ! top_make nvim; then
     exit 1
   fi
 
   if [ "$CLANG_SANITIZER" != "TSAN" ]; then
     echo "Building libnvim."
-    if ! ${MAKE_CMD} libnvim; then
+    if ! top_make libnvim; then
       exit 1
     fi
 
     echo "Building nvim-test."
-    if ! ${MAKE_CMD} nvim-test; then
+    if ! top_make nvim-test; then
       exit 1
     fi
   fi
diff --git a/ci/common/suite.sh b/ci/common/suite.sh
new file mode 100644
index 0000000000..ad7c30708f
--- /dev/null
+++ b/ci/common/suite.sh
@@ -0,0 +1,55 @@
+FAILED=0
+
+FAIL_SUMMARY=""
+
+enter_suite() {
+  local suite_name="$1"
+  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name"
+}
+
+exit_suite() {
+  if test $FAILED -ne 0 ; then
+    echo "Suite ${NVIM_TEST_CURRENT_SUITE} failed, summary:"
+    echo "${FAIL_SUMMARY}"
+  fi
+  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}"
+  if test "x$1" != "x--continue" ; then
+    exit $FAILED
+  fi
+}
+
+fail() {
+  local allow_failure=
+  if test "x$1" = "x--allow-failure" ; then
+    shift
+    allow_failure=A
+  fi
+  local test_name="$1"
+  local fail_char="$allow_failure$2"
+  local message="$3"
+
+  : ${fail_char:=F}
+  : ${message:=Test $test_name failed}
+
+  local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message"
+  FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}"
+  echo "Failed: $full_msg"
+  if test "x$allow_failure" = "x" ; then
+    FAILED=1
+  fi
+}
+
+run_test() {
+  local cmd="$1"
+  shift
+  local test_name="$1"
+  : ${test_name:=$cmd}
+  shift
+  if ! eval "$cmd" ; then
+    fail "${test_name}" "$@"
+  fi
+}
+
+succeeded() {
+  return $FAILED
+}
diff --git a/ci/common/test.sh b/ci/common/test.sh
index b28e46a4df..4936992cfd 100644
--- a/ci/common/test.sh
+++ b/ci/common/test.sh
@@ -1,3 +1,5 @@
+source "${CI_DIR}/common/build.sh"
+
 print_core() {
   local app="$1"
   local core="$2"
@@ -75,7 +77,7 @@ asan_check() {
 
 run_unittests() {
   ulimit -c unlimited
-  if ! ${MAKE_CMD} -C "${BUILD_DIR}" unittest ; then
+  if ! build_make unittest ; then
     check_core_dumps "$(which luajit)"
     exit 1
   fi
@@ -84,7 +86,7 @@ run_unittests() {
 
 run_functionaltests() {
   ulimit -c unlimited
-  if ! ${MAKE_CMD} -C "${BUILD_DIR}" ${FUNCTIONALTEST}; then
+  if ! build_make ${FUNCTIONALTEST}; then
     asan_check "${LOG_DIR}"
     valgrind_check "${LOG_DIR}"
     check_core_dumps
@@ -110,7 +112,7 @@ run_oldtests() {
 }
 
 install_nvim() {
-  ${MAKE_CMD} -C "${BUILD_DIR}" install
+  build_make install
 
   "${INSTALL_PREFIX}/bin/nvim" --version
   "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || {
diff --git a/ci/run_lint.sh b/ci/run_lint.sh
new file mode 100755
index 0000000000..c7937930d1
--- /dev/null
+++ b/ci/run_lint.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${CI_DIR}/common/build.sh"
+source "${CI_DIR}/common/suite.sh"
+
+enter_suite 'lint'
+
+run_test 'top_make clint-full' clint
+run_test 'top_make testlint' testlint
+run_test 'top_make check-single-includes' single-includes
+
+exit_suite
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index 6347ac15d4..b552e1f520 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -6,6 +6,9 @@ set -o pipefail
 CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/test.sh"
+source "${CI_DIR}/common/suite.sh"
+
+enter_suite tests
 
 check_core_dumps --delete quiet
 
@@ -15,11 +18,15 @@ build_nvim
 if [ "$CLANG_SANITIZER" != "TSAN" ]; then
   # Additional threads are only created when the builtin UI starts, which
   # doesn't happen in the unit/functional tests
-  run_unittests
-  run_functionaltests
+  run_test run_unittests
+  run_test run_functionaltests
 fi
-run_oldtests
+run_test run_oldtests
+
+run_test install_nvim
 
-install_nvim
+if succeeded ; then
+  touch "${SUCCESS_MARKER}"
+fi
 
-touch "${SUCCESS_MARKER}"
+exit_suite
diff --git a/ci/script.sh b/ci/script.sh
index 79a1bec201..a59c40cd2d 100755
--- a/ci/script.sh
+++ b/ci/script.sh
@@ -3,16 +3,11 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
-  make "${CI_TARGET}"
-  exit 0
-fi
-
 # This will pass the environment variables down to a bash process which runs
 # as $USER, while retaining the environment variables defined and belonging
 # to secondary groups given above in usermod.
 if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
-  sudo -E su "${USER}" -c "ci/run_tests.sh"
+  sudo -E su "${USER}" -c "ci/run_${CI_TARGET}.sh"
 else
-  ci/run_tests.sh
+  ci/run_${CI_TARGET}.sh
 fi
-- 
cgit 


From 2da3caef1b4d34de3403071b9bf54c7339f3c609 Mon Sep 17 00:00:00 2001
From: ZyX <kp-pav@yandex.ru>
Date: Fri, 31 Mar 2017 16:17:38 +0300
Subject: ci: Do not quote MAKE_CMD

---
 ci/common/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'ci')

diff --git a/ci/common/build.sh b/ci/common/build.sh
index 19c23bdda8..129622b522 100644
--- a/ci/common/build.sh
+++ b/ci/common/build.sh
@@ -1,5 +1,5 @@
 top_make() {
-  "${MAKE_CMD}" "$@"
+  ${MAKE_CMD} "$@"
 }
 
 build_make() {
-- 
cgit 


From 4c20733f6bbdcfe2a230766aa6b602681a399ac8 Mon Sep 17 00:00:00 2001
From: ZyX <kp-pav@yandex.ru>
Date: Fri, 31 Mar 2017 16:19:47 +0300
Subject: ci: Add ${NL} variable

---
 ci/common/suite.sh | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'ci')

diff --git a/ci/common/suite.sh b/ci/common/suite.sh
index ad7c30708f..e8c6a2b07a 100644
--- a/ci/common/suite.sh
+++ b/ci/common/suite.sh
@@ -1,3 +1,7 @@
+# HACK: get newline for use in strings given that "\n" and $'' do not work.
+NL="$(printf '\nE')"
+NL="${NL%E}"
+
 FAILED=0
 
 FAIL_SUMMARY=""
-- 
cgit 


From d9069b9fe490e3be3ac06985f7f8625af51d5129 Mon Sep 17 00:00:00 2001
From: ZyX <kp-pav@yandex.ru>
Date: Fri, 31 Mar 2017 17:39:18 +0300
Subject: ci: Check for exact value of CI_TARGET, not its emptyness

---
 ci/before_install.sh | 2 +-
 ci/before_script.sh  | 2 +-
 ci/install.sh        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'ci')

diff --git a/ci/before_install.sh b/ci/before_install.sh
index 9aac37de12..5b36adaef2 100755
--- a/ci/before_install.sh
+++ b/ci/before_install.sh
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
+if [[ "${CI_TARGET}" == lint ]]; then
   exit
 fi
 
diff --git a/ci/before_script.sh b/ci/before_script.sh
index 4a75e89fbe..445996a8df 100755
--- a/ci/before_script.sh
+++ b/ci/before_script.sh
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
+if [[ "${CI_TARGET}" == lint ]]; then
   exit
 fi
 
diff --git a/ci/install.sh b/ci/install.sh
index 98d3dc01cb..4ee99e1e44 100755
--- a/ci/install.sh
+++ b/ci/install.sh
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
+if [[ "${CI_TARGET}" == lint ]]; then
   exit
 fi
 
-- 
cgit 


From 929c398aab786473e6c28998862cbd1356de0166 Mon Sep 17 00:00:00 2001
From: ZyX <kp-pav@yandex.ru>
Date: Fri, 31 Mar 2017 19:09:18 +0300
Subject: ci: Enable tracing

---
 ci/run_tests.sh | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'ci')

diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index b552e1f520..92cb5a9fd8 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -8,6 +8,8 @@ source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/test.sh"
 source "${CI_DIR}/common/suite.sh"
 
+set -x
+
 enter_suite tests
 
 check_core_dumps --delete quiet
-- 
cgit 


From 86f5b1276bf444b164ac3a3b60b411afe4bd7ec4 Mon Sep 17 00:00:00 2001
From: ZyX <kp-pav@yandex.ru>
Date: Fri, 31 Mar 2017 19:42:18 +0300
Subject: ci: Add test watchdog and tracing for lint tests

---
 ci/common/suite.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 ci/run_lint.sh     |  4 +++-
 2 files changed, 51 insertions(+), 1 deletion(-)

(limited to 'ci')

diff --git a/ci/common/suite.sh b/ci/common/suite.sh
index e8c6a2b07a..8c44e5f974 100644
--- a/ci/common/suite.sh
+++ b/ci/common/suite.sh
@@ -54,6 +54,54 @@ run_test() {
   fi
 }
 
+run_test_wd() {
+  local timeout="$1"
+  shift
+  local cmd="$1"
+  shift
+  local test_name="$1"
+  : ${test_name:=$cmd}
+  shift
+  local output_file="$(mktemp)"
+  local status_file="$(mktemp)"
+  local restarts=5
+  local prev_tmpsize=-1
+  while test $restarts -gt 0 ; do
+    : > "${status_file}"
+    (
+      if ! (
+        set -o pipefail
+        eval "$cmd" 2>&1 | tee -a "$output_file"
+      ) ; then
+        fail "${test_name}" "$@"
+      fi
+      echo "$FAILED" > "$status_file"
+    ) &
+    local pid=$!
+    while test "$(stat -c "%s" "$status_file")" -eq 0 ; do
+      prev_tmpsize=$tmpsize
+      sleep $timeout
+      tmpsize="$(stat -c "%s" "$output_file")"
+      if test $tempsize -eq $prev_temsize ; then
+        # no output, assuming either hang or exit
+        break
+      fi
+    done
+    if test "$(stat -c "%s" "$status_file")" -eq 0 ; then
+      # status file not updated, assuming hang
+      kill -KILL $pid
+      echo "Test ${test_name} hang up, restarting"
+    else
+      local new_failed="$(cat "$status_file")"
+      if test "x$new_failed" != "x0" ; then
+        fail "${test_name}" F "Test failed in run_test_wd"
+      fi
+      return 0
+    fi
+    restarts=$[ restarts - 1 ]
+  done
+}
+
 succeeded() {
   return $FAILED
 }
diff --git a/ci/run_lint.sh b/ci/run_lint.sh
index c7937930d1..5122ffc2b6 100755
--- a/ci/run_lint.sh
+++ b/ci/run_lint.sh
@@ -10,8 +10,10 @@ source "${CI_DIR}/common/suite.sh"
 
 enter_suite 'lint'
 
+set -x
+
 run_test 'top_make clint-full' clint
 run_test 'top_make testlint' testlint
-run_test 'top_make check-single-includes' single-includes
+run_test_wd 5s 'top_make check-single-includes' single-includes
 
 exit_suite
-- 
cgit 


From 6ddaace7ac9f2ed13d7b210d87c715c9f7209c7f Mon Sep 17 00:00:00 2001
From: ZyX <kp-pav@yandex.ru>
Date: Fri, 31 Mar 2017 20:52:05 +0300
Subject: ci: Do not shift if there are not enough arguments

---
 ci/common/suite.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'ci')

diff --git a/ci/common/suite.sh b/ci/common/suite.sh
index 8c44e5f974..a70cafc92c 100644
--- a/ci/common/suite.sh
+++ b/ci/common/suite.sh
@@ -45,10 +45,10 @@ fail() {
 
 run_test() {
   local cmd="$1"
-  shift
+  test $# -gt 0 && shift
   local test_name="$1"
   : ${test_name:=$cmd}
-  shift
+  test $# -gt 0 && shift
   if ! eval "$cmd" ; then
     fail "${test_name}" "$@"
   fi
@@ -56,12 +56,12 @@ run_test() {
 
 run_test_wd() {
   local timeout="$1"
-  shift
+  test $# -gt 0 && shift
   local cmd="$1"
-  shift
+  test $# -gt 0 && shift
   local test_name="$1"
   : ${test_name:=$cmd}
-  shift
+  test $# -gt 0 && shift
   local output_file="$(mktemp)"
   local status_file="$(mktemp)"
   local restarts=5
-- 
cgit