diff options
| -rw-r--r-- | .travis.yml | 5 | ||||
| -rwxr-xr-x | ci/before_cache.sh | 5 | ||||
| -rw-r--r-- | ci/common/suite.sh | 58 | ||||
| -rw-r--r-- | ci/common/test.sh | 62 | ||||
| -rwxr-xr-x | ci/run_lint.sh | 2 | ||||
| -rwxr-xr-x | ci/run_tests.sh | 6 |
6 files changed, 75 insertions, 63 deletions
diff --git a/.travis.yml b/.travis.yml index 14899a1289..b8c4c0172f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,11 +43,6 @@ env: # If this file exists, we know that the cache contains compiled # dependencies and we can use it. - CACHE_MARKER="$HOME/.cache/nvim-deps/.travis_cache_marker" - # Test success marker. If this file exists, we know that all tests - # were successful. Required because we only want to update the cache - # if the tests were successful, but don't have this information - # available in before_cache (which is run before after_success). - - SUCCESS_MARKER="$BUILD_DIR/.tests_successful" # default target name for functional tests - FUNCTIONALTEST=functionaltest - CI_TARGET=tests diff --git a/ci/before_cache.sh b/ci/before_cache.sh index dd1fcf2bf7..3d7cc0ec5a 100755 --- a/ci/before_cache.sh +++ b/ci/before_cache.sh @@ -3,12 +3,15 @@ set -e set -o pipefail +CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${CI_DIR}/common/suite.sh" + # Don't cache pip's log and selfcheck. rm -rf "${HOME}/.cache/pip/log" rm -f "${HOME}/.cache/pip/selfcheck.json" # Update the third-party dependency cache only if the build was successful. -if [[ -f "${SUCCESS_MARKER}" ]]; then +if ended_successfully; then rm -rf "${HOME}/.cache/nvim-deps" mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps" touch "${CACHE_MARKER}" diff --git a/ci/common/suite.sh b/ci/common/suite.sh index 46207754fa..e22252c985 100644 --- a/ci/common/suite.sh +++ b/ci/common/suite.sh @@ -2,11 +2,18 @@ NL="$(printf '\nE')" NL="${NL%E}" -FAILED=0 - FAIL_SUMMARY="" +# Test success marker. If END_MARKER file exists, we know that all tests +# finished. If FAIL_SUMMARY_FILE exists we know that some tests failed, this +# file will contain information about failed tests. Build is considered +# successful if tests ended without any of them failing. +END_MARKER="$BUILD_DIR/.tests_finished" +FAIL_SUMMARY_FILE="$BUILD_DIR/.test_errors" + enter_suite() { + FAILED=0 + rm -f "${END_MARKER}" local suite_name="$1" export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name" } @@ -19,17 +26,16 @@ exit_suite() { export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}" if test "x$1" != "x--continue" ; then exit $FAILED + else + local saved_failed=$FAILED + FAILED=0 + return $saved_failed fi } fail() { - local allow_failure= - if test "x$1" = "x--allow-failure" ; then - shift - allow_failure=A - fi local test_name="$1" - local fail_char="$allow_failure$2" + local fail_char="$2" local message="$3" : ${fail_char:=F} @@ -37,10 +43,9 @@ fail() { local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message" FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}" + echo "${full_msg}" >> "${FAIL_SUMMARY_FILE}" echo "Failed: $full_msg" - if test "x$allow_failure" = "x" ; then - FAILED=1 - fi + FAILED=1 } run_test() { @@ -77,14 +82,13 @@ run_test_wd() { while test $restarts -gt 0 ; do : > "${status_file}" ( - FAILED=0 - if ! ( - set -o pipefail - eval "$cmd" 2>&1 | tee -a "$output_file" - ) ; then - fail "${test_name}" "$@" + set -o pipefail + ret=0 + if ! eval "$cmd" 2>&1 | tee -a "$output_file" ; then + ret=1 fi - echo "$FAILED" > "$status_file" + echo "$ret" > "$status_file" + exit $ret ) & local pid=$! while test "$(stat -c "%s" "$status_file")" -eq 0 ; do @@ -116,6 +120,20 @@ run_test_wd() { done } -succeeded() { - return $FAILED +ended_successfully() { + if [[ -f "${FAIL_SUMMARY_FILE}" ]]; then + echo 'Test failed, complete summary:' + cat "${FAIL_SUMMARY_FILE}" + return 1 + fi + if ! [[ -f "${END_MARKER}" ]] ; then + echo 'ended_successfully called before end marker was touched' + return 1 + fi + return 0 +} + +end_tests() { + touch "${END_MARKER}" + ended_successfully } diff --git a/ci/common/test.sh b/ci/common/test.sh index 4936992cfd..d911d9bc18 100644 --- a/ci/common/test.sh +++ b/ci/common/test.sh @@ -1,4 +1,5 @@ source "${CI_DIR}/common/build.sh" +source "${CI_DIR}/common/suite.sh" print_core() { local app="$1" @@ -40,10 +41,9 @@ check_core_dumps() { print_core "$app" "$core" fi done - if test "$app" = quiet ; then - return 0 + if test "$app" != quiet ; then + fail 'cores' E 'Core dumps found' fi - exit 1 } check_logs() { @@ -62,8 +62,7 @@ check_logs() { err=1 done if [[ -n "${err}" ]]; then - echo "Runtime errors detected." - exit 1 + fail 'logs' E 'Runtime errors detected.' fi } @@ -75,50 +74,53 @@ asan_check() { check_logs "${1}" "*san.*" } -run_unittests() { +run_unittests() {( + enter_suite unittests ulimit -c unlimited if ! build_make unittest ; then - check_core_dumps "$(which luajit)" - exit 1 + fail 'unittests' F 'Unit tests failed' fi check_core_dumps "$(which luajit)" -} + exit_suite +)} -run_functionaltests() { +run_functionaltests() {( + enter_suite functionaltests ulimit -c unlimited if ! build_make ${FUNCTIONALTEST}; then - asan_check "${LOG_DIR}" - valgrind_check "${LOG_DIR}" - check_core_dumps - exit 1 + fail 'functionaltests' F 'Functional tests failed' fi asan_check "${LOG_DIR}" valgrind_check "${LOG_DIR}" check_core_dumps -} + exit_suite +)} -run_oldtests() { +run_oldtests() {( + enter_suite oldtests ulimit -c unlimited if ! make -C "${TRAVIS_BUILD_DIR}/src/nvim/testdir"; then reset - asan_check "${LOG_DIR}" - valgrind_check "${LOG_DIR}" - check_core_dumps - exit 1 + fail 'oldtests' F 'Legacy tests failed' fi asan_check "${LOG_DIR}" valgrind_check "${LOG_DIR}" check_core_dumps -} + exit_suite +)} -install_nvim() { - build_make install +install_nvim() {( + enter_suite 'install_nvim' + if ! build_make install ; then + fail 'install' E 'make install failed' + exit_suite + fi "${INSTALL_PREFIX}/bin/nvim" --version "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || { echo "Running ':help' in the installed nvim failed." echo "Maybe the helptags have not been generated properly." - exit 1 + fail 'help' F 'Failed running :help' } local genvimsynf=syntax/vim/generated.vim @@ -127,24 +129,22 @@ install_nvim() { cd runtime ; git ls-files | grep -e '.vim$' -e '.ps$' -e '.dict$' -e '.py$' -e '.tutor$' ) ; do if ! test -e "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then - echo "It appears that $file is not installed." - exit 1 + fail 'runtime-install' F "It appears that $file is not installed." fi done # Check that generated syntax file has function names, #5060. local gpat='syn keyword vimFuncName .*eval' if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf"; then - echo "It appears that $genvimsynf does not contain $gpat." - exit 1 + fail 'funcnames' F "It appears that $genvimsynf does not contain $gpat." fi for file in $( cd runtime ; git ls-files | grep -e '.awk$' -e '.sh$' -e '.bat$' ) ; do if ! test -x "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then - echo "It appears that $file is not installed or is not executable." - exit 1 + fail 'not-exe' F "It appears that $file is not installed or is not executable." fi done -} + exit_suite +)} diff --git a/ci/run_lint.sh b/ci/run_lint.sh index 39a90102e7..2c30615725 100755 --- a/ci/run_lint.sh +++ b/ci/run_lint.sh @@ -25,4 +25,4 @@ CLICOLOR_FORCE=1 run_test_wd \ 'csi_clean' \ single-includes -exit_suite +end_tests diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 92cb5a9fd8..4abc9eea9f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -27,8 +27,4 @@ run_test run_oldtests run_test install_nvim -if succeeded ; then - touch "${SUCCESS_MARKER}" -fi - -exit_suite +end_tests |