diff options
Diffstat (limited to 'ci/common')
| -rw-r--r-- | ci/common/suite.sh | 68 | ||||
| -rw-r--r-- | ci/common/test.sh | 62 |
2 files changed, 78 insertions, 52 deletions
diff --git a/ci/common/suite.sh b/ci/common/suite.sh index 46207754fa..568d5d5bee 100644 --- a/ci/common/suite.sh +++ b/ci/common/suite.sh @@ -2,11 +2,18 @@ NL="$(printf '\nE')" NL="${NL%E}" -FAILED=0 - FAIL_SUMMARY="" +# Test success marker. If END_MARKER file exists, we know that all tests +# finished. If FAIL_SUMMARY_FILE exists we know that some tests failed, this +# file will contain information about failed tests. Build is considered +# successful if tests ended without any of them failing. +END_MARKER="$BUILD_DIR/.tests_finished" +FAIL_SUMMARY_FILE="$BUILD_DIR/.test_errors" + enter_suite() { + FAILED=0 + rm -f "${END_MARKER}" local suite_name="$1" export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name" } @@ -19,17 +26,16 @@ exit_suite() { export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}" if test "x$1" != "x--continue" ; then exit $FAILED + else + local saved_failed=$FAILED + FAILED=0 + return $saved_failed fi } fail() { - local allow_failure= - if test "x$1" = "x--allow-failure" ; then - shift - allow_failure=A - fi local test_name="$1" - local fail_char="$allow_failure$2" + local fail_char="$2" local message="$3" : ${fail_char:=F} @@ -37,10 +43,9 @@ fail() { local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message" FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}" + echo "${full_msg}" >> "${FAIL_SUMMARY_FILE}" echo "Failed: $full_msg" - if test "x$allow_failure" = "x" ; then - FAILED=1 - fi + FAILED=1 } run_test() { @@ -55,6 +60,12 @@ run_test() { } run_test_wd() { + local hang_ok= + if test "x$1" = "x--allow-hang" ; then + hang_ok=1 + shift + fi + local timeout="$1" test $# -gt 0 && shift @@ -77,14 +88,13 @@ run_test_wd() { while test $restarts -gt 0 ; do : > "${status_file}" ( - FAILED=0 - if ! ( - set -o pipefail - eval "$cmd" 2>&1 | tee -a "$output_file" - ) ; then - fail "${test_name}" "$@" + set -o pipefail + ret=0 + if ! eval "$cmd" 2>&1 | tee -a "$output_file" ; then + ret=1 fi - echo "$FAILED" > "$status_file" + echo "$ret" > "$status_file" + exit $ret ) & local pid=$! while test "$(stat -c "%s" "$status_file")" -eq 0 ; do @@ -101,7 +111,9 @@ run_test_wd() { # status file not updated, assuming hang kill -KILL $pid if test $restarts -eq 0 ; then - fail "${test_name}" E "Test hang up" + if test "x$hang_ok" = "x" ; then + fail "${test_name}" E "Test hang up" + fi else echo "Test ${test_name} hang up, restarting" eval "$restart_cmd" @@ -116,6 +128,20 @@ run_test_wd() { done } -succeeded() { - return $FAILED +ended_successfully() { + if [[ -f "${FAIL_SUMMARY_FILE}" ]]; then + echo 'Test failed, complete summary:' + cat "${FAIL_SUMMARY_FILE}" + return 1 + fi + if ! [[ -f "${END_MARKER}" ]] ; then + echo 'ended_successfully called before end marker was touched' + return 1 + fi + return 0 +} + +end_tests() { + touch "${END_MARKER}" + ended_successfully } diff --git a/ci/common/test.sh b/ci/common/test.sh index 4936992cfd..d911d9bc18 100644 --- a/ci/common/test.sh +++ b/ci/common/test.sh @@ -1,4 +1,5 @@ source "${CI_DIR}/common/build.sh" +source "${CI_DIR}/common/suite.sh" print_core() { local app="$1" @@ -40,10 +41,9 @@ check_core_dumps() { print_core "$app" "$core" fi done - if test "$app" = quiet ; then - return 0 + if test "$app" != quiet ; then + fail 'cores' E 'Core dumps found' fi - exit 1 } check_logs() { @@ -62,8 +62,7 @@ check_logs() { err=1 done if [[ -n "${err}" ]]; then - echo "Runtime errors detected." - exit 1 + fail 'logs' E 'Runtime errors detected.' fi } @@ -75,50 +74,53 @@ asan_check() { check_logs "${1}" "*san.*" } -run_unittests() { +run_unittests() {( + enter_suite unittests ulimit -c unlimited if ! build_make unittest ; then - check_core_dumps "$(which luajit)" - exit 1 + fail 'unittests' F 'Unit tests failed' fi check_core_dumps "$(which luajit)" -} + exit_suite +)} -run_functionaltests() { +run_functionaltests() {( + enter_suite functionaltests ulimit -c unlimited if ! build_make ${FUNCTIONALTEST}; then - asan_check "${LOG_DIR}" - valgrind_check "${LOG_DIR}" - check_core_dumps - exit 1 + fail 'functionaltests' F 'Functional tests failed' fi asan_check "${LOG_DIR}" valgrind_check "${LOG_DIR}" check_core_dumps -} + exit_suite +)} -run_oldtests() { +run_oldtests() {( + enter_suite oldtests ulimit -c unlimited if ! make -C "${TRAVIS_BUILD_DIR}/src/nvim/testdir"; then reset - asan_check "${LOG_DIR}" - valgrind_check "${LOG_DIR}" - check_core_dumps - exit 1 + fail 'oldtests' F 'Legacy tests failed' fi asan_check "${LOG_DIR}" valgrind_check "${LOG_DIR}" check_core_dumps -} + exit_suite +)} -install_nvim() { - build_make install +install_nvim() {( + enter_suite 'install_nvim' + if ! build_make install ; then + fail 'install' E 'make install failed' + exit_suite + fi "${INSTALL_PREFIX}/bin/nvim" --version "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || { echo "Running ':help' in the installed nvim failed." echo "Maybe the helptags have not been generated properly." - exit 1 + fail 'help' F 'Failed running :help' } local genvimsynf=syntax/vim/generated.vim @@ -127,24 +129,22 @@ install_nvim() { cd runtime ; git ls-files | grep -e '.vim$' -e '.ps$' -e '.dict$' -e '.py$' -e '.tutor$' ) ; do if ! test -e "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then - echo "It appears that $file is not installed." - exit 1 + fail 'runtime-install' F "It appears that $file is not installed." fi done # Check that generated syntax file has function names, #5060. local gpat='syn keyword vimFuncName .*eval' if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf"; then - echo "It appears that $genvimsynf does not contain $gpat." - exit 1 + fail 'funcnames' F "It appears that $genvimsynf does not contain $gpat." fi for file in $( cd runtime ; git ls-files | grep -e '.awk$' -e '.sh$' -e '.bat$' ) ; do if ! test -x "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then - echo "It appears that $file is not installed or is not executable." - exit 1 + fail 'not-exe' F "It appears that $file is not installed or is not executable." fi done -} + exit_suite +)} |