15 files changed, 308 insertions, 117 deletions
diff --git a/.travis.yml b/.travis.yml
index a02d44ea47..fe58858e5b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -50,6 +50,7 @@ env:
     - SUCCESS_MARKER="$BUILD_DIR/.tests_successful"
     # default target name for functional tests
     - FUNCTIONALTEST=functionaltest
+    - CI_TARGET=tests
 
 matrix:
   include:
diff --git a/Makefile b/Makefile
index d2f6c11b19..0c057f5656 100644
--- a/Makefile
+++ b/Makefile
@@ -126,11 +126,11 @@ distclean: clean
 install: | nvim
 	+$(BUILD_CMD) -C build install
 
-clint:
-	$(CMAKE_PRG) -DLINT_PRG=./src/clint.py \
-		-DLINT_DIR=src \
-		-DLINT_SUPPRESS_URL="$(DOC_DOWNLOAD_URL_BASE)$(CLINT_ERRORS_FILE_PATH)" \
-		-P cmake/RunLint.cmake
+clint: build/.ran-cmake
+	+$(BUILD_CMD) -C build clint
+
+clint-full: build/.ran-cmake
+	+$(BUILD_CMD) -C build clint-full
 
 check-single-includes: build/.ran-cmake
 	+$(BUILD_CMD) -C build check-single-includes
diff --git a/ci/before_install.sh b/ci/before_install.sh
index 9aac37de12..5b36adaef2 100755
--- a/ci/before_install.sh
+++ b/ci/before_install.sh
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
+if [[ "${CI_TARGET}" == lint ]]; then
   exit
 fi
 
diff --git a/ci/before_script.sh b/ci/before_script.sh
index 4a75e89fbe..445996a8df 100755
--- a/ci/before_script.sh
+++ b/ci/before_script.sh
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
+if [[ "${CI_TARGET}" == lint ]]; then
   exit
 fi
 
diff --git a/ci/common/build.sh b/ci/common/build.sh
index 44087513ee..129622b522 100644
--- a/ci/common/build.sh
+++ b/ci/common/build.sh
@@ -1,3 +1,11 @@
+top_make() {
+  ${MAKE_CMD} "$@"
+}
+
+build_make() {
+  top_make -C "${BUILD_DIR}" "$@"
+}
+
 build_deps() {
   if [[ "${BUILD_32BIT}" == ON ]]; then
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
@@ -30,7 +38,7 @@ build_deps() {
   echo "Configuring with '${DEPS_CMAKE_FLAGS}'."
   CC= cmake ${DEPS_CMAKE_FLAGS} "${TRAVIS_BUILD_DIR}/third-party/"
 
-  if ! ${MAKE_CMD}; then
+  if ! top_make; then
     exit 1
   fi
 
@@ -53,18 +61,18 @@ prepare_build() {
 
 build_nvim() {
   echo "Building nvim."
-  if ! ${MAKE_CMD} nvim; then
+  if ! top_make nvim; then
     exit 1
   fi
 
   if [ "$CLANG_SANITIZER" != "TSAN" ]; then
     echo "Building libnvim."
-    if ! ${MAKE_CMD} libnvim; then
+    if ! top_make libnvim; then
       exit 1
     fi
 
     echo "Building nvim-test."
-    if ! ${MAKE_CMD} nvim-test; then
+    if ! top_make nvim-test; then
       exit 1
     fi
   fi
diff --git a/ci/common/suite.sh b/ci/common/suite.sh
new file mode 100644
index 0000000000..a70cafc92c
--- /dev/null
+++ b/ci/common/suite.sh
@@ -0,0 +1,107 @@
+# HACK: get newline for use in strings given that "\n" and $'' do not work.
+NL="$(printf '\nE')"
+NL="${NL%E}"
+
+FAILED=0
+
+FAIL_SUMMARY=""
+
+enter_suite() {
+  local suite_name="$1"
+  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name"
+}
+
+exit_suite() {
+  if test $FAILED -ne 0 ; then
+    echo "Suite ${NVIM_TEST_CURRENT_SUITE} failed, summary:"
+    echo "${FAIL_SUMMARY}"
+  fi
+  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}"
+  if test "x$1" != "x--continue" ; then
+    exit $FAILED
+  fi
+}
+
+fail() {
+  local allow_failure=
+  if test "x$1" = "x--allow-failure" ; then
+    shift
+    allow_failure=A
+  fi
+  local test_name="$1"
+  local fail_char="$allow_failure$2"
+  local message="$3"
+
+  : ${fail_char:=F}
+  : ${message:=Test $test_name failed}
+
+  local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message"
+  FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}"
+  echo "Failed: $full_msg"
+  if test "x$allow_failure" = "x" ; then
+    FAILED=1
+  fi
+}
+
+run_test() {
+  local cmd="$1"
+  test $# -gt 0 && shift
+  local test_name="$1"
+  : ${test_name:=$cmd}
+  test $# -gt 0 && shift
+  if ! eval "$cmd" ; then
+    fail "${test_name}" "$@"
+  fi
+}
+
+run_test_wd() {
+  local timeout="$1"
+  test $# -gt 0 && shift
+  local cmd="$1"
+  test $# -gt 0 && shift
+  local test_name="$1"
+  : ${test_name:=$cmd}
+  test $# -gt 0 && shift
+  local output_file="$(mktemp)"
+  local status_file="$(mktemp)"
+  local restarts=5
+  local prev_tmpsize=-1
+  while test $restarts -gt 0 ; do
+    : > "${status_file}"
+    (
+      if ! (
+        set -o pipefail
+        eval "$cmd" 2>&1 | tee -a "$output_file"
+      ) ; then
+        fail "${test_name}" "$@"
+      fi
+      echo "$FAILED" > "$status_file"
+    ) &
+    local pid=$!
+    while test "$(stat -c "%s" "$status_file")" -eq 0 ; do
+      prev_tmpsize=$tmpsize
+      sleep $timeout
+      tmpsize="$(stat -c "%s" "$output_file")"
+      if test $tempsize -eq $prev_temsize ; then
+        # no output, assuming either hang or exit
+        break
+      fi
+    done
+    if test "$(stat -c "%s" "$status_file")" -eq 0 ; then
+      # status file not updated, assuming hang
+      kill -KILL $pid
+      echo "Test ${test_name} hang up, restarting"
+    else
+      local new_failed="$(cat "$status_file")"
+      if test "x$new_failed" != "x0" ; then
+        fail "${test_name}" F "Test failed in run_test_wd"
+      fi
+      return 0
+    fi
+    restarts=$[ restarts - 1 ]
+  done
+}
+
+succeeded() {
+  return $FAILED
+}
diff --git a/ci/common/test.sh b/ci/common/test.sh
index b28e46a4df..4936992cfd 100644
--- a/ci/common/test.sh
+++ b/ci/common/test.sh
@@ -1,3 +1,5 @@
+source "${CI_DIR}/common/build.sh"
+
 print_core() {
   local app="$1"
   local core="$2"
@@ -75,7 +77,7 @@ asan_check() {
 
 run_unittests() {
   ulimit -c unlimited
-  if ! ${MAKE_CMD} -C "${BUILD_DIR}" unittest ; then
+  if ! build_make unittest ; then
     check_core_dumps "$(which luajit)"
     exit 1
   fi
@@ -84,7 +86,7 @@ run_unittests() {
 
 run_functionaltests() {
   ulimit -c unlimited
-  if ! ${MAKE_CMD} -C "${BUILD_DIR}" ${FUNCTIONALTEST}; then
+  if ! build_make ${FUNCTIONALTEST}; then
     asan_check "${LOG_DIR}"
     valgrind_check "${LOG_DIR}"
     check_core_dumps
@@ -110,7 +112,7 @@ run_oldtests() {
 }
 
 install_nvim() {
-  ${MAKE_CMD} -C "${BUILD_DIR}" install
+  build_make install
 
   "${INSTALL_PREFIX}/bin/nvim" --version
   "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || {
diff --git a/ci/install.sh b/ci/install.sh
index 98d3dc01cb..4ee99e1e44 100755
--- a/ci/install.sh
+++ b/ci/install.sh
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
+if [[ "${CI_TARGET}" == lint ]]; then
   exit
 fi
 
diff --git a/ci/run_lint.sh b/ci/run_lint.sh
new file mode 100755
index 0000000000..5122ffc2b6
--- /dev/null
+++ b/ci/run_lint.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${CI_DIR}/common/build.sh"
+source "${CI_DIR}/common/suite.sh"
+
+enter_suite 'lint'
+
+set -x
+
+run_test 'top_make clint-full' clint
+run_test 'top_make testlint' testlint
+run_test_wd 5s 'top_make check-single-includes' single-includes
+
+exit_suite
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index 6347ac15d4..92cb5a9fd8 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -6,6 +6,11 @@ set -o pipefail
 CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/test.sh"
+source "${CI_DIR}/common/suite.sh"
+
+set -x
+
+enter_suite tests
 
 check_core_dumps --delete quiet
 
@@ -15,11 +20,15 @@ build_nvim
 if [ "$CLANG_SANITIZER" != "TSAN" ]; then
   # Additional threads are only created when the builtin UI starts, which
   # doesn't happen in the unit/functional tests
-  run_unittests
-  run_functionaltests
+  run_test run_unittests
+  run_test run_functionaltests
 fi
-run_oldtests
+run_test run_oldtests
 
-install_nvim
+run_test install_nvim
+
+if succeeded ; then
+  touch "${SUCCESS_MARKER}"
+fi
 
-touch "${SUCCESS_MARKER}"
+exit_suite
diff --git a/ci/script.sh b/ci/script.sh
index 79a1bec201..a59c40cd2d 100755
--- a/ci/script.sh
+++ b/ci/script.sh
@@ -3,16 +3,11 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
-  make "${CI_TARGET}"
-  exit 0
-fi
-
 # This will pass the environment variables down to a bash process which runs
 # as $USER, while retaining the environment variables defined and belonging
 # to secondary groups given above in usermod.
 if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
-  sudo -E su "${USER}" -c "ci/run_tests.sh"
+  sudo -E su "${USER}" -c "ci/run_${CI_TARGET}.sh"
 else
-  ci/run_tests.sh
+  ci/run_${CI_TARGET}.sh
 fi
diff --git a/cmake/Download.cmake b/cmake/Download.cmake
new file mode 100644
index 0000000000..50a77816bc
--- /dev/null
+++ b/cmake/Download.cmake
@@ -0,0 +1,18 @@
+file(
+  DOWNLOAD "${URL}" "${FILE}"
+  STATUS status
+  LOG log
+)
+
+list(GET status 0 status_code)
+list(GET status 1 status_string)
+
+if(NOT status_code EQUAL 0)
+  if(NOT ALLOW_FAILURE)
+      message(FATAL_ERROR "error: downloading '${URL}' failed
+        status_code: ${status_code}
+        status_string: ${status_string}
+        log: ${log}
+      ")
+  endif()
+endif()
diff --git a/cmake/RunLint.cmake b/cmake/RunLint.cmake
deleted file mode 100644
index 306e938232..0000000000
--- a/cmake/RunLint.cmake
+++ /dev/null
@@ -1,32 +0,0 @@
-get_filename_component(LINT_DIR ${LINT_DIR} ABSOLUTE)
-get_filename_component(LINT_PREFIX ${LINT_DIR} PATH)
-set(LINT_SUPPRESS_FILE "${LINT_PREFIX}/errors.json")
-
-if(DEFINED ENV{LINT_FILE})
-  file(GLOB_RECURSE LINT_FILES "$ENV{LINT_FILE}")
-else()
-  file(GLOB_RECURSE LINT_FILES ${LINT_DIR}/*.c ${LINT_DIR}/*.h)
-endif()
-
-set(LINT_ARGS)
-
-if(LINT_SUPPRESS_URL)
-  file(DOWNLOAD ${LINT_SUPPRESS_URL} ${LINT_SUPPRESS_FILE})
-  list(APPEND LINT_ARGS "--suppress-errors=${LINT_SUPPRESS_FILE}")
-endif()
-
-foreach(lint_file ${LINT_FILES})
-  file(RELATIVE_PATH lint_file "${LINT_PREFIX}" "${lint_file}")
-  list(APPEND LINT_ARGS "${lint_file}")
-endforeach()
-
-execute_process(
-  COMMAND ${LINT_PRG} ${LINT_ARGS}
-  RESULT_VARIABLE res
-  WORKING_DIRECTORY "${LINT_PREFIX}")
-
-file(REMOVE ${LINT_SUPPRESS_FILE})
-
-if(NOT res EQUAL 0)
-  message(FATAL_ERROR "Linting failed: ${res}.")
-endif()
diff --git a/src/clint.py b/src/clint.py
index 61c53d128e..5174521fb8 100755
--- a/src/clint.py
+++ b/src/clint.py
@@ -571,7 +571,8 @@ class _CppLintState(object):
         for category, count in self.errors_by_category.items():
             sys.stderr.write('Category \'%s\' errors found: %d\n' %
                              (category, count))
-        sys.stderr.write('Total errors found: %d\n' % self.error_count)
+        if self.error_count:
+            sys.stderr.write('Total errors found: %d\n' % self.error_count)
 
     def SuppressErrorsFrom(self, fname):
         """Open file and read a list of suppressed errors from it"""
diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index a3bacaa9d2..5a5ebc4415 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -10,6 +10,7 @@ if(USE_GCOV)
 endif()
 endif()
 
+set(TOUCHES_DIR ${PROJECT_BINARY_DIR}/touches)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
 set(DISPATCH_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendispatch.lua)
 file(GLOB API_HEADERS api/*.h)
@@ -34,11 +35,19 @@ set(UNICODE_TABLES_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genunicodetables.lua)
 set(UNICODE_DIR ${PROJECT_SOURCE_DIR}/unicode)
 file(GLOB UNICODE_FILES ${UNICODE_DIR}/*.txt)
 set(GENERATED_UNICODE_TABLES ${GENERATED_DIR}/unicode_tables.generated.h)
+set(LINT_SUPPRESS_FILE ${PROJECT_BINARY_DIR}/errors.json)
+set(LINT_SUPPRESS_URL_BASE "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint")
+set(LINT_SUPPRESS_URL "${LINT_SUPPRESS_URL_BASE}/errors.json")
+set(LINT_PRG ${PROJECT_SOURCE_DIR}/src/clint.py)
+set(DOWNLOAD_SCRIPT ${PROJECT_SOURCE_DIR}/cmake/Download.cmake)
+set(LINT_SUPPRESSES_ROOT ${PROJECT_BINARY_DIR}/errors)
+set(LINT_SUPPRESSES_URL "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint/errors.tar.gz")
 
 include_directories(${GENERATED_DIR})
 include_directories(${CACHED_GENERATED_DIR})
 include_directories(${GENERATED_INCLUDES_DIR})
 
+file(MAKE_DIRECTORY ${TOUCHES_DIR})
 file(MAKE_DIRECTORY ${GENERATED_DIR})
 file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR})
 
@@ -73,6 +82,8 @@ file(GLOB UNIT_TEST_FIXTURES ${PROJECT_SOURCE_DIR}/test/unit/fixtures/*.c)
 list(SORT NVIM_SOURCES)
 list(SORT NVIM_HEADERS)
 
+list(APPEND LINT_NVIM_SOURCES ${NVIM_SOURCES} ${NVIM_HEADERS})
+
 foreach(sfile ${NVIM_SOURCES})
   get_filename_component(f ${sfile} NAME)
   if(${f} MATCHES "^(regexp_nfa.c)$")
@@ -393,76 +404,86 @@ add_library(nvim-test MODULE EXCLUDE_FROM_ALL ${NVIM_GENERATED_FOR_SOURCES} ${NV
 target_link_libraries(nvim-test ${NVIM_LINK_LIBRARIES})
 set_property(TARGET nvim-test APPEND_STRING PROPERTY COMPILE_FLAGS -DUNIT_TESTING)
 
-set(NO_SINGLE_CHECK_HEADERS
-  cursor_shape
-  diff
-  digraph
-  ex_cmds
-  ex_getln
-  file_search
-  fold
-  getchar
-  hardcopy
-  if_cscope
-  if_cscope_defs
-  mark
-  mbyte
-  memfile_defs
-  memline
-  memline_defs
-  menu
-  misc2
-  move
-  msgpack_rpc/server
-  ops
-  option
-  os/shell
-  os_unix
-  os/win_defs
-  popupmnu
-  quickfix
-  regexp
-  regexp_defs
-  screen
-  search
-  sha256
-  sign_defs
-  spell
-  spellfile
-  syntax
-  syntax_defs
-  tag
-  terminal
-  tui/tui
-  ugrid
-  ui
-  ui_bridge
-  undo
-  undo_defs
-  version
-  window
-)
-foreach(hfile ${NVIM_HEADERS})
-  get_filename_component(full_d ${hfile} PATH)
+function(get_test_target prefix sfile relative_path_var target_var)
+  get_filename_component(full_d "${sfile}" PATH)
   file(RELATIVE_PATH d "${PROJECT_SOURCE_DIR}/src/nvim" "${full_d}")
-  if(${d} MATCHES "^[.][.]")
+  if(d MATCHES "^[.][.]")
     file(RELATIVE_PATH d "${GENERATED_DIR}" "${full_d}")
   endif()
-  get_filename_component(r ${hfile} NAME_WE)
-  if(NOT ${d} EQUAL ".")
+  get_filename_component(r "${sfile}" NAME)
+  if(NOT d MATCHES "^[.]?$")
     set(r "${d}/${r}")
   endif()
+  string(REGEX REPLACE "[/.]" "-" suffix "${r}")
+  set(${relative_path_var} ${r} PARENT_SCOPE)
+  if(prefix STREQUAL "")
+    set(${target_var} "${suffix}" PARENT_SCOPE)
+  else()
+    set(${target_var} "${prefix}-${suffix}" PARENT_SCOPE)
+  endif()
+endfunction()
+
+set(NO_SINGLE_CHECK_HEADERS
+  cursor_shape.h
+  diff.h
+  digraph.h
+  ex_cmds.h
+  ex_getln.h
+  file_search.h
+  fold.h
+  getchar.h
+  hardcopy.h
+  if_cscope.h
+  if_cscope_defs.h
+  mark.h
+  mbyte.h
+  memfile_defs.h
+  memline.h
+  memline_defs.h
+  menu.h
+  misc2.h
+  move.h
+  msgpack_rpc/server.h
+  ops.h
+  option.h
+  os/shell.h
+  os_unix.h
+  os/win_defs.h
+  popupmnu.h
+  quickfix.h
+  regexp.h
+  regexp_defs.h
+  screen.h
+  search.h
+  sha256.h
+  sign_defs.h
+  spell.h
+  spellfile.h
+  syntax.h
+  syntax_defs.h
+  tag.h
+  terminal.h
+  tui/tui.h
+  ugrid.h
+  ui.h
+  ui_bridge.h
+  undo.h
+  undo_defs.h
+  version.h
+  window.h
+)
+foreach(hfile ${NVIM_HEADERS})
+  get_test_target(test-includes "${hfile}" relative_path texe)
 
   if(NOT ${hfile} MATCHES "[.]c[.]h$")
-    set(tsource "${GENERATED_DIR}/${r}.test-include.c")
-    string(REPLACE "/" "-" texe "test-incl-${r}")
+    set(tsource "${GENERATED_DIR}/${relative_path}.test-include.c")
     write_file("${tsource}" "#include \"${hfile}\"\nint main(int argc, char **argv) { return 0; }")
     add_executable(
       ${texe}
       EXCLUDE_FROM_ALL
       ${tsource} ${NVIM_HEADERS} ${NVIM_GENERATED_FOR_HEADERS})
 
-    list(FIND NO_SINGLE_CHECK_HEADERS "${r}" hfile_exclude_idx)
+    list(FIND NO_SINGLE_CHECK_HEADERS "${relative_path}" hfile_exclude_idx)
     if(${hfile_exclude_idx} EQUAL -1)
       list(APPEND HEADER_CHECK_TARGETS ${texe})
     endif()
@@ -470,4 +491,46 @@ foreach(hfile ${NVIM_HEADERS})
 endforeach()
 add_custom_target(check-single-includes DEPENDS ${HEADER_CHECK_TARGETS})
 
+function(add_download output url allow_failure)
+  add_custom_command(
+    OUTPUT "${output}"
+    COMMAND
+      ${CMAKE_COMMAND}
+        -DURL=${url} -DFILE=${output}
+        -DALLOW_FAILURE=${allow_failure}
+        -P ${DOWNLOAD_SCRIPT}
+    DEPENDS ${DOWNLOAD_SCRIPT}
+  )
+endfunction()
+
+add_download(${LINT_SUPPRESS_FILE} ${LINT_SUPPRESS_URL} off)
+
+set(LINT_NVIM_REL_SOURCES)
+foreach(sfile ${LINT_NVIM_SOURCES})
+  get_test_target("" "${sfile}" r suffix)
+  set(suppress_file ${LINT_SUPPRESSES_ROOT}/${suffix}.json)
+  set(suppress_url "${LINT_SUPPRESS_URL_BASE}/${suffix}.json")
+  set(rsfile src/nvim/${r})
+  add_download(${suppress_file} ${suppress_url} on)
+  set(touch_file "${TOUCHES_DIR}/ran-clint-${suffix}")
+  add_custom_command(
+    OUTPUT ${touch_file}
+    COMMAND ${LINT_PRG} --suppress-errors=${suppress_file} ${rsfile}
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+    COMMAND ${CMAKE_COMMAND} -E touch ${touch_file}
+    DEPENDS ${LINT_PRG} ${sfile} ${suppress_file}
+  )
+  list(APPEND LINT_TARGETS ${touch_file})
+  list(APPEND LINT_NVIM_REL_SOURCES ${rsfile})
+endforeach()
+add_custom_target(clint DEPENDS ${LINT_TARGETS})
+
+add_custom_target(
+  clint-full
+  COMMAND
+    ${LINT_PRG} --suppress-errors=${LINT_SUPPRESS_FILE} ${LINT_NVIM_REL_SOURCES}
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+  DEPENDS ${LINT_PRG} ${LINT_NVIM_SOURCES} ${LINT_SUPPRESS_FILE}
+)
+
 add_subdirectory(po)