ci: Do not hide ci directory (#6410)

10 files changed, 432 insertions, 0 deletions

diff --git a/ci/after_success.sh b/ci/after_success.sh
new file mode 100755
index 0000000000..0215eb139b
--- /dev/null
+++ b/ci/after_success.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ -n "${GCOV}" ]]; then
+  coveralls --gcov "$(which "${GCOV}")" --encoding iso-8859-1 || echo 'coveralls upload failed.'
+fi
diff --git a/ci/before_cache.sh b/ci/before_cache.sh
new file mode 100755
index 0000000000..dd1fcf2bf7
--- /dev/null
+++ b/ci/before_cache.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+# Don't cache pip's log and selfcheck.
+rm -rf "${HOME}/.cache/pip/log"
+rm -f "${HOME}/.cache/pip/selfcheck.json"
+
+# Update the third-party dependency cache only if the build was successful.
+if [[ -f "${SUCCESS_MARKER}" ]]; then
+  rm -rf "${HOME}/.cache/nvim-deps"
+  mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps"
+  touch "${CACHE_MARKER}"
+  echo "Updated third-party dependencies (timestamp: $(stat -c '%y' "${CACHE_MARKER}"))."
+fi
diff --git a/ci/before_install.sh b/ci/before_install.sh
new file mode 100755
index 0000000000..9aac37de12
--- /dev/null
+++ b/ci/before_install.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ -n "${CI_TARGET}" ]]; then
+  exit
+fi
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  brew update
+fi
+
+echo "Upgrade Python 2 pip."
+pip2.7 -q install --user --upgrade pip
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  echo "Install Python 3."
+  brew install python3
+  echo "Upgrade Python 3 pip."
+  pip3 -q install --user --upgrade pip
+else
+  echo "Upgrade Python 3 pip."
+  pip3 -q install --user --upgrade pip
+fi
diff --git a/ci/before_script.sh b/ci/before_script.sh
new file mode 100755
index 0000000000..4a75e89fbe
--- /dev/null
+++ b/ci/before_script.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ -n "${CI_TARGET}" ]]; then
+  exit
+fi
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CI_DIR}/common/build.sh"
+
+# Test some of the configuration variables.
+if [[ -n "${GCOV}" ]] && [[ ! $(type -P "${GCOV}") ]]; then
+  echo "\$GCOV: '${GCOV}' is not executable."
+  exit 1
+fi
+if [[ -n "${LLVM_SYMBOLIZER}" ]] && [[ ! $(type -P "${LLVM_SYMBOLIZER}") ]]; then
+  echo "\$LLVM_SYMBOLIZER: '${LLVM_SYMBOLIZER}' is not executable."
+  exit 1
+fi
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  # Adds user to a dummy group.
+  # That allows to test changing the group of the file by `os_fchown`.
+  sudo dscl . -create /Groups/chown_test
+  sudo dscl . -append /Groups/chown_test GroupMembership "${USER}"
+fi
+
+# Compile dependencies.
+build_deps
+
+rm -rf "${LOG_DIR}"
+mkdir -p "${LOG_DIR}"
diff --git a/ci/build.bat b/ci/build.bat
new file mode 100644
index 0000000000..87a171b994
--- /dev/null
+++ b/ci/build.bat
@@ -0,0 +1,54 @@
+:: These are native MinGW builds, but they use the toolchain inside
+:: MSYS2, this allows using all the dependencies and tools available
+:: in MSYS2, but we cannot build inside the MSYS2 shell.
+echo on
+if "%CONFIGURATION%" == "MINGW_32" (
+  set ARCH=i686
+  set BITS=32
+) else (
+  set ARCH=x86_64
+  set BITS=64
+)
+:: We cannot have sh.exe in the PATH (MinGW)
+set PATH=%PATH:C:\Program Files\Git\usr\bin;=%
+set PATH=C:\msys64\mingw%BITS%\bin;C:\Windows\System32;C:\Windows;%PATH%
+:: The default cpack in the PATH is not CMake
+set PATH=C:\Program Files (x86)\CMake\bin\cpack.exe;%PATH%
+
+:: Build third-party dependencies
+C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm -Su" || goto :error
+C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm --needed -S mingw-w64-%ARCH%-cmake mingw-w64-%ARCH%-perl mingw-w64-%ARCH%-diffutils gperf" || goto :error
+
+:: Setup python (use AppVeyor system python)
+C:\Python27\python.exe -m pip install neovim || goto :error
+C:\Python35\python.exe -m pip install neovim || goto :error
+:: Disambiguate python3
+move c:\Python35\python.exe c:\Python35\python3.exe
+set PATH=C:\Python35;C:\Python27;%PATH%
+:: Sanity check
+python  -c "import neovim; print(str(neovim))" || goto :error
+python3 -c "import neovim; print(str(neovim))" || goto :error
+
+mkdir .deps
+cd .deps
+cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release ..\third-party\ || goto :error
+mingw32-make VERBOSE=1 || goto :error
+cd ..
+
+:: Build Neovim
+mkdir build
+cd build
+cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUSTED_OUTPUT_TYPE=gtest -DGPERF_PRG="C:\msys64\usr\bin\gperf.exe" .. || goto :error
+mingw32-make VERBOSE=1 || goto :error
+bin\nvim --version || goto :error
+
+:: Functional tests
+mingw32-make functionaltest VERBOSE=1 || goto :error
+
+:: Build artifacts
+cpack -G ZIP -C Release
+if defined APPVEYOR_REPO_TAG_NAME cpack -G NSIS -C Release
+
+goto :EOF
+:error
+exit /b %errorlevel%
diff --git a/ci/common/build.sh b/ci/common/build.sh
new file mode 100644
index 0000000000..44087513ee
--- /dev/null
+++ b/ci/common/build.sh
@@ -0,0 +1,81 @@
+build_deps() {
+  if [[ "${BUILD_32BIT}" == ON ]]; then
+    DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
+  fi
+  if [[ "${FUNCTIONALTEST}" == "functionaltest-lua" ]]; then
+    DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} -DUSE_BUNDLED_LUA=ON"
+  fi
+
+  rm -rf "${DEPS_BUILD_DIR}"
+
+  # If there is a valid cache and we're not forced to recompile,
+  # use cached third-party dependencies.
+  if [[ -f "${CACHE_MARKER}" ]] && [[ "${BUILD_NVIM_DEPS}" != true ]]; then
+    if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+      local statcmd="stat -f '%Sm'"
+    else
+      local statcmd="stat -c '%y'"
+    fi
+    echo "Using third-party dependencies from Travis's cache (last updated: $(${statcmd} "${CACHE_MARKER}"))."
+
+     mkdir -p "$(dirname "${DEPS_BUILD_DIR}")"
+     mv "${HOME}/.cache/nvim-deps" "${DEPS_BUILD_DIR}"
+  else
+    mkdir -p "${DEPS_BUILD_DIR}"
+  fi
+
+  # Even if we're using cached dependencies, run CMake and make to
+  # update CMake configuration and update to newer deps versions.
+  cd "${DEPS_BUILD_DIR}"
+  echo "Configuring with '${DEPS_CMAKE_FLAGS}'."
+  CC= cmake ${DEPS_CMAKE_FLAGS} "${TRAVIS_BUILD_DIR}/third-party/"
+
+  if ! ${MAKE_CMD}; then
+    exit 1
+  fi
+
+  cd "${TRAVIS_BUILD_DIR}"
+}
+
+prepare_build() {
+  if [[ -n "${CLANG_SANITIZER}" ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DCLANG_${CLANG_SANITIZER}=ON"
+  fi
+  if [[ "${BUILD_32BIT}" == ON ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
+  fi
+
+  mkdir -p "${BUILD_DIR}"
+  cd "${BUILD_DIR}"
+  echo "Configuring with '${CMAKE_FLAGS} $@'."
+  cmake ${CMAKE_FLAGS} "$@" "${TRAVIS_BUILD_DIR}"
+}
+
+build_nvim() {
+  echo "Building nvim."
+  if ! ${MAKE_CMD} nvim; then
+    exit 1
+  fi
+
+  if [ "$CLANG_SANITIZER" != "TSAN" ]; then
+    echo "Building libnvim."
+    if ! ${MAKE_CMD} libnvim; then
+      exit 1
+    fi
+
+    echo "Building nvim-test."
+    if ! ${MAKE_CMD} nvim-test; then
+      exit 1
+    fi
+  fi
+
+  # Invoke nvim to trigger *San early.
+  if ! (bin/nvim --version && bin/nvim -u NONE -e -c ':qall'); then
+    asan_check "${LOG_DIR}"
+    exit 1
+  fi
+  asan_check "${LOG_DIR}"
+
+
+  cd "${TRAVIS_BUILD_DIR}"
+}
diff --git a/ci/common/test.sh b/ci/common/test.sh
new file mode 100644
index 0000000000..b28e46a4df
--- /dev/null
+++ b/ci/common/test.sh
@@ -0,0 +1,148 @@
+print_core() {
+  local app="$1"
+  local core="$2"
+  if test "$app" = quiet ; then
+    echo "Found core $core"
+    return 0
+  fi
+  echo "======= Core file $core ======="
+  if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+    lldb -Q -o "bt all" -f "${app}" -c "${core}"
+  else
+    gdb -n -batch -ex 'thread apply all bt full' "${app}" -c "${core}"
+  fi
+}
+
+check_core_dumps() {
+  local del=
+  if test "$1" = "--delete" ; then
+    del=1
+    shift
+  fi
+  local app="${1:-${BUILD_DIR}/bin/nvim}"
+  if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+    local cores="$(find /cores/ -type f -print)"
+  else
+    local cores="$(find ./ -type f -name 'core.*' -print)"
+  fi
+
+  if [ -z "${cores}" ]; then
+    return
+  fi
+  local core
+  for core in $cores; do
+    if test "$del" = "1" ; then
+      print_core "$app" "$core" >&2
+      rm "$core"
+    else
+      print_core "$app" "$core"
+    fi
+  done
+  if test "$app" = quiet ; then
+    return 0
+  fi
+  exit 1
+}
+
+check_logs() {
+  # Iterate through each log to remove an useless warning.
+  for log in $(find "${1}" -type f -name "${2}"); do
+    sed -i "${log}" \
+      -e '/Warning: noted but unhandled ioctl/d' \
+      -e '/could cause spurious value errors to appear/d' \
+      -e '/See README_MISSING_SYSCALL_OR_IOCTL for guidance/d'
+  done
+
+  # Now do it again, but only consider files with size > 0.
+  local err=""
+  for log in $(find "${1}" -type f -name "${2}" -size +0); do
+    cat "${log}"
+    err=1
+  done
+  if [[ -n "${err}" ]]; then
+    echo "Runtime errors detected."
+    exit 1
+  fi
+}
+
+valgrind_check() {
+  check_logs "${1}" "valgrind-*"
+}
+
+asan_check() {
+  check_logs "${1}" "*san.*"
+}
+
+run_unittests() {
+  ulimit -c unlimited
+  if ! ${MAKE_CMD} -C "${BUILD_DIR}" unittest ; then
+    check_core_dumps "$(which luajit)"
+    exit 1
+  fi
+  check_core_dumps "$(which luajit)"
+}
+
+run_functionaltests() {
+  ulimit -c unlimited
+  if ! ${MAKE_CMD} -C "${BUILD_DIR}" ${FUNCTIONALTEST}; then
+    asan_check "${LOG_DIR}"
+    valgrind_check "${LOG_DIR}"
+    check_core_dumps
+    exit 1
+  fi
+  asan_check "${LOG_DIR}"
+  valgrind_check "${LOG_DIR}"
+  check_core_dumps
+}
+
+run_oldtests() {
+  ulimit -c unlimited
+  if ! make -C "${TRAVIS_BUILD_DIR}/src/nvim/testdir"; then
+    reset
+    asan_check "${LOG_DIR}"
+    valgrind_check "${LOG_DIR}"
+    check_core_dumps
+    exit 1
+  fi
+  asan_check "${LOG_DIR}"
+  valgrind_check "${LOG_DIR}"
+  check_core_dumps
+}
+
+install_nvim() {
+  ${MAKE_CMD} -C "${BUILD_DIR}" install
+
+  "${INSTALL_PREFIX}/bin/nvim" --version
+  "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || {
+    echo "Running ':help' in the installed nvim failed."
+    echo "Maybe the helptags have not been generated properly."
+    exit 1
+  }
+
+  local genvimsynf=syntax/vim/generated.vim
+  # Check that all runtime files were installed
+  for file in doc/tags $genvimsynf $(
+    cd runtime ; git ls-files | grep -e '.vim$' -e '.ps$' -e '.dict$' -e '.py$' -e '.tutor$'
+  ) ; do
+    if ! test -e "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
+      echo "It appears that $file is not installed."
+      exit 1
+    fi
+  done
+
+  # Check that generated syntax file has function names, #5060.
+  local gpat='syn keyword vimFuncName .*eval'
+  if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf"; then
+    echo "It appears that $genvimsynf does not contain $gpat."
+    exit 1
+  fi
+
+  for file in $(
+    cd runtime ; git ls-files | grep -e '.awk$' -e '.sh$' -e '.bat$'
+  ) ; do
+    if ! test -x "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
+      echo "It appears that $file is not installed or is not executable."
+      exit 1
+    fi
+  done
+}
diff --git a/ci/install.sh b/ci/install.sh
new file mode 100755
index 0000000000..98d3dc01cb
--- /dev/null
+++ b/ci/install.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ -n "${CI_TARGET}" ]]; then
+  exit
+fi
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  brew install gettext
+  brew reinstall -s libtool
+fi
+
+# Use default CC to avoid compilation problems when installing Python modules.
+echo "Install neovim module and coveralls for Python 2."
+CC=cc pip2.7 -q install --user --upgrade neovim cpp-coveralls
+
+echo "Install neovim module for Python 3."
+CC=cc pip3 -q install --user --upgrade neovim
+
+echo "Install neovim RubyGem."
+gem install --no-document --version ">= 0.2.0" neovim
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
new file mode 100755
index 0000000000..6347ac15d4
--- /dev/null
+++ b/ci/run_tests.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CI_DIR}/common/build.sh"
+source "${CI_DIR}/common/test.sh"
+
+check_core_dumps --delete quiet
+
+prepare_build
+build_nvim
+
+if [ "$CLANG_SANITIZER" != "TSAN" ]; then
+  # Additional threads are only created when the builtin UI starts, which
+  # doesn't happen in the unit/functional tests
+  run_unittests
+  run_functionaltests
+fi
+run_oldtests
+
+install_nvim
+
+touch "${SUCCESS_MARKER}"
diff --git a/ci/script.sh b/ci/script.sh
new file mode 100755
index 0000000000..79a1bec201
--- /dev/null
+++ b/ci/script.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ -n "${CI_TARGET}" ]]; then
+  make "${CI_TARGET}"
+  exit 0
+fi
+
+# This will pass the environment variables down to a bash process which runs
+# as $USER, while retaining the environment variables defined and belonging
+# to secondary groups given above in usermod.
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  sudo -E su "${USER}" -c "ci/run_tests.sh"
+else
+  ci/run_tests.sh
+fi