Merge branch 'master' into luaviml'/lua

12 files changed, 596 insertions, 0 deletions

diff --git a/ci/after_success.sh b/ci/after_success.sh
new file mode 100755
index 0000000000..0215eb139b
--- /dev/null
+++ b/ci/after_success.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ -n "${GCOV}" ]]; then
+  coveralls --gcov "$(which "${GCOV}")" --encoding iso-8859-1 || echo 'coveralls upload failed.'
+fi
diff --git a/ci/before_cache.sh b/ci/before_cache.sh
new file mode 100755
index 0000000000..dd1fcf2bf7
--- /dev/null
+++ b/ci/before_cache.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+# Don't cache pip's log and selfcheck.
+rm -rf "${HOME}/.cache/pip/log"
+rm -f "${HOME}/.cache/pip/selfcheck.json"
+
+# Update the third-party dependency cache only if the build was successful.
+if [[ -f "${SUCCESS_MARKER}" ]]; then
+  rm -rf "${HOME}/.cache/nvim-deps"
+  mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps"
+  touch "${CACHE_MARKER}"
+  echo "Updated third-party dependencies (timestamp: $(stat -c '%y' "${CACHE_MARKER}"))."
+fi
diff --git a/ci/before_install.sh b/ci/before_install.sh
new file mode 100755
index 0000000000..5b36adaef2
--- /dev/null
+++ b/ci/before_install.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ "${CI_TARGET}" == lint ]]; then
+  exit
+fi
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  brew update
+fi
+
+echo "Upgrade Python 2 pip."
+pip2.7 -q install --user --upgrade pip
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  echo "Install Python 3."
+  brew install python3
+  echo "Upgrade Python 3 pip."
+  pip3 -q install --user --upgrade pip
+else
+  echo "Upgrade Python 3 pip."
+  pip3 -q install --user --upgrade pip
+fi
diff --git a/ci/before_script.sh b/ci/before_script.sh
new file mode 100755
index 0000000000..445996a8df
--- /dev/null
+++ b/ci/before_script.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ "${CI_TARGET}" == lint ]]; then
+  exit
+fi
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CI_DIR}/common/build.sh"
+
+# Test some of the configuration variables.
+if [[ -n "${GCOV}" ]] && [[ ! $(type -P "${GCOV}") ]]; then
+  echo "\$GCOV: '${GCOV}' is not executable."
+  exit 1
+fi
+if [[ -n "${LLVM_SYMBOLIZER}" ]] && [[ ! $(type -P "${LLVM_SYMBOLIZER}") ]]; then
+  echo "\$LLVM_SYMBOLIZER: '${LLVM_SYMBOLIZER}' is not executable."
+  exit 1
+fi
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  # Adds user to a dummy group.
+  # That allows to test changing the group of the file by `os_fchown`.
+  sudo dscl . -create /Groups/chown_test
+  sudo dscl . -append /Groups/chown_test GroupMembership "${USER}"
+fi
+
+# Compile dependencies.
+build_deps
+
+rm -rf "${LOG_DIR}"
+mkdir -p "${LOG_DIR}"
diff --git a/ci/build.bat b/ci/build.bat
new file mode 100644
index 0000000000..87a171b994
--- /dev/null
+++ b/ci/build.bat
@@ -0,0 +1,54 @@
+:: These are native MinGW builds, but they use the toolchain inside
+:: MSYS2, this allows using all the dependencies and tools available
+:: in MSYS2, but we cannot build inside the MSYS2 shell.
+echo on
+if "%CONFIGURATION%" == "MINGW_32" (
+  set ARCH=i686
+  set BITS=32
+) else (
+  set ARCH=x86_64
+  set BITS=64
+)
+:: We cannot have sh.exe in the PATH (MinGW)
+set PATH=%PATH:C:\Program Files\Git\usr\bin;=%
+set PATH=C:\msys64\mingw%BITS%\bin;C:\Windows\System32;C:\Windows;%PATH%
+:: The default cpack in the PATH is not CMake
+set PATH=C:\Program Files (x86)\CMake\bin\cpack.exe;%PATH%
+
+:: Build third-party dependencies
+C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm -Su" || goto :error
+C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm --needed -S mingw-w64-%ARCH%-cmake mingw-w64-%ARCH%-perl mingw-w64-%ARCH%-diffutils gperf" || goto :error
+
+:: Setup python (use AppVeyor system python)
+C:\Python27\python.exe -m pip install neovim || goto :error
+C:\Python35\python.exe -m pip install neovim || goto :error
+:: Disambiguate python3
+move c:\Python35\python.exe c:\Python35\python3.exe
+set PATH=C:\Python35;C:\Python27;%PATH%
+:: Sanity check
+python  -c "import neovim; print(str(neovim))" || goto :error
+python3 -c "import neovim; print(str(neovim))" || goto :error
+
+mkdir .deps
+cd .deps
+cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release ..\third-party\ || goto :error
+mingw32-make VERBOSE=1 || goto :error
+cd ..
+
+:: Build Neovim
+mkdir build
+cd build
+cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUSTED_OUTPUT_TYPE=gtest -DGPERF_PRG="C:\msys64\usr\bin\gperf.exe" .. || goto :error
+mingw32-make VERBOSE=1 || goto :error
+bin\nvim --version || goto :error
+
+:: Functional tests
+mingw32-make functionaltest VERBOSE=1 || goto :error
+
+:: Build artifacts
+cpack -G ZIP -C Release
+if defined APPVEYOR_REPO_TAG_NAME cpack -G NSIS -C Release
+
+goto :EOF
+:error
+exit /b %errorlevel%
diff --git a/ci/common/build.sh b/ci/common/build.sh
new file mode 100644
index 0000000000..d5211405ca
--- /dev/null
+++ b/ci/common/build.sh
@@ -0,0 +1,90 @@
+top_make() {
+  ${MAKE_CMD} "$@"
+}
+
+build_make() {
+  top_make -C "${BUILD_DIR}" "$@"
+}
+
+build_deps() {
+  if [[ "${BUILD_32BIT}" == ON ]]; then
+    DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
+  fi
+  if [[ "${FUNCTIONALTEST}" == "functionaltest-lua" ]] \
+     || [[ "${CLANG_SANITIZER}" == "ASAN_UBSAN" ]]; then
+    DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} -DUSE_BUNDLED_LUA=ON"
+  fi
+
+  rm -rf "${DEPS_BUILD_DIR}"
+
+  # If there is a valid cache and we're not forced to recompile,
+  # use cached third-party dependencies.
+  if [[ -f "${CACHE_MARKER}" ]] && [[ "${BUILD_NVIM_DEPS}" != true ]]; then
+    if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+      local statcmd="stat -f '%Sm'"
+    else
+      local statcmd="stat -c '%y'"
+    fi
+    echo "Using third-party dependencies from Travis's cache (last updated: $(${statcmd} "${CACHE_MARKER}"))."
+
+     mkdir -p "$(dirname "${DEPS_BUILD_DIR}")"
+     mv "${HOME}/.cache/nvim-deps" "${DEPS_BUILD_DIR}"
+  else
+    mkdir -p "${DEPS_BUILD_DIR}"
+  fi
+
+  # Even if we're using cached dependencies, run CMake and make to
+  # update CMake configuration and update to newer deps versions.
+  cd "${DEPS_BUILD_DIR}"
+  echo "Configuring with '${DEPS_CMAKE_FLAGS}'."
+  CC= cmake ${DEPS_CMAKE_FLAGS} "${TRAVIS_BUILD_DIR}/third-party/"
+
+  if ! top_make; then
+    exit 1
+  fi
+
+  cd "${TRAVIS_BUILD_DIR}"
+}
+
+prepare_build() {
+  if [[ -n "${CLANG_SANITIZER}" ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DCLANG_${CLANG_SANITIZER}=ON"
+  fi
+  if [[ "${BUILD_32BIT}" == ON ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
+  fi
+
+  mkdir -p "${BUILD_DIR}"
+  cd "${BUILD_DIR}"
+  echo "Configuring with '${CMAKE_FLAGS} $@'."
+  cmake ${CMAKE_FLAGS} "$@" "${TRAVIS_BUILD_DIR}"
+}
+
+build_nvim() {
+  echo "Building nvim."
+  if ! top_make nvim; then
+    exit 1
+  fi
+
+  if [ "$CLANG_SANITIZER" != "TSAN" ]; then
+    echo "Building libnvim."
+    if ! top_make libnvim; then
+      exit 1
+    fi
+
+    echo "Building nvim-test."
+    if ! top_make nvim-test; then
+      exit 1
+    fi
+  fi
+
+  # Invoke nvim to trigger *San early.
+  if ! (bin/nvim --version && bin/nvim -u NONE -e -c ':qall'); then
+    asan_check "${LOG_DIR}"
+    exit 1
+  fi
+  asan_check "${LOG_DIR}"
+
+
+  cd "${TRAVIS_BUILD_DIR}"
+}
diff --git a/ci/common/suite.sh b/ci/common/suite.sh
new file mode 100644
index 0000000000..46207754fa
--- /dev/null
+++ b/ci/common/suite.sh
@@ -0,0 +1,121 @@
+# HACK: get newline for use in strings given that "\n" and $'' do not work.
+NL="$(printf '\nE')"
+NL="${NL%E}"
+
+FAILED=0
+
+FAIL_SUMMARY=""
+
+enter_suite() {
+  local suite_name="$1"
+  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name"
+}
+
+exit_suite() {
+  if test $FAILED -ne 0 ; then
+    echo "Suite ${NVIM_TEST_CURRENT_SUITE} failed, summary:"
+    echo "${FAIL_SUMMARY}"
+  fi
+  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}"
+  if test "x$1" != "x--continue" ; then
+    exit $FAILED
+  fi
+}
+
+fail() {
+  local allow_failure=
+  if test "x$1" = "x--allow-failure" ; then
+    shift
+    allow_failure=A
+  fi
+  local test_name="$1"
+  local fail_char="$allow_failure$2"
+  local message="$3"
+
+  : ${fail_char:=F}
+  : ${message:=Test $test_name failed}
+
+  local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message"
+  FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}"
+  echo "Failed: $full_msg"
+  if test "x$allow_failure" = "x" ; then
+    FAILED=1
+  fi
+}
+
+run_test() {
+  local cmd="$1"
+  test $# -gt 0 && shift
+  local test_name="$1"
+  : ${test_name:=$cmd}
+  test $# -gt 0 && shift
+  if ! eval "$cmd" ; then
+    fail "${test_name}" "$@"
+  fi
+}
+
+run_test_wd() {
+  local timeout="$1"
+  test $# -gt 0 && shift
+
+  local cmd="$1"
+  test $# -gt 0 && shift
+
+  local restart_cmd="$1"
+  : ${restart_cmd:=true}
+  test $# -gt 0 && shift
+
+  local test_name="$1"
+  : ${test_name:=$cmd}
+  test $# -gt 0 && shift
+
+  local output_file="$(mktemp)"
+  local status_file="$(mktemp)"
+
+  local restarts=5
+  local prev_tmpsize=-1
+  while test $restarts -gt 0 ; do
+    : > "${status_file}"
+    (
+      FAILED=0
+      if ! (
+        set -o pipefail
+        eval "$cmd" 2>&1 | tee -a "$output_file"
+      ) ; then
+        fail "${test_name}" "$@"
+      fi
+      echo "$FAILED" > "$status_file"
+    ) &
+    local pid=$!
+    while test "$(stat -c "%s" "$status_file")" -eq 0 ; do
+      prev_tmpsize=$tmpsize
+      sleep $timeout
+      tmpsize="$(stat -c "%s" "$output_file")"
+      if test $tempsize -eq $prev_temsize ; then
+        # no output, assuming either hang or exit
+        break
+      fi
+    done
+    restarts=$[ restarts - 1 ]
+    if test "$(stat -c "%s" "$status_file")" -eq 0 ; then
+      # status file not updated, assuming hang
+      kill -KILL $pid
+      if test $restarts -eq 0 ; then
+        fail "${test_name}" E "Test hang up"
+      else
+        echo "Test ${test_name} hang up, restarting"
+        eval "$restart_cmd"
+      fi
+    else
+      local new_failed="$(cat "$status_file")"
+      if test "x$new_failed" != "x0" ; then
+        fail "${test_name}" F "Test failed in run_test_wd"
+      fi
+      return 0
+    fi
+  done
+}
+
+succeeded() {
+  return $FAILED
+}
diff --git a/ci/common/test.sh b/ci/common/test.sh
new file mode 100644
index 0000000000..4936992cfd
--- /dev/null
+++ b/ci/common/test.sh
@@ -0,0 +1,150 @@
+source "${CI_DIR}/common/build.sh"
+
+print_core() {
+  local app="$1"
+  local core="$2"
+  if test "$app" = quiet ; then
+    echo "Found core $core"
+    return 0
+  fi
+  echo "======= Core file $core ======="
+  if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+    lldb -Q -o "bt all" -f "${app}" -c "${core}"
+  else
+    gdb -n -batch -ex 'thread apply all bt full' "${app}" -c "${core}"
+  fi
+}
+
+check_core_dumps() {
+  local del=
+  if test "$1" = "--delete" ; then
+    del=1
+    shift
+  fi
+  local app="${1:-${BUILD_DIR}/bin/nvim}"
+  if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+    local cores="$(find /cores/ -type f -print)"
+  else
+    local cores="$(find ./ -type f -name 'core.*' -print)"
+  fi
+
+  if [ -z "${cores}" ]; then
+    return
+  fi
+  local core
+  for core in $cores; do
+    if test "$del" = "1" ; then
+      print_core "$app" "$core" >&2
+      rm "$core"
+    else
+      print_core "$app" "$core"
+    fi
+  done
+  if test "$app" = quiet ; then
+    return 0
+  fi
+  exit 1
+}
+
+check_logs() {
+  # Iterate through each log to remove an useless warning.
+  for log in $(find "${1}" -type f -name "${2}"); do
+    sed -i "${log}" \
+      -e '/Warning: noted but unhandled ioctl/d' \
+      -e '/could cause spurious value errors to appear/d' \
+      -e '/See README_MISSING_SYSCALL_OR_IOCTL for guidance/d'
+  done
+
+  # Now do it again, but only consider files with size > 0.
+  local err=""
+  for log in $(find "${1}" -type f -name "${2}" -size +0); do
+    cat "${log}"
+    err=1
+  done
+  if [[ -n "${err}" ]]; then
+    echo "Runtime errors detected."
+    exit 1
+  fi
+}
+
+valgrind_check() {
+  check_logs "${1}" "valgrind-*"
+}
+
+asan_check() {
+  check_logs "${1}" "*san.*"
+}
+
+run_unittests() {
+  ulimit -c unlimited
+  if ! build_make unittest ; then
+    check_core_dumps "$(which luajit)"
+    exit 1
+  fi
+  check_core_dumps "$(which luajit)"
+}
+
+run_functionaltests() {
+  ulimit -c unlimited
+  if ! build_make ${FUNCTIONALTEST}; then
+    asan_check "${LOG_DIR}"
+    valgrind_check "${LOG_DIR}"
+    check_core_dumps
+    exit 1
+  fi
+  asan_check "${LOG_DIR}"
+  valgrind_check "${LOG_DIR}"
+  check_core_dumps
+}
+
+run_oldtests() {
+  ulimit -c unlimited
+  if ! make -C "${TRAVIS_BUILD_DIR}/src/nvim/testdir"; then
+    reset
+    asan_check "${LOG_DIR}"
+    valgrind_check "${LOG_DIR}"
+    check_core_dumps
+    exit 1
+  fi
+  asan_check "${LOG_DIR}"
+  valgrind_check "${LOG_DIR}"
+  check_core_dumps
+}
+
+install_nvim() {
+  build_make install
+
+  "${INSTALL_PREFIX}/bin/nvim" --version
+  "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || {
+    echo "Running ':help' in the installed nvim failed."
+    echo "Maybe the helptags have not been generated properly."
+    exit 1
+  }
+
+  local genvimsynf=syntax/vim/generated.vim
+  # Check that all runtime files were installed
+  for file in doc/tags $genvimsynf $(
+    cd runtime ; git ls-files | grep -e '.vim$' -e '.ps$' -e '.dict$' -e '.py$' -e '.tutor$'
+  ) ; do
+    if ! test -e "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
+      echo "It appears that $file is not installed."
+      exit 1
+    fi
+  done
+
+  # Check that generated syntax file has function names, #5060.
+  local gpat='syn keyword vimFuncName .*eval'
+  if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf"; then
+    echo "It appears that $genvimsynf does not contain $gpat."
+    exit 1
+  fi
+
+  for file in $(
+    cd runtime ; git ls-files | grep -e '.awk$' -e '.sh$' -e '.bat$'
+  ) ; do
+    if ! test -x "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
+      echo "It appears that $file is not installed or is not executable."
+      exit 1
+    fi
+  done
+}
diff --git a/ci/install.sh b/ci/install.sh
new file mode 100755
index 0000000000..4ee99e1e44
--- /dev/null
+++ b/ci/install.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ "${CI_TARGET}" == lint ]]; then
+  exit
+fi
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  brew install gettext
+  brew reinstall -s libtool
+fi
+
+# Use default CC to avoid compilation problems when installing Python modules.
+echo "Install neovim module and coveralls for Python 2."
+CC=cc pip2.7 -q install --user --upgrade neovim cpp-coveralls
+
+echo "Install neovim module for Python 3."
+CC=cc pip3 -q install --user --upgrade neovim
+
+echo "Install neovim RubyGem."
+gem install --no-document --version ">= 0.2.0" neovim
diff --git a/ci/run_lint.sh b/ci/run_lint.sh
new file mode 100755
index 0000000000..39a90102e7
--- /dev/null
+++ b/ci/run_lint.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${CI_DIR}/common/build.sh"
+source "${CI_DIR}/common/suite.sh"
+
+enter_suite 'lint'
+
+set -x
+
+csi_clean() {
+  find "${BUILD_DIR}/bin" -name 'test-includes-*' -delete
+  find "${BUILD_DIR}" -name '*test-include*.o' -delete
+}
+
+run_test 'top_make clint-full' clint
+run_test 'top_make testlint' testlint
+CLICOLOR_FORCE=1 run_test_wd \
+  5s \
+  'top_make check-single-includes' \
+  'csi_clean' \
+  single-includes
+
+exit_suite
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
new file mode 100755
index 0000000000..92cb5a9fd8
--- /dev/null
+++ b/ci/run_tests.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CI_DIR}/common/build.sh"
+source "${CI_DIR}/common/test.sh"
+source "${CI_DIR}/common/suite.sh"
+
+set -x
+
+enter_suite tests
+
+check_core_dumps --delete quiet
+
+prepare_build
+build_nvim
+
+if [ "$CLANG_SANITIZER" != "TSAN" ]; then
+  # Additional threads are only created when the builtin UI starts, which
+  # doesn't happen in the unit/functional tests
+  run_test run_unittests
+  run_test run_functionaltests
+fi
+run_test run_oldtests
+
+run_test install_nvim
+
+if succeeded ; then
+  touch "${SUCCESS_MARKER}"
+fi
+
+exit_suite
diff --git a/ci/script.sh b/ci/script.sh
new file mode 100755
index 0000000000..a59c40cd2d
--- /dev/null
+++ b/ci/script.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+# This will pass the environment variables down to a bash process which runs
+# as $USER, while retaining the environment variables defined and belonging
+# to secondary groups given above in usermod.
+if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  sudo -E su "${USER}" -c "ci/run_${CI_TARGET}.sh"
+else
+  ci/run_${CI_TARGET}.sh
+fi