Merge branch 'master' into yagebu/option-fixes

10 files changed, 234 insertions, 101 deletions

diff --git a/ci/after_success.sh b/ci/after_success.sh
deleted file mode 100755
index 0215eb139b..0000000000
--- a/ci/after_success.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -o pipefail
-
-if [[ -n "${GCOV}" ]]; then
-  coveralls --gcov "$(which "${GCOV}")" --encoding iso-8859-1 || echo 'coveralls upload failed.'
-fi
diff --git a/ci/before_cache.sh b/ci/before_cache.sh
index 3d7cc0ec5a..d99b8e68c6 100755
--- a/ci/before_cache.sh
+++ b/ci/before_cache.sh
@@ -4,16 +4,24 @@ set -e
 set -o pipefail
 
 CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/suite.sh"
 
 # Don't cache pip's log and selfcheck.
 rm -rf "${HOME}/.cache/pip/log"
 rm -f "${HOME}/.cache/pip/selfcheck.json"
 
+echo "before_cache.sh: cache size"
+du -d 2 "${HOME}/.cache" | sort -n
+
 # Update the third-party dependency cache only if the build was successful.
 if ended_successfully; then
   rm -rf "${HOME}/.cache/nvim-deps"
   mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps"
+
+  rm -rf "${HOME}/.cache/nvim-deps-downloads"
+  mv "${DEPS_DOWNLOAD_DIR}" "${HOME}/.cache/nvim-deps-downloads"
+
   touch "${CACHE_MARKER}"
-  echo "Updated third-party dependencies (timestamp: $(stat -c '%y' "${CACHE_MARKER}"))."
+  echo "Updated third-party dependencies (timestamp: $(_stat "${CACHE_MARKER}"))."
 fi
diff --git a/ci/before_install.sh b/ci/before_install.sh
index 5b36adaef2..9a3e192536 100755
--- a/ci/before_install.sh
+++ b/ci/before_install.sh
@@ -11,15 +11,38 @@ if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
   brew update
 fi
 
-echo "Upgrade Python 2 pip."
-pip2.7 -q install --user --upgrade pip
+echo 'python info:'
+(
+  2>&1 python --version || true
+  2>&1 python2 --version || true
+  2>&1 python3 --version || true
+  2>&1 pip --version || true
+  2>&1 pip2 --version || true
+  2>&1 pip3 --version || true
+  echo 'pyenv versions:'
+  2>&1 pyenv versions || true
+) | sed 's/^/  /'
 
 if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
-  echo "Install Python 3."
-  brew install python3
+  echo "Upgrade Python 3."
+  brew upgrade python
   echo "Upgrade Python 3 pip."
   pip3 -q install --user --upgrade pip
 else
+  echo "Upgrade Python 2 pip."
+  pip2.7 -q install --user --upgrade pip
   echo "Upgrade Python 3 pip."
-  pip3 -q install --user --upgrade pip
+  # Allow failure. pyenv pip3 on travis is broken:
+  # https://github.com/travis-ci/travis-ci/issues/8363
+  pip3 -q install --user --upgrade pip || true
 fi
+
+echo "Install node (LTS)"
+
+if [[ "${TRAVIS_OS_NAME}" == osx ]] || [ ! -f ~/.nvm/nvm.sh ]; then
+  curl -o ~/.nvm/nvm.sh https://raw.githubusercontent.com/creationix/nvm/master/nvm.sh
+fi
+
+source ~/.nvm/nvm.sh
+nvm install --lts
+nvm use --lts
diff --git a/ci/build.bat b/ci/build.bat
deleted file mode 100644
index c871c6b849..0000000000
--- a/ci/build.bat
+++ /dev/null
@@ -1,54 +0,0 @@
-:: These are native MinGW builds, but they use the toolchain inside
-:: MSYS2, this allows using all the dependencies and tools available
-:: in MSYS2, but we cannot build inside the MSYS2 shell.
-echo on
-if "%CONFIGURATION%" == "MINGW_32" (
-  set ARCH=i686
-  set BITS=32
-) else (
-  set ARCH=x86_64
-  set BITS=64
-)
-:: We cannot have sh.exe in the PATH (MinGW)
-set PATH=%PATH:C:\Program Files\Git\usr\bin;=%
-set PATH=C:\msys64\mingw%BITS%\bin;C:\Windows\System32;C:\Windows;%PATH%
-:: The default cpack in the PATH is not CMake
-set PATH=C:\Program Files (x86)\CMake\bin\cpack.exe;%PATH%
-
-:: Build third-party dependencies
-C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm -Su" || goto :error
-C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm --needed -S mingw-w64-%ARCH%-cmake mingw-w64-%ARCH%-perl mingw-w64-%ARCH%-diffutils mingw-w64-%ARCH%-unibilium gperf" || goto :error
-
-:: Setup python (use AppVeyor system python)
-C:\Python27\python.exe -m pip install neovim || goto :error
-C:\Python35\python.exe -m pip install neovim || goto :error
-:: Disambiguate python3
-move c:\Python35\python.exe c:\Python35\python3.exe
-set PATH=C:\Python35;C:\Python27;%PATH%
-:: Sanity check
-python  -c "import neovim; print(str(neovim))" || goto :error
-python3 -c "import neovim; print(str(neovim))" || goto :error
-
-mkdir .deps
-cd .deps
-cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=RelWithDebInfo ..\third-party\ || goto :error
-mingw32-make VERBOSE=1 || goto :error
-cd ..
-
-:: Build Neovim
-mkdir build
-cd build
-cmake -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUSTED_OUTPUT_TYPE=nvim -DGPERF_PRG="C:\msys64\usr\bin\gperf.exe" .. || goto :error
-mingw32-make VERBOSE=1 || goto :error
-bin\nvim --version || goto :error
-
-:: Functional tests
-mingw32-make functionaltest VERBOSE=1 || goto :error
-
-:: Build artifacts
-cpack -G ZIP -C RelWithDebInfo
-if defined APPVEYOR_REPO_TAG_NAME cpack -G NSIS -C RelWithDebInfo
-
-goto :EOF
-:error
-exit /b %errorlevel%
diff --git a/ci/build.ps1 b/ci/build.ps1
new file mode 100644
index 0000000000..df70b44458
--- /dev/null
+++ b/ci/build.ps1
@@ -0,0 +1,123 @@
+Set-PSDebug -Trace 1
+
+$env:CONFIGURATION -match '^(?<compiler>\w+)_(?<bits>32|64)(?:-(?<option>\w+))?$'
+$compiler = $Matches.compiler
+$compileOption = $Matches.option
+$bits = $Matches.bits
+$cmakeBuildType = 'RelWithDebInfo'
+$depsCmakeVars = @{
+  CMAKE_BUILD_TYPE = $cmakeBuildType;
+}
+$nvimCmakeVars = @{
+  CMAKE_BUILD_TYPE = $cmakeBuildType;
+  BUSTED_OUTPUT_TYPE = 'nvim';
+}
+
+function exitIfFailed() {
+  if ($LastExitCode -ne 0) {
+    exit $LastExitCode
+  }
+}
+
+if ($compiler -eq 'MINGW') {
+  if ($bits -eq 32) {
+    $arch = 'i686'
+  }
+  elseif ($bits -eq 64) {
+    $arch = 'x86_64'
+  }
+  if ($compileOption -eq 'gcov') {
+    $nvimCmakeVars['USE_GCOV'] = 'ON'
+    $uploadToCodecov = $true
+  }
+  # These are native MinGW builds, but they use the toolchain inside
+  # MSYS2, this allows using all the dependencies and tools available
+  # in MSYS2, but we cannot build inside the MSYS2 shell.
+  $cmakeGenerator = 'MinGW Makefiles'
+  $cmakeGeneratorArgs = 'VERBOSE=1'
+
+  # Add MinGW to the PATH
+  $env:PATH = "C:\msys64\mingw$bits\bin;$env:PATH"
+
+  # Build third-party dependencies
+  C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm -Su" ; exitIfFailed
+  C:\msys64\usr\bin\bash -lc "pacman --verbose --noconfirm --needed -S mingw-w64-$arch-cmake mingw-w64-$arch-perl mingw-w64-$arch-diffutils mingw-w64-$arch-unibilium" ; exitIfFailed
+}
+elseif ($compiler -eq 'MSVC') {
+  $cmakeGeneratorArgs = '/verbosity:normal'
+  if ($bits -eq 32) {
+    $cmakeGenerator = 'Visual Studio 15 2017'
+  }
+  elseif ($bits -eq 64) {
+    $cmakeGenerator = 'Visual Studio 15 2017 Win64'
+  }
+}
+
+# Remove Git Unix utilities from the PATH
+$env:PATH = $env:PATH.Replace('C:\Program Files\Git\usr\bin', '')
+
+# Setup python (use AppVeyor system python)
+C:\Python27\python.exe -m pip install neovim ; exitIfFailed
+C:\Python35\python.exe -m pip install neovim ; exitIfFailed
+# Disambiguate python3
+move c:\Python35\python.exe c:\Python35\python3.exe
+$env:PATH = "C:\Python35;C:\Python27;$env:PATH"
+# Sanity check
+python  -c "import neovim; print(str(neovim))" ; exitIfFailed
+python3 -c "import neovim; print(str(neovim))" ; exitIfFailed
+
+$env:PATH = "C:\Ruby24\bin;$env:PATH"
+cmd /c gem.cmd install neovim ; exitIfFailed
+where.exe neovim-ruby-host.bat ; exitIfFailed
+
+cmd /c npm.cmd install -g neovim ; exitIfFailed
+where.exe neovim-node-host.cmd ; exitIfFailed
+
+function convertToCmakeArgs($vars) {
+  return $vars.GetEnumerator() | foreach { "-D$($_.Key)=$($_.Value)" }
+}
+
+mkdir .deps
+cd .deps
+cmake -G $cmakeGenerator $(convertToCmakeArgs($depsCmakeVars)) ..\third-party\ ; exitIfFailed
+cmake --build . --config $cmakeBuildType -- $cmakeGeneratorArgs ; exitIfFailed
+cd ..
+
+# Build Neovim
+mkdir build
+cd build
+cmake -G $cmakeGenerator $(convertToCmakeArgs($nvimCmakeVars)) .. ; exitIfFailed
+cmake --build . --config $cmakeBuildType -- $cmakeGeneratorArgs ; exitIfFailed
+bin\nvim --version ; exitIfFailed
+
+# Functional tests
+# The $LastExitCode from MSBuild can't be trusted
+$failed = $false
+# Temporarily turn off tracing to reduce log file output
+Set-PSDebug -Off
+cmake --build . --config $cmakeBuildType --target functionaltest -- $cmakeGeneratorArgs |
+  foreach { $failed = $failed -or
+    $_ -match 'Running functional tests failed with error'; $_ }
+Set-PSDebug -Trace 1
+if ($failed) {
+  exit $LastExitCode
+}
+
+
+if ($uploadToCodecov) {
+  C:\msys64\usr\bin\bash -lc "cd /c/projects/neovim; bash <(curl -s https://codecov.io/bash) -c -F functionaltest || echo 'codecov upload failed.'"
+}
+
+# Old tests
+$env:PATH += ';C:\msys64\usr\bin'
+& "C:\msys64\mingw$bits\bin\mingw32-make.exe" -C $(Convert-Path ..\src\nvim\testdir) VERBOSE=1
+
+if ($uploadToCodecov) {
+  C:\msys64\usr\bin\bash -lc "cd /c/projects/neovim; bash <(curl -s https://codecov.io/bash) -c -F oldtest || echo 'codecov upload failed.'"
+}
+
+# Build artifacts
+cpack -G ZIP -C RelWithDebInfo
+if ($env:APPVEYOR_REPO_TAG_NAME -ne $null) {
+  cpack -G NSIS -C RelWithDebInfo
+}
diff --git a/ci/common/build.sh b/ci/common/build.sh
index f398a1a1cc..80973b64c9 100644
--- a/ci/common/build.sh
+++ b/ci/common/build.sh
@@ -1,5 +1,16 @@
+_stat() {
+  if test "${TRAVIS_OS_NAME}" = osx ; then
+    stat -f %Sm "${@}"
+  else
+    stat -c %y "${@}"
+  fi
+}
+
 top_make() {
-  ${MAKE_CMD} "$@"
+  echo '================================================================================'
+  # Travis has 1.5 virtual cores according to:
+  # http://docs.travis-ci.com/user/speeding-up-the-build/#Paralellizing-your-build-on-one-VM
+  ninja "$@"
 }
 
 build_make() {
@@ -15,28 +26,21 @@ build_deps() {
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} -DUSE_BUNDLED_LUA=ON"
   fi
 
-  rm -rf "${DEPS_BUILD_DIR}"
-
-  # If there is a valid cache and we're not forced to recompile,
-  # use cached third-party dependencies.
-  if test -f "${CACHE_MARKER}" && test "${BUILD_NVIM_DEPS}" != "true" ; then
-    local statcmd="stat -c '%y'"
-    if test "${TRAVIS_OS_NAME}" = osx ; then
-      statcmd="stat -f '%Sm'"
-    fi
-    echo "Using third-party dependencies from Travis's cache (last updated: $(${statcmd} "${CACHE_MARKER}"))."
+  mkdir -p "${DEPS_BUILD_DIR}"
+  mkdir -p "${DEPS_DOWNLOAD_DIR}"
 
-    mkdir -p "$(dirname "${DEPS_BUILD_DIR}")"
-    mv "${HOME}/.cache/nvim-deps" "${DEPS_BUILD_DIR}"
-  else
-    mkdir -p "${DEPS_BUILD_DIR}"
+  # Use cached dependencies if $CACHE_MARKER exists.
+  if test -f "${CACHE_MARKER}" && ! test "${CACHE_ENABLE}" = "false" ; then
+    echo "Using third-party dependencies from Travis cache (last update: $(_stat "${CACHE_MARKER}"))."
+    cp -r "${HOME}/.cache/nvim-deps" "${DEPS_BUILD_DIR}"
+    cp -r "${HOME}/.cache/nvim-deps-downloads" "${DEPS_DOWNLOAD_DIR}"
   fi
 
   # Even if we're using cached dependencies, run CMake and make to
   # update CMake configuration and update to newer deps versions.
   cd "${DEPS_BUILD_DIR}"
   echo "Configuring with '${DEPS_CMAKE_FLAGS}'."
-  CC= cmake ${DEPS_CMAKE_FLAGS} "${TRAVIS_BUILD_DIR}/third-party/"
+  CC= cmake -G Ninja ${DEPS_CMAKE_FLAGS} "${TRAVIS_BUILD_DIR}/third-party/"
 
   if ! top_make; then
     exit 1
@@ -56,7 +60,7 @@ prepare_build() {
   mkdir -p "${BUILD_DIR}"
   cd "${BUILD_DIR}"
   echo "Configuring with '${CMAKE_FLAGS} $@'."
-  cmake ${CMAKE_FLAGS} "$@" "${TRAVIS_BUILD_DIR}"
+  cmake -G Ninja ${CMAKE_FLAGS} "$@" "${TRAVIS_BUILD_DIR}"
 }
 
 build_nvim() {
@@ -71,9 +75,11 @@ build_nvim() {
       exit 1
     fi
 
-    echo "Building nvim-test."
-    if ! top_make nvim-test ; then
-      exit 1
+    if test "${FUNCTIONALTEST}" != "functionaltest-lua"; then
+      echo "Building nvim-test."
+      if ! top_make nvim-test ; then
+        exit 1
+      fi
     fi
   fi
 
@@ -87,3 +93,12 @@ build_nvim() {
 
   cd "${TRAVIS_BUILD_DIR}"
 }
+
+macos_rvm_dance() {
+  # neovim-ruby gem requires a ruby newer than the macOS default.
+  source ~/.rvm/scripts/rvm
+  rvm get stable --auto-dotfiles
+  rvm reload
+  rvm use 2.2.5
+  rvm use
+}
diff --git a/ci/common/test.sh b/ci/common/test.sh
index 55f76ca798..bc80dfead7 100644
--- a/ci/common/test.sh
+++ b/ci/common/test.sh
@@ -1,6 +1,15 @@
 . "${CI_DIR}/common/build.sh"
 . "${CI_DIR}/common/suite.sh"
 
+submit_coverage() {
+  if [ -n "${GCOV}" ]; then
+    if curl --fail --output codecov.bash --silent https://codecov.io/bash; then
+      bash codecov.bash -c -F "$1" || echo "codecov upload failed."
+      rm -f codecov.bash
+    fi
+  fi
+}
+
 print_core() {
   local app="$1"
   local core="$2"
@@ -71,7 +80,9 @@ valgrind_check() {
 }
 
 asan_check() {
-  check_logs "${1}" "*san.*"
+  if test "${CLANG_SANITIZER}" = "ASAN_UBSAN" ; then
+    check_logs "${1}" "*san.*" | asan_symbolize
+  fi
 }
 
 run_unittests() {(
@@ -80,6 +91,7 @@ run_unittests() {(
   if ! build_make unittest ; then
     fail 'unittests' F 'Unit tests failed'
   fi
+  submit_coverage unittest
   check_core_dumps "$(which luajit)"
   exit_suite
 )}
@@ -90,6 +102,7 @@ run_functionaltests() {(
   if ! build_make ${FUNCTIONALTEST}; then
     fail 'functionaltests' F 'Functional tests failed'
   fi
+  submit_coverage functionaltest
   asan_check "${LOG_DIR}"
   valgrind_check "${LOG_DIR}"
   check_core_dumps
@@ -103,6 +116,7 @@ run_oldtests() {(
     reset
     fail 'oldtests' F 'Legacy tests failed'
   fi
+  submit_coverage oldtest
   asan_check "${LOG_DIR}"
   valgrind_check "${LOG_DIR}"
   check_core_dumps
diff --git a/ci/install.sh b/ci/install.sh
index 4ee99e1e44..053549d6db 100755
--- a/ci/install.sh
+++ b/ci/install.sh
@@ -8,16 +8,26 @@ if [[ "${CI_TARGET}" == lint ]]; then
 fi
 
 if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
+  brew install ninja
   brew install gettext
   brew reinstall -s libtool
 fi
 
-# Use default CC to avoid compilation problems when installing Python modules.
-echo "Install neovim module and coveralls for Python 2."
-CC=cc pip2.7 -q install --user --upgrade neovim cpp-coveralls
-
 echo "Install neovim module for Python 3."
-CC=cc pip3 -q install --user --upgrade neovim
+# Allow failure. pyenv pip3 on travis is broken:
+# https://github.com/travis-ci/travis-ci/issues/8363
+CC=cc pip3 -q install --user --upgrade neovim || true
+
+if ! [ "${TRAVIS_OS_NAME}" = osx ] ; then
+  # Update PATH for pip.
+  export PATH="$(python2.7 -c 'import site; print(site.getuserbase())')/bin:$PATH"
+  # Use default CC to avoid compilation problems when installing Python modules.
+  echo "Install neovim module for Python 2."
+  CC=cc pip2.7 -q install --user --upgrade neovim
+
+  echo "Install neovim RubyGem."
+  gem install --no-document --version ">= 0.2.0" neovim
+fi
 
-echo "Install neovim RubyGem."
-gem install --no-document --version ">= 0.2.0" neovim
+echo "Install neovim npm package"
+npm install -g neovim
diff --git a/ci/run_lint.sh b/ci/run_lint.sh
index e7f6727448..ae9adb7c87 100755
--- a/ci/run_lint.sh
+++ b/ci/run_lint.sh
@@ -10,19 +10,19 @@ source "${CI_DIR}/common/suite.sh"
 
 enter_suite 'clint'
 
-run_test 'top_make clint-full' clint
+run_test 'make clint-full' clint
 
 exit_suite --continue
 
 enter_suite 'testlint'
 
-run_test 'top_make testlint' testlint
+run_test 'make testlint' testlint
 
 exit_suite --continue
 
 enter_suite 'lualint'
 
-run_test 'top_make lualint' lualint
+run_test 'make lualint' lualint
 
 exit_suite --continue
 
@@ -31,7 +31,7 @@ enter_suite single-includes
 CLICOLOR_FORCE=1 run_test_wd \
   --allow-hang \
   10s \
-  'top_make check-single-includes' \
+  'make check-single-includes' \
   'csi_clean' \
   single-includes
 
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index a0bf6e010d..c175910da5 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -22,7 +22,9 @@ enter_suite tests
 if test "$CLANG_SANITIZER" != "TSAN" ; then
   # Additional threads are only created when the builtin UI starts, which
   # doesn't happen in the unit/functional tests
-  run_test run_unittests
+  if test "${FUNCTIONALTEST}" != "functionaltest-lua"; then
+    run_test run_unittests
+  fi
   run_test run_functionaltests
 fi
 run_test run_oldtests