API: Test: Setup basic test infrastructure

- Add a 'expect' utility script that can run simple API tests using clients
  developed for any platform.
- Extend travis build matrix to run API tests using the python client and
  valgrind.

This script can be used to write API tests without having to manage nvim's
lifetime:

- It starts a single nvim instance listening on a known socket
- Invokes the test runner, which should connect to NEOVIM_LISTEN_ADDRESS
- The nvim instance started by the script provides a `BeforeEachTest` function,
  which should be called before each test to reset nvim to a clean state.
- It takes care of shutting down nvim once the tests are finished.

As explained
[here](https://github.com/neovim/neovim/pull/737#issuecomment-43941520), it's
not possible to fully reset nvim to it's initial state, but the `BeforeEachTest`
function should be enough for most test cases. Tests requiring a fully clean
nvim instance should take care of starting/stopping nvim.

3 files changed, 142 insertions, 7 deletions

diff --git a/.travis.yml b/.travis.yml
index 25ab961ec4..d883410123 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,5 +4,6 @@ env:
   - TRAVIS_BUILD_TYPE=gcc/ia32
   - TRAVIS_BUILD_TYPE=gcc/unittest
   - TRAVIS_BUILD_TYPE=clint
+  - TRAVIS_BUILD_TYPE=api/python
 script:
   - ./scripts/travis.sh
diff --git a/scripts/run-api-tests.exp b/scripts/run-api-tests.exp
new file mode 100755
index 0000000000..6a568e17cb
--- /dev/null
+++ b/scripts/run-api-tests.exp
@@ -0,0 +1,86 @@
+#!/usr/bin/env expect
+
+if {$argc < 2} {
+  puts "Need commands for running the tests and for starting nvim"
+  exit 1
+}
+
+set timeout 10
+set run_tests [split [lindex $argv 0] " "]
+set run_nvim [split [lindex $argv 1] " "]
+
+# don't echo to stdout
+log_user 0
+# set NEOVIM_LISTEN_ADDRESS, so nvim will listen on a known socket
+set env(NEOVIM_LISTEN_ADDRESS) "/tmp/nvim-[exec date +%s%N].sock"
+# start nvim
+spawn {*}$run_nvim
+# save the job descriptor
+set nvim_id $spawn_id
+# Reset function that can be invoked by test runners to put nvim in a cleaner
+# state
+send {
+:function BeforeEachTest()
+  set all&
+  redir => groups
+  silent augroup
+  redir END
+  for group in split(groups)
+    exe 'augroup '.group
+    autocmd!
+    augroup NONE
+    exe 'augroup! '.group
+  endfor
+  tabnew
+  let curbufnum = eval(bufnr('%'))
+  redir => buflist
+  silent ls!
+  redir END
+  let bufnums = []
+  for buf in split(buflist, '\n')
+    let bufnum = eval(split(buf, '[ u]')[0])
+    if bufnum != curbufnum
+      call add(bufnums, bufnum)
+    endif
+  endfor
+  if len(bufnums) > 0
+    exe 'silent bwipeout! '.join(bufnums, ' ')
+  endif
+  silent tabonly
+  for k in keys(g:)
+    exe 'unlet g:'.k
+  endfor
+  filetype plugin indent off
+  mapclear
+  mapclear!
+  abclear
+  comclear
+endfunction
+:echo "read"."y"
+}
+# wait until nvim is ready
+expect "ready"
+# run tests
+spawn {*}$run_tests
+set tests_id $spawn_id
+set status 1
+# listen for test output in the background
+expect_background {
+  * {
+    # show test output to the user
+    send_user -- $expect_out(buffer)
+  }
+  eof {
+    # collect the exit status code
+    set spawn_id $tests_id
+    catch wait result
+    set status [lindex $result 3]
+    set spawn_id $nvim_id
+    # quit nvim
+    send ":qa!\r"
+  }
+}
+# switch back nvim and wait until it exits 
+set spawn_id $nvim_id
+expect eof
+exit $status
diff --git a/scripts/travis.sh b/scripts/travis.sh
index 62c4fbe98e..3daa2652e5 100755
--- a/scripts/travis.sh
+++ b/scripts/travis.sh
@@ -1,8 +1,44 @@
 #!/bin/sh -e
 
-check_and_report() {
+tmpdir="$(pwd)/tmp"
+rm -rf "$tmpdir"
+mkdir -p "$tmpdir"
+
+valgrind_check() {
 	(
-	cd $tmpdir
+	cd $1
+	set -- valgrind-[*] valgrind-*
+	case $1$2 in
+		'valgrind-[*]valgrind-*')
+			;;
+		*)
+			shift
+			local err=''
+			for valgrind_log in "$@"; do
+				# Remove useless warning
+				sed -i "$valgrind_log" \
+					-e '/Warning: noted but unhandled ioctl/d' \
+					-e '/could cause spurious value errors to appear/d' \
+					-e '/See README_MISSING_SYSCALL_OR_IOCTL for guidance/d'
+				if [ "$(stat -c %s $valgrind_log)" != "0" ]; then
+					# if after removing the warning, the log still has errors, show its
+					# contents and set the flag so we exit with non-zero status
+					cat "$valgrind_log"
+					err=1
+				fi
+			done
+			if [ -n "$err" ]; then
+				echo "Runtime errors detected"
+				exit 1
+			fi
+			;;
+	esac
+	)
+}
+
+asan_check() {
+	(
+	cd $1
 	set -- [*]san.[*] *san.*
 	case $1$2 in
 		'[*]san.[*]*san.*')
@@ -68,9 +104,6 @@ if [ "$TRAVIS_BUILD_TYPE" = "clang/asan" ]; then
 
 	install_dir="$(pwd)/dist"
 	# temporary directory for writing sanitizer logs
-	tmpdir="$(pwd)/tmp"
-	rm -rf "$tmpdir"
-	mkdir -p "$tmpdir"
 
 	# need the symbolizer path for stack traces with source information
 	if [ -n "$USE_CLANG_34" ]; then
@@ -91,10 +124,10 @@ if [ "$TRAVIS_BUILD_TYPE" = "clang/asan" ]; then
 	$MAKE_CMD
 	if ! $MAKE_CMD test; then
 		reset
-		check_and_report
+		asan_check "$tmpdir"
 		exit 1
 	fi
-	check_and_report
+	asan_check "$tmpdir"
 	coveralls --encoding iso-8859-1 || echo 'coveralls upload failed.'
 	$MAKE_CMD install
 elif [ "$TRAVIS_BUILD_TYPE" = "gcc/unittest" ]; then
@@ -129,4 +162,19 @@ elif [ "$TRAVIS_BUILD_TYPE" = "gcc/ia32" ]; then
 	$MAKE_CMD test
 elif [ "$TRAVIS_BUILD_TYPE" = "clint" ]; then
 	./scripts/clint.sh
+elif [ "$TRAVIS_BUILD_TYPE" = "api/python" ]; then
+	set_environment /opt/neovim-deps
+	$MAKE_CMD
+	sudo apt-get install expect valgrind
+	git clone --depth=1 -b master git://github.com/neovim/python-client
+	cd python-client
+  sudo pip install .
+  sudo pip install nose
+	test_cmd="nosetests --verbosity=2"
+	nvim_cmd="valgrind -q --track-origins=yes --log-file=$tmpdir/valgrind-%p.log ../build/bin/nvim -u NONE"
+	if ! ../scripts/run-api-tests.exp "$test_cmd" "$nvim_cmd"; then
+		valgrind_check "$tmpdir"
+		exit 1
+	fi
+	valgrind_check "$tmpdir"
 fi