ci: Better core dump checking

- Do not exclude any directories from `find` search, remove dumps before tests
  instead.
- Install `apport` on travis so that linux tests should produce core dumps
  (based on information from travis-ci/travis-ci#3754, not sure whether it still
  applies).
- Check cores in lua so that one has an idea which test is failing exactly. Do
  this only 10% of time on linux because traversing the file system is slow.

Unit tests are still not touched, though it is what `app` argument in
`check_cores` is for.

TODO? consider using `find`, it may be faster. Consider retiring `os.execute`,
      dealing with escaping is bad.

1 files changed, 104 insertions, 0 deletions

diff --git a/test/helpers.lua b/test/helpers.lua
index 3f7a9c2b74..fab617da50 100644
--- a/test/helpers.lua
+++ b/test/helpers.lua
@@ -17,6 +17,34 @@ local ok = function(res)
   return assert.is_true(res)
 end
 
+local function glob(initial_path, re, exc_re)
+  local paths_to_check = {initial_path}
+  local ret = {}
+  local checked_files = {}
+  while #paths_to_check > 0 do
+    local cur_path = paths_to_check[#paths_to_check]
+    paths_to_check[#paths_to_check] = nil
+    for e in lfs.dir(cur_path) do
+      local full_path = cur_path .. '/' .. e
+      local checked_path = full_path:sub(#initial_path + 1)
+      if ((not exc_re or not checked_path:match(exc_re))
+          and e:sub(1, 1) ~= '.') then
+        local attrs = lfs.attributes(full_path)
+        local check_key = attrs.dev .. ':' .. tostring(attrs.ino)
+        if not checked_files[check_key] then
+          checked_files[check_key] = true
+          if attrs.mode == 'directory' then
+            paths_to_check[#paths_to_check + 1] = full_path
+          elseif not re or checked_path:match(re) then
+            ret[#ret + 1] = full_path
+          end
+        end
+      end
+    end
+  end
+  return ret
+end
+
 local function check_logs()
   local log_dir = os.getenv('LOG_DIR')
   local runtime_errors = 0
@@ -109,6 +137,79 @@ local function filter(filter_func, tab)
   return rettab
 end
 
+local function hasenv(name)
+  local env = os.getenv(name)
+  if env and env ~= '' then
+    return env
+  end
+  return nil
+end
+
+local tests_skipped = 0
+
+local function check_cores(app)
+  app = app or 'build/bin/nvim'
+  local initial_path, re, exc_re
+  local gdb_db_cmd = 'gdb -n -batch -ex "thread apply all bt full" "$_NVIM_TEST_APP" -c "$_NVIM_TEST_CORE"'
+  local lldb_db_cmd = 'lldb -Q -o "bt all" -f "$_NVIM_TEST_APP" -c "$_NVIM_TEST_CORE"'
+  local random_skip = false
+  local db_cmd
+  if hasenv('NVIM_TEST_CORE_GLOB_DIRECTORY') then
+    initial_path = os.getenv('NVIM_TEST_CORE_GLOB_DIRECTORY')
+    re = os.getenv('NVIM_TEST_CORE_GLOB_RE')
+    exc_re = os.getenv('NVIM_TEST_CORE_EXC_RE')
+    db_cmd = os.getenv('NVIM_TEST_CORE_DB_CMD') or gdb_db_cmd
+    random_skip = os.getenv('NVIM_TEST_CORE_RANDOM_SKIP')
+  elseif os.getenv('TRAVIS_OS_NAME') == 'osx' then
+    initial_path = '/cores'
+    re = nil
+    exc_re = nil
+    db_cmd = lldb_db_cmd
+  else
+    initial_path = '.'
+    re = 'core[^/]*'
+    exc_re = '^/%.deps$'
+    db_cmd = gdb_db_cmd
+    random_skip = true
+  end
+  -- Finding cores takes too much time on linux
+  if random_skip and math.random() < 0.9 then
+    tests_skipped = tests_skipped + 1
+    return
+  end
+  local cores = glob(initial_path, re, exc_re)
+  local found_cores = 0
+  local out = io.stdout
+  for _, core in ipairs(cores) do
+    local len = 80 - #core - #('Core file ') - 2
+    local esigns = ('='):rep(len / 2)
+    out:write(('\n%s Core file %s %s\n'):format(esigns, core, esigns))
+    out:flush()
+    local pipe = io.popen(
+        db_cmd:gsub('%$_NVIM_TEST_APP', app):gsub('%$_NVIM_TEST_CORE', core)
+        .. ' 2>&1', 'r')
+    if pipe then
+      local bt = pipe:read('*a')
+      if bt then
+        out:write(bt)
+        out:write('\n')
+      else
+        out:write('Failed to read from the pipe\n')
+      end
+    else
+      out:write('Failed to create pipe\n')
+    end
+    out:flush()
+    found_cores = found_cores + 1
+    os.remove(core)
+  end
+  if found_cores ~= 0 then
+    out:write(('\nTests covered by this check: %u\n'):format(tests_skipped + 1))
+  end
+  tests_skipped = 0
+  assert(0 == found_cores)
+end
+
 return {
   eq = eq,
   neq = neq,
@@ -118,4 +219,7 @@ return {
   tmpname = tmpname,
   map = map,
   filter = filter,
+  glob = glob,
+  check_cores = check_cores,
+  hasenv = hasenv,
 }