Merge PR #2314 'Add vimexpect library and example gdb plugin'

5 files changed, 527 insertions, 13 deletions

diff --git a/contrib/neovim_gdb/neovim_gdb.vim b/contrib/neovim_gdb/neovim_gdb.vim
new file mode 100644
index 0000000000..d61e7bc0cc
--- /dev/null
+++ b/contrib/neovim_gdb/neovim_gdb.vim
@@ -0,0 +1,344 @@
+sign define GdbBreakpoint text=●
+sign define GdbCurrentLine text=⇒
+
+
+let s:gdb_port = 7778
+let s:run_gdb = "gdb -q -f build/bin/nvim"
+let s:breakpoints = {}
+let s:max_breakpoint_sign_id = 0
+
+
+let s:GdbServer = {}
+
+
+function s:GdbServer.new(gdb)
+  let this = copy(self)
+  let this._gdb = a:gdb
+  return this
+endfunction
+
+
+function s:GdbServer.on_exit()
+  let self._gdb._server_exited = 1
+endfunction
+
+
+let s:GdbPaused = vimexpect#State([
+      \ ['Continuing.', 'continue'],
+      \ ['\v[\o32]{2}([^:]+):(\d+):\d+', 'jump'],
+      \ ['Remote communication error.  Target disconnected.:', 'retry'],
+      \ ])
+
+
+function s:GdbPaused.continue(...)
+  call self._parser.switch(s:GdbRunning)
+  call self.update_current_line_sign(0)
+endfunction
+
+
+function s:GdbPaused.jump(file, line, ...)
+  if tabpagenr() != self._tab
+    " Don't jump if we are not in the debugger tab
+    return
+  endif
+  let window = winnr()
+  exe self._jump_window 'wincmd w'
+  let self._current_buf = bufnr('%')
+  let target_buf = bufnr(a:file, 1)
+  if bufnr('%') != target_buf
+    exe 'buffer ' target_buf
+    let self._current_buf = target_buf
+  endif
+  exe ':' a:line
+  let self._current_line = a:line
+  exe window 'wincmd w'
+  call self.update_current_line_sign(1)
+endfunction
+
+
+function s:GdbPaused.retry(...)
+  if self._server_exited
+    return
+  endif
+  sleep 1
+  call self.attach()
+  call self.send('continue')
+endfunction
+
+
+let s:GdbRunning = vimexpect#State([
+      \ ['\v^Breakpoint \d+', 'pause'],
+      \ ['\v\[Inferior\ +.{-}\ +exited\ +normally', 'disconnected'],
+      \ ['(gdb)', 'pause'],
+      \ ])
+
+
+function s:GdbRunning.pause(...)
+  call self._parser.switch(s:GdbPaused)
+  if !self._initialized
+    call self.send('set confirm off')
+    call self.send('set pagination off')
+    if !empty(self._server_addr)
+      call self.send('set remotetimeout 50')
+      call self.attach()
+      call s:RefreshBreakpoints()
+      call self.send('c')
+    endif
+    let self._initialized = 1
+  endif
+endfunction
+
+
+function s:GdbRunning.disconnected(...)
+  if !self._server_exited && self._reconnect
+    " Refresh to force a delete of all watchpoints
+    call s:RefreshBreakpoints()
+    sleep 1
+    call self.attach()
+    call self.send('continue')
+  endif
+endfunction
+
+
+let s:Gdb = {}
+
+
+function s:Gdb.kill()
+  tunmap <f8>
+  tunmap <f10>
+  tunmap <f11>
+  tunmap <f12>
+  call self.update_current_line_sign(0)
+  exe 'bd! '.self._client_buf
+  if self._server_buf != -1
+    exe 'bd! '.self._server_buf
+  endif
+  exe 'tabnext '.self._tab
+  tabclose
+  unlet g:gdb
+endfunction
+
+
+function! s:Gdb.send(data)
+  call jobsend(self._client_id, a:data."\<cr>")
+endfunction
+
+
+function! s:Gdb.attach()
+  call self.send(printf('target remote %s', self._server_addr))
+endfunction
+
+
+function! s:Gdb.update_current_line_sign(add)
+  " to avoid flicker when removing/adding the sign column(due to the change in
+  " line width), we switch ids for the line sign and only remove the old line
+  " sign after marking the new one
+  let old_line_sign_id = get(self, '_line_sign_id', 4999)
+  let self._line_sign_id = old_line_sign_id == 4999 ? 4998 : 4999
+  if a:add && self._current_line != -1 && self._current_buf != -1
+    exe 'sign place '.self._line_sign_id.' name=GdbCurrentLine line='
+          \.self._current_line.' buffer='.self._current_buf
+  endif
+  exe 'sign unplace '.old_line_sign_id
+endfunction
+
+
+function! s:Spawn(server_cmd, client_cmd, server_addr, reconnect)
+  if exists('g:gdb')
+    throw 'Gdb already running'
+  endif
+  let gdb = vimexpect#Parser(s:GdbRunning, copy(s:Gdb))
+  " gdbserver port
+  let gdb._server_addr = a:server_addr
+  let gdb._reconnect = a:reconnect
+  let gdb._initialized = 0
+  " window number that will be displaying the current file
+  let gdb._jump_window = 1
+  let gdb._current_buf = -1
+  let gdb._current_line = -1
+  let gdb._has_breakpoints = 0 
+  let gdb._server_exited = 0
+  " Create new tab for the debugging view
+  tabnew
+  let gdb._tab = tabpagenr()
+  " create horizontal split to display the current file and maybe gdbserver
+  sp
+  let gdb._server_buf = -1
+  if type(a:server_cmd) == type('')
+    " spawn gdbserver in a vertical split
+    let server = s:GdbServer.new(gdb)
+    vsp | enew | let gdb._server_id = termopen(a:server_cmd, server)
+    let gdb._jump_window = 2
+    let gdb._server_buf = bufnr('%')
+  endif
+  " go to the bottom window and spawn gdb client
+  wincmd j
+  enew | let gdb._client_id = termopen(a:client_cmd, gdb)
+  let gdb._client_buf = bufnr('%')
+  tnoremap <silent> <f8> <c-\><c-n>:GdbContinue<cr>i
+  tnoremap <silent> <f10> <c-\><c-n>:GdbNext<cr>i
+  tnoremap <silent> <f11> <c-\><c-n>:GdbStep<cr>i
+  tnoremap <silent> <f12> <c-\><c-n>:GdbFinish<cr>i
+  " go to the window that displays the current file
+  exe gdb._jump_window 'wincmd w'
+  let g:gdb = gdb
+endfunction
+
+
+function! s:Test(bang, filter)
+  let cmd = "GDB=1 make test"
+  if a:bang == '!'
+    let server_addr = '| vgdb'
+    let cmd = printf('VALGRIND=1 %s', cmd)
+  else
+    let server_addr = printf('localhost:%d', s:gdb_port)
+    let cmd = printf('GDBSERVER_PORT=%d %s', s:gdb_port, cmd)
+  endif
+  if a:filter != ''
+    let cmd = printf('TEST_SCREEN_TIMEOUT=1000000 TEST_FILTER="%s" %s', a:filter, cmd)
+  endif
+  call s:Spawn(cmd, s:run_gdb, server_addr, 1)
+endfunction
+
+
+function! s:ToggleBreak()
+  let file_name = bufname('%')
+  let file_breakpoints = get(s:breakpoints, file_name, {})
+  let linenr = line('.')
+  if has_key(file_breakpoints, linenr)
+    call remove(file_breakpoints, linenr)
+  else
+    let file_breakpoints[linenr] = 1
+  endif
+  let s:breakpoints[file_name] = file_breakpoints
+  call s:RefreshBreakpointSigns()
+  call s:RefreshBreakpoints()
+endfunction
+
+
+function! s:ClearBreak()
+  let s:breakpoints = {}
+  call s:RefreshBreakpointSigns()
+  call s:RefreshBreakpoints()
+endfunction
+
+
+function! s:RefreshBreakpointSigns()
+  let buf = bufnr('%')
+  let i = 5000
+  while i <= s:max_breakpoint_sign_id
+    exe 'sign unplace '.i
+    let i += 1
+  endwhile
+  let s:max_breakpoint_sign_id = 0
+  let id = 5000
+  for linenr in keys(get(s:breakpoints, bufname('%'), {}))
+    exe 'sign place '.id.' name=GdbBreakpoint line='.linenr.' buffer='.buf
+    let s:max_breakpoint_sign_id = id
+    let id += 1
+  endfor
+endfunction
+
+
+function! s:RefreshBreakpoints()
+  if !exists('g:gdb')
+    return
+  endif
+  if g:gdb._parser.state() == s:GdbRunning
+    " pause first
+    call jobsend(g:gdb._client_id, "\<c-c>")
+  endif
+  if g:gdb._has_breakpoints
+    call g:gdb.send('delete')
+  endif
+  let g:gdb._has_breakpoints = 0
+  for [file, breakpoints] in items(s:breakpoints)
+    for linenr in keys(breakpoints)
+      let g:gdb._has_breakpoints = 1
+      call g:gdb.send('break '.file.':'.linenr)
+    endfor
+  endfor
+endfunction
+
+
+function! s:GetExpression(...) range
+  let [lnum1, col1] = getpos("'<")[1:2]
+  let [lnum2, col2] = getpos("'>")[1:2]
+  let lines = getline(lnum1, lnum2)
+  let lines[-1] = lines[-1][:col2 - 1]
+  let lines[0] = lines[0][col1 - 1:]
+  return join(lines, "\n")
+endfunction
+
+
+function! s:Send(data)
+  if !exists('g:gdb')
+    throw 'Gdb is not running'
+  endif
+  call g:gdb.send(a:data)
+endfunction
+
+
+function! s:Eval(expr)
+  call s:Send(printf('print %s', a:expr))
+endfunction
+
+
+function! s:Watch(expr)
+  let expr = a:expr
+  if expr[0] != '&'
+    let expr = '&' . expr
+  endif
+
+  call s:Eval(expr)
+  call s:Send('watch *$')
+endfunction
+
+
+function! s:Interrupt()
+  if !exists('g:gdb')
+    throw 'Gdb is not running'
+  endif
+  call jobsend(g:gdb._client_id, "\<c-c>info line\<cr>")
+endfunction
+
+
+function! s:Kill()
+  if !exists('g:gdb')
+    throw 'Gdb is not running'
+  endif
+  call g:gdb.kill()
+endfunction
+
+
+command! GdbDebugNvim call s:Spawn(printf('make && gdbserver localhost:%d build/bin/nvim', s:gdb_port), s:run_gdb, printf('localhost:%d', s:gdb_port), 0)
+command! -nargs=1 GdbDebugServer call s:Spawn(0, s:run_gdb, 'localhost:'.<q-args>, 0)
+command! -bang -nargs=? GdbDebugTest call s:Test(<q-bang>, <q-args>)
+command! -nargs=1 -complete=file GdbInspectCore call s:Spawn(0, printf('gdb -q -f -c %s build/bin/nvim', <q-args>), 0, 0)
+command! GdbDebugStop call s:Kill()
+command! GdbToggleBreakpoint call s:ToggleBreak()
+command! GdbClearBreakpoints call s:ClearBreak()
+command! GdbContinue call s:Send("c")
+command! GdbNext call s:Send("n")
+command! GdbStep call s:Send("s")
+command! GdbFinish call s:Send("finish")
+command! GdbFrameUp call s:Send("up")
+command! GdbFrameDown call s:Send("down")
+command! GdbInterrupt call s:Interrupt()
+command! GdbEvalWord call s:Eval(expand('<cword>'))
+command! -range GdbEvalRange call s:Eval(s:GetExpression(<f-args>))
+command! GdbWatchWord call s:Watch(expand('<cword>')
+command! -range GdbWatchRange call s:Watch(s:GetExpression(<f-args>))
+
+
+nnoremap <silent> <f8> :GdbContinue<cr>
+nnoremap <silent> <f10> :GdbNext<cr>
+nnoremap <silent> <f11> :GdbStep<cr>
+nnoremap <silent> <f12> :GdbFinish<cr>
+nnoremap <silent> <c-b> :GdbToggleBreakpoint<cr>
+nnoremap <silent> <m-pageup> :GdbFrameUp<cr>
+nnoremap <silent> <m-pagedown> :GdbFrameDown<cr>
+nnoremap <silent> <f9> :GdbEvalWord<cr>
+vnoremap <silent> <f9> :GdbEvalRange<cr>
+nnoremap <silent> <m-f9> :GdbWatchWord<cr>
+vnoremap <silent> <m-f9> :GdbWatchRange<cr>
diff --git a/runtime/autoload/vimexpect.vim b/runtime/autoload/vimexpect.vim
new file mode 100644
index 0000000000..16e7d30d6c
--- /dev/null
+++ b/runtime/autoload/vimexpect.vim
@@ -0,0 +1,154 @@
+" vimexpect.vim is a small object-oriented library that simplifies the task of
+" scripting communication with jobs or any interactive program. The name
+" `expect` comes from the famous tcl extension that has the same purpose.
+"
+" This library is built upon two simple concepts: Parsers and States.
+"
+" A State represents a program state and associates a set of regular
+" expressions(to parse program output) with method names(to deal with parsed
+" output). States are created with the vimexpect#State(patterns) function.
+"
+" A Parser manages data received from the program. It also manages State
+" objects by storing them into a stack, where the top of the stack is the
+" current State. Parsers are created with the vimexpect#Parser(initial_state,
+" target) function
+"
+" The State methods are defined by the user, and are always called with `self`
+" set as the Parser target. Advanced control flow is achieved by changing the
+" current state with the `push`/`pop`/`switch` parser methods.
+"
+" An example of this library in action can be found in Neovim source
+" code(contrib/neovim_gdb subdirectory)
+
+let s:State = {}
+
+
+" Create a new State instance with a list where each item is a [regexp, name]
+" pair. A method named `name` must be defined in the created instance.
+function s:State.create(patterns)
+  let this = copy(self)
+  let this._patterns = a:patterns
+  return this
+endfunction
+
+
+let s:Parser = {}
+let s:Parser.LINE_BUFFER_MAX_LEN = 100
+
+
+" Create a new Parser instance with the initial state and a target. The target
+" is a dictionary that will be the `self` of every State method call associated
+" with the parser, and may contain options normally passed to
+" `jobstart`(on_stdout/on_stderr will be overriden). Returns the target so it
+" can be called directly as the second argument of `jobstart`:
+"
+" call jobstart(prog_argv, vimexpect#Parser(initial_state, {'pty': 1}))
+function s:Parser.create(initial_state, target)
+  let parser = copy(self)
+  let parser._line_buffer = []
+  let parser._stack = [a:initial_state]
+  let parser._target = a:target
+  let parser._target.on_stdout = function('s:JobOutput')
+  let parser._target.on_stderr = function('s:JobOutput')
+  let parser._target._parser = parser
+  return parser._target
+endfunction
+
+
+" Push a state to the state stack
+function s:Parser.push(state)
+  call add(self._stack, a:state)
+endfunction
+
+
+" Pop a state from the state stack. Fails if there's only one state remaining.
+function s:Parser.pop()
+  if len(self._stack) == 1
+    throw 'vimexpect:emptystack:State stack cannot be empty'
+  endif
+  return remove(self._stack, -1)
+endfunction
+
+
+" Replace the state currently in the top of the stack.
+function s:Parser.switch(state)
+  let old_state = self._stack[-1]
+  let self._stack[-1] = a:state
+  return old_state
+endfunction
+
+
+" Append a list of lines to the parser line buffer and try to match it the
+" current state. This will shift old lines if the buffer crosses its
+" limit(defined by the LINE_BUFFER_MAX_LEN field). During normal operation,
+" this function is called by the job handler provided by this module, but it
+" may be called directly by the user for other purposes(testing for example)
+function s:Parser.feed(lines)
+  if empty(a:lines)
+    return
+  endif
+  let lines = a:lines
+  let linebuf = self._line_buffer
+  if lines[0] != "\n" && !empty(linebuf)
+    " continue the previous line
+    let linebuf[-1] .= lines[0]
+    call remove(lines, 0)
+  endif
+  " append the newly received lines to the line buffer
+  let linebuf += lines
+  " keep trying to match handlers while the line isnt empty
+  while !empty(linebuf)
+    let match_idx = self.parse(linebuf)
+    if match_idx == -1
+      break
+    endif
+    let linebuf = linebuf[match_idx + 1 : ]
+  endwhile
+  " shift excess lines from the buffer
+  while len(linebuf) > self.LINE_BUFFER_MAX_LEN
+    call remove(linebuf, 0)
+  endwhile
+  let self._line_buffer = linebuf
+endfunction
+
+
+" Try to match a list of lines with the current state and call the handler if
+" the match succeeds. Return the index in `lines` of the first match.
+function s:Parser.parse(lines)
+  let lines = a:lines
+  if empty(lines)
+    return -1
+  endif
+  let state = self.state()
+  " search for a match using the list of patterns
+  for [pattern, handler] in state._patterns
+    let matches = matchlist(lines, pattern)
+    if empty(matches)
+      continue
+    endif
+    let match_idx = match(lines, pattern)
+    call call(state[handler], matches[1:], self._target)
+    return match_idx
+  endfor
+endfunction
+
+
+" Return the current state
+function s:Parser.state()
+  return self._stack[-1]
+endfunction
+
+
+" Job handler that simply forwards lines to the parser.
+function! s:JobOutput(id, lines)
+  call self._parser.feed(a:lines)
+endfunction
+
+function vimexpect#Parser(initial_state, target)
+  return s:Parser.create(a:initial_state, a:target)
+endfunction
+
+
+function vimexpect#State(patterns)
+  return s:State.create(a:patterns)
+endfunction
diff --git a/src/nvim/eval.c b/src/nvim/eval.c
index 9d8421ef04..4ab31985b5 100644
--- a/src/nvim/eval.c
+++ b/src/nvim/eval.c
@@ -5560,8 +5560,10 @@ static int free_unref_items(int copyID)
   bool did_free = false;
 
   // Go through the list of dicts and free items without the copyID.
+  // Don't free dicts that are referenced internally.
   for (dict_T *dd = first_dict; dd != NULL; ) {
-    if ((dd->dv_copyID & COPYID_MASK) != (copyID & COPYID_MASK)) {
+    if ((dd->dv_copyID & COPYID_MASK) != (copyID & COPYID_MASK)
+        && !dd->internal_refcount) {
       // Free the Dictionary and ordinary items it contains, but don't
       // recurse into Lists and Dictionaries, they will be in the list
       // of dicts or list of lists. */
@@ -5671,6 +5673,7 @@ dict_T *dict_alloc(void) FUNC_ATTR_NONNULL_RET
   d->dv_scope = 0;
   d->dv_refcount = 0;
   d->dv_copyID = 0;
+  d->internal_refcount = 0;
 
   return d;
 }
@@ -10969,6 +10972,9 @@ static void f_jobwait(typval_T *argvars, typval_T *rettv)
     }
   }
 
+  // poll to ensure any pending callbacks from the last job are invoked
+  event_poll(0);
+
   for (listitem_T *arg = args->lv_first; arg != NULL; arg = arg->li_next) {
     Job *job = NULL;
     if (arg->li_tv.v_type != VAR_NUMBER
@@ -20064,6 +20070,7 @@ static inline void common_job_callbacks(dict_T *vopts, ufunc_T **on_stdout,
     return;
   }
 
+  vopts->internal_refcount++;
   vopts->dv_refcount++;
 }
 
@@ -20097,7 +20104,11 @@ static inline void free_term_job_data(TerminalJobData *data) {
   if (data->on_exit) {
     user_func_unref(data->on_exit);
   }
-  dict_unref(data->self);
+
+  if (data->self) {
+    data->self->internal_refcount--;
+    dict_unref(data->self);
+  }
   free(data);
 }
 
diff --git a/src/nvim/eval_defs.h b/src/nvim/eval_defs.h
index d2de830d6c..34a36004d6 100644
--- a/src/nvim/eval_defs.h
+++ b/src/nvim/eval_defs.h
@@ -111,6 +111,8 @@ struct dictvar_S {
   dict_T      *dv_copydict;     /* copied dict used by deepcopy() */
   dict_T      *dv_used_next;    /* next dict in used dicts list */
   dict_T      *dv_used_prev;    /* previous dict in used dicts list */
+  int internal_refcount;        // number of internal references to
+                                // prevent garbage collection
 };
 
 #endif // NVIM_EVAL_DEFS_H
diff --git a/test/functional/job/job_spec.lua b/test/functional/job/job_spec.lua
index c1c559eb34..c517ae4c1b 100644
--- a/test/functional/job/job_spec.lua
+++ b/test/functional/job/job_spec.lua
@@ -200,19 +200,22 @@ describe('jobs', function()
     it('will run callbacks while waiting', function()
       source([[
       let g:dict = {'id': 10}
-      let g:l = []
-      function g:dict.on_stdout(id, data)
-        call add(g:l, a:data[0])
+      let g:exits = 0
+      function g:dict.on_exit(id, code)
+        if a:code != 5
+          throw 'Error!'
+        endif
+        let g:exits += 1
       endfunction
       call jobwait([
-      \  jobstart([&sh, '-c', 'sleep 0.010; echo 4'], g:dict),
-      \  jobstart([&sh, '-c', 'sleep 0.030; echo 5'], g:dict),
-      \  jobstart([&sh, '-c', 'sleep 0.050; echo 6'], g:dict),
-      \  jobstart([&sh, '-c', 'sleep 0.070; echo 7'], g:dict)
+      \  jobstart([&sh, '-c', 'sleep 0.010; exit 5'], g:dict),
+      \  jobstart([&sh, '-c', 'sleep 0.030; exit 5'], g:dict),
+      \  jobstart([&sh, '-c', 'sleep 0.050; exit 5'], g:dict),
+      \  jobstart([&sh, '-c', 'sleep 0.070; exit 5'], g:dict)
       \  ])
-      call rpcnotify(g:channel, 'wait', g:l)
+      call rpcnotify(g:channel, 'wait', g:exits)
       ]])
-      eq({'notification', 'wait', {{'4', '5', '6', '7'}}}, next_msg())
+      eq({'notification', 'wait', {4}}, next_msg())
     end)
 
     it('will return status codes in the order of passed ids', function()
@@ -250,8 +253,8 @@ describe('jobs', function()
       it('will return -1 if the wait timed out', function()
         source([[
         call rpcnotify(g:channel, 'wait', jobwait([
-        \  jobstart([&sh, '-c', 'sleep 0.05; exit 4']),
-        \  jobstart([&sh, '-c', 'sleep 0.3; exit 5']),
+        \  jobstart([&sh, '-c', 'exit 4']),
+        \  jobstart([&sh, '-c', 'sleep 10000; exit 5']),
         \  ], 100))
         ]])
         eq({'notification', 'wait', {{4, -1}}}, next_msg())