Merge PR #2247 'Refactor/enhance job api'

8 files changed, 258 insertions, 49 deletions

diff --git a/runtime/doc/Makefile b/runtime/doc/Makefile
index 2f9063231d..4bced03d7e 100644
--- a/runtime/doc/Makefile
+++ b/runtime/doc/Makefile
@@ -55,6 +55,7 @@ DOCS = \
 	nvim_intro.txt \
 	nvim_provider.txt \
 	nvim_python.txt \
+	nvim_terminal_emulator.txt \
 	options.txt \
 	os_dos.txt \
 	os_mac.txt \
@@ -176,6 +177,7 @@ HTMLS = \
 	nvim_intro.html \
 	nvim_provider.html \
 	nvim_python.html \
+	nvim_terminal_emulator.html \
 	options.html \
 	os_dos.html \
 	os_mac.html \
diff --git a/runtime/doc/autocmd.txt b/runtime/doc/autocmd.txt
index dc236fc78f..1393131ab7 100644
--- a/runtime/doc/autocmd.txt
+++ b/runtime/doc/autocmd.txt
@@ -253,6 +253,7 @@ Name			triggered by ~
 |BufNew|		just after creating a new buffer
 
 |SwapExists|		detected an existing swap file
+|TermOpen|		when a terminal buffer is starting
 
 	Options
 |FileType|		when the 'filetype' option has been set
@@ -307,7 +308,6 @@ Name			triggered by ~
 |InsertLeave|		when leaving Insert mode
 |InsertCharPre|		when a character was typed in Insert mode, before
 			inserting it
-|JobActivity|		when something interesting happens with a job
 
 |TextChanged|		after a change was made to the text in Normal mode
 |TextChangedI|		after a change was made to the text in Insert mode
@@ -733,10 +733,6 @@ InsertEnter			Just before starting Insert mode.  Also for
 							*InsertLeave*
 InsertLeave			When leaving Insert mode.  Also when using
 				CTRL-O |i_CTRL-O|.  But not for |i_CTRL-C|.
-						{Nvim}  *JobActivity*
-JobActivity			When something interesting happens with a job
-				spawned by |jobstart()|. See |job-control| for
-				details. 
 							*MenuPopup*
 MenuPopup			Just before showing the popup menu (under the
 				right mouse button).  Useful for adjusting the
@@ -876,6 +872,11 @@ TermChanged			After the value of 'term' has changed.  Useful
 				for re-loading the syntax file to update the
 				colors, fonts and other terminal-dependent
 				settings.  Executed for all loaded buffers.
+						 {Nvim} *TermOpen*
+TermOpen			When a terminal buffer is starting.  This can
+				be used to configure the terminal emulator by
+				setting buffer variables.
+				See |nvim-terminal-emulator| for details.
 							*TermResponse*
 TermResponse			After the response to |t_RV| is received from
 				the terminal.  The value of |v:termresponse|
diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index 1d18a61dbd..5f572e9caa 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -4012,6 +4012,15 @@ items({dict})						*items()*
 		entry and the value of this entry.  The |List| is in arbitrary
 		order.
 
+jobclose({job}[, {stream}])				{Nvim} *jobclose()*
+		Close {job}'s {stream}, which can be one "stdin", "stdout" or
+		"stderr". If {stream} is omitted, all streams are closed.
+
+jobresize({job}, {width}, {height})			{Nvim} *jobresize()*
+		Resize {job}'s pseudo terminal window to {width} and {height}.
+		This function will fail if used on jobs started without the
+		"pty" option.
+
 jobsend({job}, {data})					{Nvim} *jobsend()*
 		Send data to {job} by writing it to the stdin of the process.
 		Returns 1 if the write succeeded, 0 otherwise.
@@ -4024,14 +4033,28 @@ jobsend({job}, {data})					{Nvim} *jobsend()*
 			:call jobsend(j, ["abc", "123\n456", ""])
 < 		will send "abc<NL>123<NUL>456<NL>".
 
-jobstart({name}, {prog}[, {argv}])			{Nvim} *jobstart()*
-		Spawns {prog} as a job and associate it with the {name} string,
-		which will be used to match the "filename pattern" in
-		|JobActivity| events. It returns:
-		  - The job id on success, which is used by |jobsend()| and
+jobstart({argv}[, {opts}])			{Nvim} *jobstart()*
+		Spawns {argv}(list) as a job. If passed, {opts} must be a
+		dictionary with any of the following keys:
+		  - on_stdout: stdout event handler
+		  - on_stderr: stderr event handler
+		  - on_exit: exit event handler
+		  - pty: If set, the job will be connected to a new pseudo
+		    terminal, and the job streams are connected to the master
+		    file descriptor.
+		  - width: Width of the terminal screen(only if pty is set)
+		  - height: Height of the terminal screen(only if pty is set)
+		  - TERM: $TERM environment variable(only if pty is set)
+		Either funcrefs or function names can be passed as event
+		handlers. The {opts} object is also used as the "self"
+		argument for the callback, so the caller may pass arbitrary
+		data by setting other key.(see |Dictionary-function| for more
+		information).
+		Returns:
+		  - The job ID on success, which is used by |jobsend()| and
 		    |jobstop()|
 		  - 0 when the job table is full or on invalid arguments
-		  - -1 when {prog} is not executable
+		  - -1 when {argv}[0] is not executable
 		See |job-control| for more information.
 
 jobstop({job})						{Nvim} *jobstop()*
@@ -4042,6 +4065,24 @@ jobstop({job})						{Nvim} *jobstop()*
 		`v:job_data[0]` set to `exited`.  See |job-control| for more
 		information.
 
+jobwait({ids}[, {timeout}])				{Nvim} *jobwait()*
+		Wait for a set of jobs to finish. The {ids} argument is a list
+		of ids for jobs that will be waited for. If passed, {timeout}
+		is the maximum number of milliseconds to wait. While this
+		function is executing, callbacks for jobs not in the {ids}
+		list can be executed. Also, the screen wont be updated unless
+		|:redraw| is invoked by one of the callbacks.
+
+		Returns a list of integers with the same length as {ids}, with
+		each integer representing the wait result for the
+		corresponding job id. The possible values for the resulting
+		integers are:
+
+		* the job return code if the job exited
+		* -1 if the wait timed out for the job
+		* -2 if the job was interrupted
+		* -3 if the job id is invalid.
+
 join({list} [, {sep}])					*join()*
 		Join the items in {list} together into one String.
 		When {sep} is specified it is put in between the items.  If
@@ -6277,6 +6318,17 @@ tempname()					*tempname()* *temp-file-name*
 		For MS-Windows forward slashes are used when the 'shellslash'
 		option is set or when 'shellcmdflag' starts with '-'.
 
+termopen({command}[, {opts}])			{Nvim} *termopen()*
+		Spawns {command} using the shell in a new pseudo-terminal
+		session connected to the current buffer.  This function fails
+		if the current buffer is modified (all buffer contents are
+		destroyed).  The {opts} dict is similar to the one passed to
+		|jobstart()|, but the `pty`, `width`, `height`, and `TERM` fields are
+		ignored: `height`/`width` are taken from the current window and
+		$TERM is set to "xterm-256color".  Returns the same values as
+		|jobstart()|.
+
+		See |nvim-terminal-emulator| for more information.
 
 tan({expr})						*tan()*
 		Return the tangent of {expr}, measured in radians, as a |Float|
diff --git a/runtime/doc/job_control.txt b/runtime/doc/job_control.txt
index 1faf9bcd94..dc746bbe99 100644
--- a/runtime/doc/job_control.txt
+++ b/runtime/doc/job_control.txt
@@ -37,37 +37,28 @@ for details
 ==============================================================================
 2. Usage						*job-control-usage*
 
-Here's a quick one-liner that creates a job which invokes the "ls" shell
-command and prints the result:
->
-    call jobstart('', 'ls', ['-a'])|au JobActivity * echo v:job_data|au!
-    JobActivity
-
-In the one-liner above, creating the JobActivity event handler immediately
-after the call to jobstart() is not a race because the Nvim job system will
-not publish the job result (even though it may receive it) until evaluation of
-the chained user commands (`expr1|expr2|...|exprN`) has completed.
-
 Job control is achieved by calling a combination of the |jobstart()|,
-|jobsend()| and |jobstop()| functions, and by listening to the |JobActivity|
-event. The best way to understand is with a complete example:
+|jobsend()| and |jobstop()| functions. Here's an example:
 >
-    let job1 = jobstart('shell1', 'bash')
-    let job2 = jobstart('shell2', 'bash', ['-c', 'for ((i = 0; i < 10; i++)); do echo hello $i!; sleep 1; done'])
-
-    function JobHandler()
-      if v:job_data[1] == 'stdout'
-        let str = 'shell '. v:job_data[0].' stdout: '.join(v:job_data[2])
-      elseif v:job_data[1] == 'stderr'
-        let str = 'shell '.v:job_data[0].' stderr: '.join(v:job_data[2])
+    function s:JobHandler(job_id, data, event)
+      if a:event == 'stdout'
+        let str = self.shell.' stdout: '.join(a:data)
+      elseif a:event == 'stderr'
+        let str = self.shell.' stderr: '.join(a:data)
       else
-        let str = 'shell '.v:job_data[0].' exited'
+        let str = self.shell.' exited'
       endif
 
       call append(line('$'), str)
     endfunction
+    let s:callbacks = {
+    \ 'on_stdout': function('s:JobHandler'),
+    \ 'on_stderr': function('s:JobHandler'),
+    \ 'on_exit': function('s:JobHandler')
+    \ }
+    let job1 = jobstart(['bash'], extend({'shell': 'shell 1'}, s:callbacks))
+    let job2 = jobstart(['bash', '-c', 'for i in {1..10}; do echo hello $i!; sleep 1; done'], extend({'shell': 'shell 2'}, s:callbacks))
 
-    au JobActivity shell* call JobHandler()
 <
 To test the above, copy it to the file ~/jobcontrol.vim and start with a clean
 nvim instance:
@@ -82,16 +73,51 @@ Here's what is happening:
 - The second shell is started with the -c argument, causing it to execute a
   command then exit. In this case, the command is a for loop that will print 0
   through 9 then exit.
-- The `JobHandler()` function is called by the `JobActivity` autocommand (notice
-  how the shell* pattern matches the names `shell1` and `shell2` passed to
-  |jobstart()|), and it takes care of displaying stdout/stderr received from
+- The `JobHandler()` function is a callback passed to |jobstart()| to handle
+  various job events. It takes care of displaying stdout/stderr received from
   the shells.
-- The v:job_data is an array set by the JobActivity event. It has the
-  following elements:
+- The arguments passed to `JobHandler()` are: 
+
   0: The job id
-  1: The kind of activity: one of "stdout", "stderr" or "exit"
-  2: When "activity" is "stdout" or "stderr", this will contain a list of
-     lines read from stdout or stderr
+  1: If the event is "stdout" or "stderr", a list with lines read from the
+     corresponding stream. For "exit", it is the status returned by the
+     program.
+  2: The event type, which is "stdout", "stderr" or "exit".
+
+The options dictionary is passed as the "self" variable to the callback
+function. Here's a more object-oriented version of the above:
+>
+    let Shell = {}
+    
+    function Shell.on_stdout(job_id, data)
+      call append(line('$'), self.get_name().' stdout: '.join(a:data))
+    endfunction
+    
+    function Shell.on_stderr(job_id, data)
+      call append(line('$'), self.get_name().' stderr: '.join(a:data))
+    endfunction
+    
+    function Shell.on_exit(job_id, data)
+      call append(line('$'), self.get_name().' exited')
+    endfunction
+    
+    function Shell.get_name()
+      return 'shell '.self.name
+    endfunction
+    
+    function Shell.new(name, ...)
+      let instance = extend(copy(g:Shell), {'name': a:name})
+      let argv = ['bash']
+      if a:0 > 0
+        let argv += ['-c', a:1]
+      endif
+      let instance.id = jobstart(argv, instance)
+      return instance
+    endfunction
+    
+    let s1 = Shell.new('1')
+    let s2 = Shell.new('2', 'for i in {1..10}; do echo hello $i!; sleep 1; done')
+
 
 To send data to the job's stdin, one can use the |jobsend()| function, like
 this:
diff --git a/runtime/doc/nvim_intro.txt b/runtime/doc/nvim_intro.txt
index d44648c575..40f65620af 100644
--- a/runtime/doc/nvim_intro.txt
+++ b/runtime/doc/nvim_intro.txt
@@ -13,13 +13,14 @@ see |help.txt|.
 For now, it is just an index with the most relevant topics/features that
 differentiate Nvim from Vim:
 
-1. Differences from Vim		|vim-differences|
-2. Msgpack-RPC			|msgpack-rpc|
-3. Job control			|job-control|
-4. Python plugins		|nvim-python|
-5. Clipboard integration	|nvim-clipboard|
-6. Remote plugins		|remote-plugin|
-7. Provider infrastructure	|nvim-provider|
+1. Differences from Vim		 |vim-differences|
+2. Msgpack-RPC			 |msgpack-rpc|
+3. Job control			 |job-control|
+4. Python plugins		 |nvim-python|
+5. Clipboard integration	 |nvim-clipboard|
+6. Remote plugins		 |remote-plugin|
+7. Provider infrastructure	 |nvim-provider|
+8. Integrated terminal emulator  |nvim-terminal-emulator|
 
 ==============================================================================
  vim:tw=78:ts=8:noet:ft=help:norl:
diff --git a/runtime/doc/nvim_terminal_emulator.txt b/runtime/doc/nvim_terminal_emulator.txt
new file mode 100644
index 0000000000..160242a7fa
--- /dev/null
+++ b/runtime/doc/nvim_terminal_emulator.txt
@@ -0,0 +1,114 @@
+*nvim_terminal_emulator.txt*    For Nvim.				{Nvim}
+
+
+		 NVIM REFERENCE MANUAL    by Thiago de Arruda
+
+
+Nvim integrated terminal emulator		     *nvim-terminal-emulator*
+
+1. Introduction			|nvim-terminal-emulator-introduction|
+2. Spawning			|nvim-terminal-emulator-spawning|
+3. Input			|nvim-terminal-emulator-input|
+4. Configuration		|nvim-terminal-emulator-configuration|
+
+==============================================================================
+1. Introduction			      *nvim-terminal-emulator-introduction*
+
+One feature that distinguishes Nvim from Vim is that it implements a mostly
+complete VT220/xterm-like terminal emulator. The terminal is presented to the
+user as a special buffer type, one that is asynchronously updated to mirror
+the virtual terminal display as data is received from the program connected
+to it. For most purposes, terminal buffers behave a lot like normal buffers
+with 'nomodifiable' set.
+
+
+The implementation is powered by libvterm[1], a powerful abstract terminal
+emulation library.
+
+[1]: http://www.leonerd.org.uk/code/libvterm/
+
+==============================================================================
+2. Spawning			      *nvim-terminal-emulator-spawning*
+
+There are 3 ways to create a terminal buffer:
+
+- By invoking the |:terminal| ex command.
+- By calling the |termopen()| function.
+- By editing a file with a name matching `term://(.{-}//(\d+:)?)?\zs.*`.
+  For example:
+>
+    :e term://bash
+    :vsp term://top
+<
+When the terminal spawns the program, the buffer will start to mirror the
+terminal display and change its name to `term://$CWD//$PID:$COMMAND`.
+Note that |:mksession| will "save" the terminal buffers by restarting all
+programs when the session is restored.
+
+==============================================================================
+3. Input			      *nvim-terminal-emulator-input*
+
+Sending input is possible by entering terminal mode, which is achieved by
+pressing any key that would enter insert mode in a normal buffer (|i| or |a|
+for example). The |:terminal| ex command will automatically enter terminal
+mode once it's spawned. While in terminal mode, Nvim will forward all keys to
+the underlying program. The only exception is the <C-\><C-n> key combo,
+which will exit back to normal mode.
+
+Terminal mode has its own namespace for mappings, which is accessed with the
+"t" prefix. It's possible to use terminal mappings to customize interaction
+with the terminal. For example, here's how to map <Esc> to exit terminal mode:
+>
+    :tnoremap <Esc> <C-\><C-n>
+<
+Navigating to other windows is only possible by exiting to normal mode, which
+can be cumbersome with <C-\><C-n> keys. Here are some mappings to improve
+the navigation experience:
+>
+    :tnoremap <A-h> <C-\><C-n><C-w>h
+    :tnoremap <A-j> <C-\><C-n><C-w>j
+    :tnoremap <A-k> <C-\><C-n><C-w>k
+    :tnoremap <A-l> <C-\><C-n><C-w>l
+    :nnoremap <A-h> <C-w>h
+    :nnoremap <A-j> <C-w>j
+    :nnoremap <A-k> <C-w>k
+    :nnoremap <A-l> <C-w>l
+<
+This allows using `Alt+{h,j,k,l}` to navigate between windows no matter if
+they are displaying a normal buffer or a terminal buffer in terminal mode.
+
+Mouse input is also fully supported, and has the following behavior:
+
+- If the program has enabled mouse events, the corresponding events will be
+  forwarded to the program.
+- If mouse events are disabled (the default), terminal focus will be lost and
+  the event will be processed as in a normal buffer.
+- If another window is clicked, terminal focus will be lost and nvim will jump
+  to the clicked window
+- If the mouse wheel is used while the mouse is positioned in another window,
+  the terminal wont lose focus and the hovered window will be scrolled.
+
+==============================================================================
+4. Configuration		     *nvim-terminal-emulator-configuration*
+
+Terminal buffers can be customized through the following global/buffer-local
+variables (set via the |TermOpen| autocmd):
+
+- `{g,b}:terminal_scrollback_buffer_size`: Scrollback buffer size, between 1
+  and 100000 inclusive. The default is 1000.
+- `{g,b}:terminal_color_$NUM`: The terminal color palette, where `$NUM` is the
+  color index, between 0 and 255 inclusive. This only affects UIs with RGB
+  capabilities; for normal terminals the color index is simply forwarded.
+- `{g,b}:terminal_focused_cursor_highlight`: Highlight group applied to the
+  cursor in a focused terminal. The default equivalent to having a group with
+  `cterm=reverse` `gui=reverse``.
+- `{g,b}:terminal_unfocused_cursor_highlight`: Highlight group applied to the
+  cursor in an unfocused terminal. The default equivalent to having a group with
+  `ctermbg=11` `guibg=#fce94f``.
+
+The configuration variables are only processed when the terminal starts, which
+is why it needs to be done with the |TermOpen| autocmd or setting global
+variables before the terminal is started.
+
+==============================================================================
+ vim:tw=78:ts=8:noet:ft=help:norl:
diff --git a/runtime/doc/options.txt b/runtime/doc/options.txt
index 0322fae991..0c964ae519 100644
--- a/runtime/doc/options.txt
+++ b/runtime/doc/options.txt
@@ -1277,6 +1277,9 @@ A jump table for the options with a short description can be found at |Q_op|.
 			or list of locations |:lwindow|
 	  help		help buffer (you are not supposed to set this
 			manually)
+	  terminal	terminal buffer, this is set automatically when a 
+			terminal is created. See |nvim-terminal-emulator| for
+			more information.
 
 	This option is used together with 'bufhidden' and 'swapfile' to
 	specify special kinds of buffers.   See |special-buffers|.
diff --git a/runtime/doc/various.txt b/runtime/doc/various.txt
index 7797b02ba8..d138fcf456 100644
--- a/runtime/doc/various.txt
+++ b/runtime/doc/various.txt
@@ -224,6 +224,16 @@ g8			Print the hex values of the bytes used in the
 						  *:sh* *:shell* *E371* *E360*
 :sh[ell]		Removed. |vim-differences| {Nvim}
 
+						  *:term* *:terminal*
+:term[inal][!] {cmd}	Spawns {command} using the current value of 'shell'
+			in a new terminal buffer.  This is equivalent to: >
+
+			      :enew | call termopen('{cmd}') | startinsert
+<
+			Like |:enew|, it will fail if the current buffer is
+			modified, but can be forced with "!".  See |termopen()|
+			and |nvim-terminal-emulator| for more information.
+
 							*:!cmd* *:!* *E34*
 :!{cmd}			Execute {cmd} with the shell. See also 'shell'.