Merge branch 'master' into add-scroll-events

20 files changed, 302 insertions, 112 deletions

diff --git a/runtime/autoload/provider/clipboard.vim b/runtime/autoload/provider/clipboard.vim
index 275d18a5a9..c2195fa02d 100644
--- a/runtime/autoload/provider/clipboard.vim
+++ b/runtime/autoload/provider/clipboard.vim
@@ -91,19 +91,19 @@ function! provider#clipboard#Executable() abort
     let s:paste['*'] = s:paste['+']
     let s:cache_enabled = 0
     return 'pbcopy'
-  elseif exists('$WAYLAND_DISPLAY') && executable('wl-copy') && executable('wl-paste')
+  elseif !empty($WAYLAND_DISPLAY) && executable('wl-copy') && executable('wl-paste')
     let s:copy['+'] = ['wl-copy', '--foreground', '--type', 'text/plain']
     let s:paste['+'] = ['wl-paste', '--no-newline']
     let s:copy['*'] = ['wl-copy', '--foreground', '--primary', '--type', 'text/plain']
     let s:paste['*'] = ['wl-paste', '--no-newline', '--primary']
     return 'wl-copy'
-  elseif exists('$DISPLAY') && executable('xclip')
+  elseif !empty($DISPLAY) && executable('xclip')
     let s:copy['+'] = ['xclip', '-quiet', '-i', '-selection', 'clipboard']
     let s:paste['+'] = ['xclip', '-o', '-selection', 'clipboard']
     let s:copy['*'] = ['xclip', '-quiet', '-i', '-selection', 'primary']
     let s:paste['*'] = ['xclip', '-o', '-selection', 'primary']
     return 'xclip'
-  elseif exists('$DISPLAY') && executable('xsel') && s:cmd_ok('xsel -o -b')
+  elseif !empty($DISPLAY) && executable('xsel') && s:cmd_ok('xsel -o -b')
     let s:copy['+'] = ['xsel', '--nodetach', '-i', '-b']
     let s:paste['+'] = ['xsel', '-o', '-b']
     let s:copy['*'] = ['xsel', '--nodetach', '-i', '-p']
@@ -132,7 +132,7 @@ function! provider#clipboard#Executable() abort
     let s:copy['*'] = s:copy['+']
     let s:paste['*'] = s:paste['+']
     return 'win32yank'
-  elseif exists('$TMUX') && executable('tmux')
+  elseif !empty($TMUX) && executable('tmux')
     let s:copy['+'] = ['tmux', 'load-buffer', '-']
     let s:paste['+'] = ['tmux', 'save-buffer', '-']
     let s:copy['*'] = s:copy['+']
diff --git a/runtime/doc/api.txt b/runtime/doc/api.txt
index 083128ab80..0c726ddd86 100644
--- a/runtime/doc/api.txt
+++ b/runtime/doc/api.txt
@@ -1783,6 +1783,17 @@ nvim_buf_del_var({buffer}, {name})                        *nvim_buf_del_var()*
                     {buffer}  Buffer handle, or 0 for current buffer
                     {name}    Variable name
 
+nvim_buf_delete({buffer}, {opts})                          *nvim_buf_delete()*
+                Deletes the buffer. See |:bwipeout|
+
+                Parameters: ~
+                    {buffer}  Buffer handle, or 0 for current buffer
+                    {opts}    Optional parameters. Keys:
+                              • force: Force deletion and ignore unsaved
+                                changes.
+                              • unload: Unloaded only, do not delete. See
+                                |:bunload|
+
 nvim_buf_detach({buffer})                                  *nvim_buf_detach()*
                 Deactivates buffer-update events on the channel.
 
diff --git a/runtime/doc/autocmd.txt b/runtime/doc/autocmd.txt
index 75cdc71039..66d15d446a 100644
--- a/runtime/doc/autocmd.txt
+++ b/runtime/doc/autocmd.txt
@@ -700,9 +700,14 @@ InsertEnter			Just before starting Insert mode.  Also for
 				The cursor is restored afterwards.  If you do
 				not want that set |v:char| to a non-empty
 				string.
+							*InsertLeavePre*
+InsertLeavePre			Just before leaving Insert mode.  Also when
+				using CTRL-O |i_CTRL-O|.  Be caseful not to
+				change mode or use `:normal`, it will likely
+				cause trouble.
 							*InsertLeave*
-InsertLeave			When leaving Insert mode.  Also when using
-				CTRL-O |i_CTRL-O|.  But not for |i_CTRL-C|.
+InsertLeave			Just after leaving Insert mode.  Also when
+				using CTRL-O |i_CTRL-O|.  But not for |i_CTRL-C|.
 							*MenuPopup*
 MenuPopup			Just before showing the popup menu (under the
 				right mouse button).  Useful for adjusting the
diff --git a/runtime/doc/change.txt b/runtime/doc/change.txt
index dcebbc524c..5c67359002 100644
--- a/runtime/doc/change.txt
+++ b/runtime/doc/change.txt
@@ -1615,6 +1615,10 @@ B	When joining lines, don't insert a space between two multi-byte
 	characters.  Overruled by the 'M' flag.
 1	Don't break a line after a one-letter word.  It's broken before it
 	instead (if possible).
+]	Respect textwidth rigorously. With this flag set, no line can be
+	longer than textwidth, unless line-break-prohibition rules make this
+	impossible.  Mainly for CJK scripts and works only if 'encoding' is
+	"utf-8".
 j	Where it makes sense, remove a comment leader when joining lines.  For
 	example, joining:
 		int i;   // the index ~
diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index a710ef5d14..800de63a55 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -2336,6 +2336,7 @@ repeat({expr}, {count})		String	repeat {expr} {count} times
 resolve({filename})		String	get filename a shortcut points to
 reverse({list})			List	reverse {list} in-place
 round({expr})			Float	round off {expr}
+rubyeval({expr})		any	evaluate |Ruby| expression
 rpcnotify({channel}, {event}[, {args}...])
 				Sends an |RPC| notification to {channel}
 rpcrequest({channel}, {method}[, {args}...])
@@ -7070,6 +7071,17 @@ rpcstart({prog}[, {argv}])					*rpcstart()*
 <		with >
 			:let id = jobstart(['prog', 'arg1', 'arg2'], {'rpc': v:true})
 
+rubyeval({expr})					*rubyeval()*
+		Evaluate Ruby expression {expr} and return its result
+		converted to Vim data structures.
+		Numbers, floats and strings are returned as they are (strings
+		are copied though).
+		Arrays are represented as Vim |List| type.
+		Hashes are represented as Vim |Dictionary| type.
+		Other objects are represented as strings resulted from their
+		"Object#to_s" method.
+		{only available when compiled with the |+ruby| feature}
+
 screenattr({row}, {col})					*screenattr()*
 		Like |screenchar()|, but return the attribute.  This is a rather
 		arbitrary number that can only be used to compare to the
diff --git a/runtime/doc/if_ruby.txt b/runtime/doc/if_ruby.txt
index 87047712f3..c8d2409549 100644
--- a/runtime/doc/if_ruby.txt
+++ b/runtime/doc/if_ruby.txt
@@ -185,4 +185,13 @@ $curwin		The current window object.
 $curbuf		The current buffer object.
 
 ==============================================================================
+6. rubyeval() Vim function				*ruby-rubyeval*
+
+To facilitate bi-directional interface, you can use |rubyeval()| function to
+evaluate Ruby expressions and pass their values to Vim script.
+
+The Ruby value "true", "false" and "nil" are converted to v:true, v:false and
+v:null, respectively.
+
+==============================================================================
  vim:tw=78:ts=8:noet:ft=help:norl:
diff --git a/runtime/doc/indent.txt b/runtime/doc/indent.txt
index 1df0331239..f2278f8453 100644
--- a/runtime/doc/indent.txt
+++ b/runtime/doc/indent.txt
@@ -566,9 +566,15 @@ The examples below assume a 'shiftwidth' of 4.
 	      with "#" does not work.
 
 
+	PN    When N is non-zero recognize C pragmas, and indent them like any
+	      other code; does not concern other preprocessor directives.
+	      When N is zero (default): don't recognize C pragmas, treating
+	      them like every other preprocessor directive.
+
+
 The defaults, spelled out in full, are:
 	cinoptions=>s,e0,n0,f0,{0,}0,^0,L-1,:s,=s,l0,b0,gs,hs,N0,E0,ps,ts,is,+s,
-			c3,C0,/0,(2s,us,U0,w0,W0,k0,m0,j0,J0,)20,*70,#0
+			c3,C0,/0,(2s,us,U0,w0,W0,k0,m0,j0,J0,)20,*70,#0,P0
 
 Vim puts a line in column 1 if:
 - It starts with '#' (preprocessor directives), if 'cinkeys' contains '#0'.
diff --git a/runtime/doc/lsp.txt b/runtime/doc/lsp.txt
index 5a61017b1d..33d65406a1 100644
--- a/runtime/doc/lsp.txt
+++ b/runtime/doc/lsp.txt
@@ -488,8 +488,8 @@ get_active_clients()                            *vim.lsp.get_active_clients()*
                     Table of |vim.lsp.client| objects
 
 get_client_by_id({client_id})                     *vim.lsp.get_client_by_id()*
-                Gets an active client by id, or nil if the id is invalid or
-                the client is not yet initialized.
+		Gets a client by id, or nil if the id is invalid.
+		The returned client may not yet be fully initialized.
 
                 Parameters: ~
                     {client_id}  client id number
diff --git a/runtime/doc/options.txt b/runtime/doc/options.txt
index beb5e9f4c2..b83d2c4484 100644
--- a/runtime/doc/options.txt
+++ b/runtime/doc/options.txt
@@ -1386,6 +1386,21 @@ A jump table for the options with a short description can be found at |Q_op|.
 	This option cannot be set from a |modeline| or in the |sandbox|, for
 	security reasons.
 
+						*'completeslash'* *'csl'*
+'completeslash' 'csl'	string	(default: "")
+			local to buffer
+			{not in Vi}  {only for MS-Windows}
+	When this option is set it overrules 'shellslash' for completion:
+	- When this option is set to "slash", a forward slash is used for path
+	  completion in insert mode. This is useful when editing HTML tag, or
+	  Makefile with 'noshellslash' on Windows.
+	- When this option is set to "backslash", backslash is used. This is
+	  useful when editing a batch file with 'shellslash' set on Windows.
+	- When this option is empty, same character is used as for
+	  'shellslash'.
+	For Insert mode completion the buffer-local value is used.  For
+	command line completion the global value is used.
+
 						*'completeopt'* *'cot'*
 'completeopt' 'cot'	string	(default: "menu,preview")
 			global
@@ -2903,6 +2918,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 								*'go-c'*
 	  'c'	Use console dialogs instead of popup dialogs for simple
 		choices.
+								*'go-d'*
+	  'd'	Use dark theme variant if available.
 								*'go-e'*
 	  'e'	Add tab pages when indicated with 'showtabline'.
 		'guitablabel' can be used to change the text in the labels.
@@ -5289,7 +5306,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	'shellslash' only works when a backslash can be used as a path
 	separator.  To test if this is so use: >
 		if exists('+shellslash')
-<
+<	Also see 'completeslash'.
+
 			*'shelltemp'* *'stmp'* *'noshelltemp'* *'nostmp'*
 'shelltemp' 'stmp'	boolean	(Vim default on, Vi default off)
 			global
@@ -5779,7 +5797,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 	normal text.  Each status line item is of the form:
 	  %-0{minwid}.{maxwid}{item}
 	All fields except the {item} are optional.  A single percent sign can
-	be given as "%%".  Up to 80 items can be specified.  *E541*
+	be given as "%%".
 
 	When the option starts with "%!" then it is used as an expression,
 	evaluated and the result is used as the option value.  Example: >
diff --git a/runtime/doc/treesitter.txt b/runtime/doc/treesitter.txt
index 7f644486f7..aaf13d1640 100644
--- a/runtime/doc/treesitter.txt
+++ b/runtime/doc/treesitter.txt
@@ -136,6 +136,15 @@ tsnode:has_error()					*tsnode:has_error()*
 tsnode:sexpr()						*tsnode:sexpr()*
 	Get an S-expression representing the node as a string.
 
+tsnode:id()							*tsnode:id()*
+	Get an unique identier for the node inside its own tree.
+
+	No guarantees are made about this identifer's internal representation,
+	except for being a primitive lua type with value equality (so not a table).
+	Presently it is a (non-printable) string.
+
+	NB: the id is not guaranteed to be unique for nodes from different trees.
+
 tsnode:descendant_for_range({start_row}, {start_col}, {end_row}, {end_col})
 						*tsnode:descendant_for_range()*
 	Get the smallest node within this node that spans the given range of
diff --git a/runtime/doc/vim_diff.txt b/runtime/doc/vim_diff.txt
index ae60c1c5e8..1fcb6611b4 100644
--- a/runtime/doc/vim_diff.txt
+++ b/runtime/doc/vim_diff.txt
@@ -450,6 +450,9 @@ Eval:
   *js_decode()*
   *v:none* (used by Vim to represent JavaScript "undefined"); use |v:null| instead.
 
+Events:
+  *SigUSR1* Use |Signal| to detect `SIGUSR1` signal instead.
+
 Highlight groups:
   *hl-StatusLineTerm* *hl-StatusLineTermNC* are unnecessary because Nvim
     supports 'winhighlight' window-local highlights.
diff --git a/runtime/ftplugin/markdown.vim b/runtime/ftplugin/markdown.vim
index 277ba94e8b..fc1d9e068b 100644
--- a/runtime/ftplugin/markdown.vim
+++ b/runtime/ftplugin/markdown.vim
@@ -1,7 +1,7 @@
 " Vim filetype plugin
 " Language:		Markdown
 " Maintainer:		Tim Pope <vimNOSPAM@tpope.org>
-" Last Change:		2016 Aug 29
+" Last Change:		2019 Dec 05
 
 if exists("b:did_ftplugin")
   finish
@@ -9,7 +9,7 @@ endif
 
 runtime! ftplugin/html.vim ftplugin/html_*.vim ftplugin/html/*.vim
 
-setlocal comments=fb:*,fb:-,fb:+,n:> commentstring=>\ %s
+setlocal comments=fb:*,fb:-,fb:+,n:> commentstring=<!--%s-->
 setlocal formatoptions+=tcqln formatoptions-=r formatoptions-=o
 setlocal formatlistpat=^\\s*\\d\\+\\.\\s\\+\\\|^[-*+]\\s\\+\\\|^\\[^\\ze[^\\]]\\+\\]:
 
@@ -19,32 +19,56 @@ else
   let b:undo_ftplugin = "setl cms< com< fo< flp<"
 endif
 
-function! MarkdownFold()
+function! s:NotCodeBlock(lnum) abort
+  return synIDattr(synID(v:lnum, 1, 1), 'name') !=# 'markdownCode'
+endfunction
+
+function! MarkdownFold() abort
   let line = getline(v:lnum)
 
-  " Regular headers
-  let depth = match(line, '\(^#\+\)\@<=\( .*$\)\@=')
-  if depth > 0
-    return ">" . depth
+  if line =~# '^#\+ ' && s:NotCodeBlock(v:lnum)
+    return ">" . match(line, ' ')
   endif
 
-  " Setext style headings
   let nextline = getline(v:lnum + 1)
-  if (line =~ '^.\+$') && (nextline =~ '^=\+$')
+  if (line =~ '^.\+$') && (nextline =~ '^=\+$') && s:NotCodeBlock(v:lnum + 1)
     return ">1"
   endif
 
-  if (line =~ '^.\+$') && (nextline =~ '^-\+$')
+  if (line =~ '^.\+$') && (nextline =~ '^-\+$') && s:NotCodeBlock(v:lnum + 1)
     return ">2"
   endif
 
   return "="
 endfunction
 
+function! s:HashIndent(lnum) abort
+  let hash_header = matchstr(getline(a:lnum), '^#\{1,6}')
+  if len(hash_header)
+    return hash_header
+  else
+    let nextline = getline(a:lnum + 1)
+    if nextline =~# '^=\+\s*$'
+      return '#'
+    elseif nextline =~# '^-\+\s*$'
+      return '##'
+    endif
+  endif
+endfunction
+
+function! MarkdownFoldText() abort
+  let hash_indent = s:HashIndent(v:foldstart)
+  let title = substitute(getline(v:foldstart), '^#\+\s*', '', '')
+  let foldsize = (v:foldend - v:foldstart + 1)
+  let linecount = '['.foldsize.' lines]'
+  return hash_indent.' '.title.' '.linecount
+endfunction
+
 if has("folding") && exists("g:markdown_folding")
   setlocal foldexpr=MarkdownFold()
   setlocal foldmethod=expr
-  let b:undo_ftplugin .= " foldexpr< foldmethod<"
+  setlocal foldtext=MarkdownFoldText()
+  let b:undo_ftplugin .= " foldexpr< foldmethod< foldtext<"
 endif
 
 " vim:set sw=2:
diff --git a/runtime/lua/vim/lsp.lua b/runtime/lua/vim/lsp.lua
index 585528dd5a..1a0015e2db 100644
--- a/runtime/lua/vim/lsp.lua
+++ b/runtime/lua/vim/lsp.lua
@@ -25,6 +25,27 @@ local lsp = {
   -- format_rpc_error = lsp_rpc.format_rpc_error;
 }
 
+-- maps request name to the required resolved_capability in the client.
+lsp._request_name_to_capability = {
+  ['textDocument/hover'] = 'hover';
+  ['textDocument/signatureHelp'] = 'signature_help';
+  ['textDocument/definition'] = 'goto_definition';
+  ['textDocument/implementation'] = 'implementation';
+  ['textDocument/declaration'] = 'declaration';
+  ['textDocument/typeDefinition'] = 'type_definition';
+  ['textDocument/documentSymbol'] = 'document_symbol';
+  ['textDocument/workspaceSymbol'] = 'workspace_symbol';
+  ['textDocument/prepareCallHierarchy'] = 'call_hierarchy';
+  ['textDocument/rename'] = 'rename';
+  ['textDocument/codeAction'] = 'code_action';
+  ['workspace/executeCommand'] = 'execute_command';
+  ['textDocument/references'] = 'find_references';
+  ['textDocument/rangeFormatting'] = 'document_range_formatting';
+  ['textDocument/formatting'] = 'document_formatting';
+  ['textDocument/completion'] = 'completion';
+  ['textDocument/documentHighlight'] = 'document_highlight';
+}
+
 -- TODO improve handling of scratch buffers with LSP attached.
 
 --@private
@@ -51,6 +72,16 @@ local function resolve_bufnr(bufnr)
 end
 
 --@private
+--- callback called by the client when trying to call a method that's not
+--- supported in any of the servers registered for the current buffer.
+--@param method (string) name of the method
+function lsp._unsupported_method(method)
+  local msg = string.format("method %s is not supported by any of the servers registered for the current buffer", method)
+  log.warn(msg)
+  return lsp.rpc_response_error(protocol.ErrorCodes.MethodNotFound, msg)
+end
+
+--@private
 --- Checks whether a given path is a directory.
 ---
 --@param filename (string) path to check
@@ -397,9 +428,8 @@ end
 --@param trace:  "off" | "messages" | "verbose" | nil passed directly to the language
 --- server in the initialize request. Invalid/empty values will default to "off"
 ---
---@returns Client id. |vim.lsp.get_client_by_id()| Note: client is only
---- available after it has been initialized, which may happen after a small
---- delay (or never if there is an error). Use `on_init` to do any actions once
+--@returns Client id. |vim.lsp.get_client_by_id()| Note: client may not be
+--- fully initialized. Use `on_init` to do any actions once
 --- the client has been initialized.
 function lsp.start_client(config)
   local cleaned_config = validate_client_config(config)
@@ -576,6 +606,15 @@ function lsp.start_client(config)
       -- These are the cleaned up capabilities we use for dynamically deciding
       -- when to send certain events to clients.
       client.resolved_capabilities = protocol.resolve_capabilities(client.server_capabilities)
+      client.supports_method = function(method)
+        local required_capability = lsp._request_name_to_capability[method]
+        -- if we don't know about the method, assume that the client supports it.
+        if not required_capability then
+          return true
+        end
+
+        return client.resolved_capabilities[required_capability]
+      end
       if config.on_init then
         local status, err = pcall(config.on_init, client, result)
         if not status then
@@ -599,19 +638,6 @@ function lsp.start_client(config)
   end
 
   --@private
-  --- Throws error for a method that is not supported by the current LSP
-  --- server.
-  ---
-  --@param method (string) an LSP method name not supported by the LSP server.
-  --@returns (error) a 'MethodNotFound' JSON-RPC error response.
-  local function unsupported_method(method)
-    local msg = "server doesn't support "..method
-    local _ = log.warn() and log.warn(msg)
-    err_message(msg)
-    return lsp.rpc_response_error(protocol.ErrorCodes.MethodNotFound, msg)
-  end
-
-  --@private
   --- Sends a request to the server.
   ---
   --- This is a thin wrapper around {client.rpc.request} with some additional
@@ -638,20 +664,6 @@ function lsp.start_client(config)
         or error(string.format("not found: %q request callback for client %q.", method, client.name))
     end
     local _ = log.debug() and log.debug(log_prefix, "client.request", client_id, method, params, callback, bufnr)
-    -- TODO keep these checks or just let it go anyway?
-    if (not client.resolved_capabilities.hover and method == 'textDocument/hover')
-      or (not client.resolved_capabilities.signature_help and method == 'textDocument/signatureHelp')
-      or (not client.resolved_capabilities.goto_definition and method == 'textDocument/definition')
-      or (not client.resolved_capabilities.implementation and method == 'textDocument/implementation')
-      or (not client.resolved_capabilities.declaration and method == 'textDocument/declaration')
-      or (not client.resolved_capabilities.type_definition and method == 'textDocument/typeDefinition')
-      or (not client.resolved_capabilities.document_symbol and method == 'textDocument/documentSymbol')
-      or (not client.resolved_capabilities.workspace_symbol and method == 'textDocument/workspaceSymbol')
-      or (not client.resolved_capabilities.call_hierarchy and method == 'textDocument/prepareCallHierarchy')
-    then
-      callback(unsupported_method(method), method, nil, client_id, bufnr)
-      return
-    end
     return rpc.request(method, params, function(err, result)
       callback(err, method, result, client_id, bufnr)
     end)
@@ -910,14 +922,14 @@ function lsp.buf_is_attached(bufnr, client_id)
   return (all_buffer_active_clients[bufnr] or {})[client_id] == true
 end
 
---- Gets an active client by id, or nil if the id is invalid or the
---- client is not yet initialized.
----
+--- Gets a client by id, or nil if the id is invalid.
+--- The returned client may not yet be fully initialized.
+--
 --@param client_id client id number
 ---
 --@returns |vim.lsp.client| object, or nil
 function lsp.get_client_by_id(client_id)
-  return active_clients[client_id]
+  return active_clients[client_id] or uninitialized_clients[client_id]
 end
 
 --- Stops a client(s).
@@ -998,16 +1010,32 @@ function lsp.buf_request(bufnr, method, params, callback)
     callback = { callback, 'f', true };
   }
   local client_request_ids = {}
-  for_each_buffer_client(bufnr, function(client, client_id, resolved_bufnr)
-    local request_success, request_id = client.request(method, params, callback, resolved_bufnr)
 
-    -- This could only fail if the client shut down in the time since we looked
-    -- it up and we did the request, which should be rare.
-    if request_success then
-      client_request_ids[client_id] = request_id
+  local method_supported = false
+  for_each_buffer_client(bufnr, function(client, client_id, resolved_bufnr)
+    if client.supports_method(method) then
+      method_supported = true
+      local request_success, request_id = client.request(method, params, callback, resolved_bufnr)
+
+      -- This could only fail if the client shut down in the time since we looked
+      -- it up and we did the request, which should be rare.
+      if request_success then
+        client_request_ids[client_id] = request_id
+      end
     end
   end)
 
+  -- if no clients support the given method, call the callback with the proper
+  -- error message.
+  if not method_supported then
+    local unsupported_err = lsp._unsupported_method(method)
+    local cb = callback or lsp.callbacks[method]
+    if cb then
+      cb(unsupported_err, method, bufnr)
+    end
+    return
+  end
+
   local function _cancel_all_requests()
     for client_id, request_id in pairs(client_request_ids) do
       local client = active_clients[client_id]
diff --git a/runtime/lua/vim/lsp/callbacks.lua b/runtime/lua/vim/lsp/callbacks.lua
index 4e7a8333a0..3270d1d2a9 100644
--- a/runtime/lua/vim/lsp/callbacks.lua
+++ b/runtime/lua/vim/lsp/callbacks.lua
@@ -82,18 +82,6 @@ M['textDocument/publishDiagnostics'] = function(_, _, result)
     return
   end
 
-  -- Unloaded buffers should not handle diagnostics.
-  --    When the buffer is loaded, we'll call on_attach, which sends textDocument/didOpen.
-  --    This should trigger another publish of the diagnostics.
-  --
-  -- In particular, this stops a ton of spam when first starting a server for current
-  -- unloaded buffers.
-  if not api.nvim_buf_is_loaded(bufnr) then
-    return
-  end
-
-  util.buf_clear_diagnostics(bufnr)
-
   -- https://microsoft.github.io/language-server-protocol/specifications/specification-current/#diagnostic
   -- The diagnostic's severity. Can be omitted. If omitted it is up to the
   -- client to interpret diagnostics as error, warning, info or hint.
@@ -104,7 +92,23 @@ M['textDocument/publishDiagnostics'] = function(_, _, result)
     end
   end
 
+  util.buf_clear_diagnostics(bufnr)
+
+  -- Always save the diagnostics, even if the buf is not loaded.
+  -- Language servers may report compile or build errors via diagnostics
+  -- Users should be able to find these, even if they're in files which
+  -- are not loaded.
   util.buf_diagnostics_save_positions(bufnr, result.diagnostics)
+
+  -- Unloaded buffers should not handle diagnostics.
+  --    When the buffer is loaded, we'll call on_attach, which sends textDocument/didOpen.
+  --    This should trigger another publish of the diagnostics.
+  --
+  -- In particular, this stops a ton of spam when first starting a server for current
+  -- unloaded buffers.
+  if not api.nvim_buf_is_loaded(bufnr) then
+    return
+  end
   util.buf_diagnostics_underline(bufnr, result.diagnostics)
   util.buf_diagnostics_virtual_text(bufnr, result.diagnostics)
   util.buf_diagnostics_signs(bufnr, result.diagnostics)
diff --git a/runtime/lua/vim/lsp/protocol.lua b/runtime/lua/vim/lsp/protocol.lua
index 4e926381e0..70862320c5 100644
--- a/runtime/lua/vim/lsp/protocol.lua
+++ b/runtime/lua/vim/lsp/protocol.lua
@@ -632,15 +632,18 @@ function protocol.make_client_capabilities()
 
         codeActionLiteralSupport = {
           codeActionKind = {
-            valueSet = {};
+            valueSet = vim.tbl_values(protocol.CodeActionKind);
           };
         };
       };
       completion = {
         dynamicRegistration = false;
         completionItem = {
+          -- Until we can actually expand snippet, move cursor and allow for true snippet experience,
+          -- this should be disabled out of the box.
+          -- However, users can turn this back on if they have a snippet plugin.
+          snippetSupport = false;
 
-          snippetSupport = true;
           commitCharactersSupport = false;
           preselectSupport = false;
           deprecatedSupport = false;
@@ -703,6 +706,10 @@ function protocol.make_client_capabilities()
         };
         hierarchicalDocumentSymbolSupport = true;
       };
+      rename = {
+        dynamicRegistration = false;
+        prepareSupport = true;
+      };
     };
     workspace = {
       symbol = {
@@ -914,6 +921,7 @@ function protocol.resolve_capabilities(server_capabilities)
       return nil, string.format("Invalid type for textDocumentSync: %q", type(textDocumentSync))
     end
   end
+  general_properties.completion = server_capabilities.completionProvider ~= nil
   general_properties.hover = server_capabilities.hoverProvider or false
   general_properties.goto_definition = server_capabilities.definitionProvider or false
   general_properties.find_references = server_capabilities.referencesProvider or false
@@ -923,14 +931,21 @@ function protocol.resolve_capabilities(server_capabilities)
   general_properties.document_formatting = server_capabilities.documentFormattingProvider or false
   general_properties.document_range_formatting = server_capabilities.documentRangeFormattingProvider or false
   general_properties.call_hierarchy = server_capabilities.callHierarchyProvider or false
+  general_properties.execute_command = server_capabilities.executeCommandProvider ~= nil
+
+  if server_capabilities.renameProvider == nil then
+    general_properties.rename = false
+  elseif type(server_capabilities.renameProvider) == 'boolean' then
+    general_properties.rename = server_capabilities.renameProvider
+  else
+    general_properties.rename = true
+  end
 
   if server_capabilities.codeActionProvider == nil then
     general_properties.code_action = false
-  elseif type(server_capabilities.codeActionProvider) == 'boolean' then
+  elseif type(server_capabilities.codeActionProvider) == 'boolean'
+    or type(server_capabilities.codeActionProvider) == 'table' then
     general_properties.code_action = server_capabilities.codeActionProvider
-  elseif type(server_capabilities.codeActionProvider) == 'table' then
-    -- TODO(ashkan) support CodeActionKind
-    general_properties.code_action = false
   else
     error("The server sent invalid codeActionProvider")
   end
diff --git a/runtime/lua/vim/lsp/rpc.lua b/runtime/lua/vim/lsp/rpc.lua
index 749a51fecc..17c411f952 100644
--- a/runtime/lua/vim/lsp/rpc.lua
+++ b/runtime/lua/vim/lsp/rpc.lua
@@ -42,13 +42,28 @@ local function is_dir(filename)
 end
 
 local NIL = vim.NIL
+
+--@private
+local recursive_convert_NIL
+recursive_convert_NIL = function(v, tbl_processed)
+  if v == NIL then
+    return nil
+  elseif not tbl_processed[v] and type(v) == 'table' then
+    tbl_processed[v] = true
+    return vim.tbl_map(function(x)
+      return recursive_convert_NIL(x, tbl_processed)
+    end, v)
+  end
+
+  return v
+end
+
 --@private
 --- Returns its argument, but converts `vim.NIL` to Lua `nil`.
 --@param v (any) Argument
 --@returns (any)
 local function convert_NIL(v)
-  if v == NIL then return nil end
-  return v
+  return recursive_convert_NIL(v, {})
 end
 
 --@private
diff --git a/runtime/lua/vim/lsp/util.lua b/runtime/lua/vim/lsp/util.lua
index b5f171a985..9ed19b938d 100644
--- a/runtime/lua/vim/lsp/util.lua
+++ b/runtime/lua/vim/lsp/util.lua
@@ -214,13 +214,16 @@ end
 function M.apply_text_document_edit(text_document_edit)
   local text_document = text_document_edit.textDocument
   local bufnr = vim.uri_to_bufnr(text_document.uri)
-  if text_document.version then
-    -- `VersionedTextDocumentIdentifier`s version may be null https://microsoft.github.io/language-server-protocol/specification#versionedTextDocumentIdentifier
-    if text_document.version ~= vim.NIL and M.buf_versions[bufnr] ~= nil and M.buf_versions[bufnr] > text_document.version then
-      print("Buffer ", text_document.uri, " newer than edits.")
-      return
-    end
+
+  -- `VersionedTextDocumentIdentifier`s version may be null
+  --  https://microsoft.github.io/language-server-protocol/specification#versionedTextDocumentIdentifier
+  if text_document.version
+      and M.buf_versions[bufnr]
+      and M.buf_versions[bufnr] > text_document.version then
+    print("Buffer ", text_document.uri, " newer than edits.")
+    return
   end
+
   M.apply_text_edits(text_document_edit.edits, bufnr)
 end
 
@@ -532,7 +535,7 @@ function M.convert_signature_help_to_markdown_lines(signature_help)
       }
       --]=]
       -- TODO highlight parameter
-      if parameter.documentation and parameter.documentation ~= vim.NIL then
+      if parameter.documentation then
         M.convert_input_to_markdown_lines(parameter.documentation, contents)
       end
     end
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index decde08019..6714bb6354 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -157,7 +157,7 @@ local function on_line_impl(self, buf, line)
       a.nvim_buf_set_extmark(buf, ns, start_row, start_col,
                              { end_line = end_row, end_col = end_col,
                                hl_group = hl,
-                               ephemeral = true
+                               ephemeral = true,
                               })
     end
     if start_row > line then
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 2903c5905c..cc7dc2656d 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -25,7 +25,7 @@ local function filter_files(file_list)
     end
   end
 
-  return { main, unpack(after) }
+  return main and { main, unpack(after) } or after
 end
 
 local function runtime_query_path(lang, query_name)
diff --git a/runtime/syntax/markdown.vim b/runtime/syntax/markdown.vim
index 1955a7443e..17b61c2fa4 100644
--- a/runtime/syntax/markdown.vim
+++ b/runtime/syntax/markdown.vim
@@ -2,7 +2,7 @@
 " Language:     Markdown
 " Maintainer:   Tim Pope <vimNOSPAM@tpope.org>
 " Filenames:    *.markdown
-" Last Change:  2016 Aug 29
+" Last Change:  2020 Jan 14
 
 if exists("b:current_syntax")
   finish
@@ -18,37 +18,46 @@ unlet! b:current_syntax
 if !exists('g:markdown_fenced_languages')
   let g:markdown_fenced_languages = []
 endif
+let s:done_include = {}
 for s:type in map(copy(g:markdown_fenced_languages),'matchstr(v:val,"[^=]*$")')
+  if has_key(s:done_include, matchstr(s:type,'[^.]*'))
+    continue
+  endif
   if s:type =~ '\.'
     let b:{matchstr(s:type,'[^.]*')}_subtype = matchstr(s:type,'\.\zs.*')
   endif
   exe 'syn include @markdownHighlight'.substitute(s:type,'\.','','g').' syntax/'.matchstr(s:type,'[^.]*').'.vim'
   unlet! b:current_syntax
+  let s:done_include[matchstr(s:type,'[^.]*')] = 1
 endfor
 unlet! s:type
+unlet! s:done_include
 
-syn sync minlines=10
+if !exists('g:markdown_minlines')
+  let g:markdown_minlines = 50
+endif
+execute 'syn sync minlines=' . g:markdown_minlines
 syn case ignore
 
-syn match markdownValid '[<>]\c[a-z/$!]\@!'
-syn match markdownValid '&\%(#\=\w*;\)\@!'
+syn match markdownValid '[<>]\c[a-z/$!]\@!' transparent contains=NONE
+syn match markdownValid '&\%(#\=\w*;\)\@!' transparent contains=NONE
 
 syn match markdownLineStart "^[<@]\@!" nextgroup=@markdownBlock,htmlSpecialChar
 
 syn cluster markdownBlock contains=markdownH1,markdownH2,markdownH3,markdownH4,markdownH5,markdownH6,markdownBlockquote,markdownListMarker,markdownOrderedListMarker,markdownCodeBlock,markdownRule
-syn cluster markdownInline contains=markdownLineBreak,markdownLinkText,markdownItalic,markdownBold,markdownCode,markdownEscape,@htmlTop,markdownError
+syn cluster markdownInline contains=markdownLineBreak,markdownLinkText,markdownItalic,markdownBold,markdownCode,markdownEscape,@htmlTop,markdownError,markdownValid
 
 syn match markdownH1 "^.\+\n=\+$" contained contains=@markdownInline,markdownHeadingRule,markdownAutomaticLink
 syn match markdownH2 "^.\+\n-\+$" contained contains=@markdownInline,markdownHeadingRule,markdownAutomaticLink
 
 syn match markdownHeadingRule "^[=-]\+$" contained
 
-syn region markdownH1 matchgroup=markdownHeadingDelimiter start="##\@!"      end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
-syn region markdownH2 matchgroup=markdownHeadingDelimiter start="###\@!"     end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
-syn region markdownH3 matchgroup=markdownHeadingDelimiter start="####\@!"    end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
-syn region markdownH4 matchgroup=markdownHeadingDelimiter start="#####\@!"   end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
-syn region markdownH5 matchgroup=markdownHeadingDelimiter start="######\@!"  end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
-syn region markdownH6 matchgroup=markdownHeadingDelimiter start="#######\@!" end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
+syn region markdownH1 matchgroup=markdownH1Delimiter start="##\@!"      end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
+syn region markdownH2 matchgroup=markdownH2Delimiter start="###\@!"     end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
+syn region markdownH3 matchgroup=markdownH3Delimiter start="####\@!"    end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
+syn region markdownH4 matchgroup=markdownH4Delimiter start="#####\@!"   end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
+syn region markdownH5 matchgroup=markdownH5Delimiter start="######\@!"  end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
+syn region markdownH6 matchgroup=markdownH6Delimiter start="#######\@!" end="#*\s*$" keepend oneline contains=@markdownInline,markdownAutomaticLink contained
 
 syn match markdownBlockquote ">\%(\s\|$\)" contained nextgroup=@markdownBlock
 
@@ -70,31 +79,40 @@ syn region markdownUrlTitle matchgroup=markdownUrlTitleDelimiter start=+"+ end=+
 syn region markdownUrlTitle matchgroup=markdownUrlTitleDelimiter start=+'+ end=+'+ keepend contained
 syn region markdownUrlTitle matchgroup=markdownUrlTitleDelimiter start=+(+ end=+)+ keepend contained
 
-syn region markdownLinkText matchgroup=markdownLinkTextDelimiter start="!\=\[\%(\_[^]]*]\%( \=[[(]\)\)\@=" end="\]\%( \=[[(]\)\@=" nextgroup=markdownLink,markdownId skipwhite contains=@markdownInline,markdownLineStart
+syn region markdownLinkText matchgroup=markdownLinkTextDelimiter start="!\=\[\%(\%(\_[^][]\|\[\_[^][]*\]\)*]\%( \=[[(]\)\)\@=" end="\]\%( \=[[(]\)\@=" nextgroup=markdownLink,markdownId skipwhite contains=@markdownInline,markdownLineStart
 syn region markdownLink matchgroup=markdownLinkDelimiter start="(" end=")" contains=markdownUrl keepend contained
 syn region markdownId matchgroup=markdownIdDelimiter start="\[" end="\]" keepend contained
 syn region markdownAutomaticLink matchgroup=markdownUrlDelimiter start="<\%(\w\+:\|[[:alnum:]_+-]\+@\)\@=" end=">" keepend oneline
 
-let s:concealends = has('conceal') ? ' concealends' : ''
-exe 'syn region markdownItalic matchgroup=markdownItalicDelimiter start="\S\@<=\*\|\*\S\@=" end="\S\@<=\*\|\*\S\@=" keepend contains=markdownLineStart' . s:concealends
-exe 'syn region markdownItalic matchgroup=markdownItalicDelimiter start="\S\@<=_\|_\S\@=" end="\S\@<=_\|_\S\@=" keepend contains=markdownLineStart' . s:concealends
-exe 'syn region markdownBold matchgroup=markdownBoldDelimiter start="\S\@<=\*\*\|\*\*\S\@=" end="\S\@<=\*\*\|\*\*\S\@=" keepend contains=markdownLineStart,markdownItalic' . s:concealends
-exe 'syn region markdownBold matchgroup=markdownBoldDelimiter start="\S\@<=__\|__\S\@=" end="\S\@<=__\|__\S\@=" keepend contains=markdownLineStart,markdownItalic' . s:concealends
-exe 'syn region markdownBoldItalic matchgroup=markdownBoldItalicDelimiter start="\S\@<=\*\*\*\|\*\*\*\S\@=" end="\S\@<=\*\*\*\|\*\*\*\S\@=" keepend contains=markdownLineStart' . s:concealends
-exe 'syn region markdownBoldItalic matchgroup=markdownBoldItalicDelimiter start="\S\@<=___\|___\S\@=" end="\S\@<=___\|___\S\@=" keepend contains=markdownLineStart' . s:concealends
+let s:concealends = ''
+if has('conceal') && get(g:, 'markdown_syntax_conceal', 1) == 1
+  let s:concealends = ' concealends'
+endif
+exe 'syn region markdownItalic matchgroup=markdownItalicDelimiter start="\S\@<=\*\|\*\S\@=" end="\S\@<=\*\|\*\S\@=" skip="\\\*" contains=markdownLineStart,@Spell' . s:concealends
+exe 'syn region markdownItalic matchgroup=markdownItalicDelimiter start="\w\@<!_\S\@=" end="\S\@<=_\w\@!" skip="\\_" contains=markdownLineStart,@Spell' . s:concealends
+exe 'syn region markdownBold matchgroup=markdownBoldDelimiter start="\S\@<=\*\*\|\*\*\S\@=" end="\S\@<=\*\*\|\*\*\S\@=" skip="\\\*" contains=markdownLineStart,markdownItalic,@Spell' . s:concealends
+exe 'syn region markdownBold matchgroup=markdownBoldDelimiter start="\w\@<!__\S\@=" end="\S\@<=__\w\@!" skip="\\_" contains=markdownLineStart,markdownItalic,@Spell' . s:concealends
+exe 'syn region markdownBoldItalic matchgroup=markdownBoldItalicDelimiter start="\S\@<=\*\*\*\|\*\*\*\S\@=" end="\S\@<=\*\*\*\|\*\*\*\S\@=" skip="\\\*" contains=markdownLineStart,@Spell' . s:concealends
+exe 'syn region markdownBoldItalic matchgroup=markdownBoldItalicDelimiter start="\w\@<!___\S\@=" end="\S\@<=___\w\@!" skip="\\_" contains=markdownLineStart,@Spell' . s:concealends
 
 syn region markdownCode matchgroup=markdownCodeDelimiter start="`" end="`" keepend contains=markdownLineStart
 syn region markdownCode matchgroup=markdownCodeDelimiter start="`` \=" end=" \=``" keepend contains=markdownLineStart
-syn region markdownCode matchgroup=markdownCodeDelimiter start="^\s*```.*$" end="^\s*```\ze\s*$" keepend
+syn region markdownCode matchgroup=markdownCodeDelimiter start="^\s*````*.*$" end="^\s*````*\ze\s*$" keepend
 
 syn match markdownFootnote "\[^[^\]]\+\]"
 syn match markdownFootnoteDefinition "^\[^[^\]]\+\]:"
 
 if main_syntax ==# 'markdown'
+  let s:done_include = {}
   for s:type in g:markdown_fenced_languages
-    exe 'syn region markdownHighlight'.substitute(matchstr(s:type,'[^=]*$'),'\..*','','').' matchgroup=markdownCodeDelimiter start="^\s*```\s*'.matchstr(s:type,'[^=]*').'\>.*$" end="^\s*```\ze\s*$" keepend contains=@markdownHighlight'.substitute(matchstr(s:type,'[^=]*$'),'\.','','g')
+    if has_key(s:done_include, matchstr(s:type,'[^.]*'))
+      continue
+    endif
+    exe 'syn region markdownHighlight'.substitute(matchstr(s:type,'[^=]*$'),'\..*','','').' matchgroup=markdownCodeDelimiter start="^\s*````*\s*\%({.\{-}\.\)\='.matchstr(s:type,'[^=]*').'}\=\S\@!.*$" end="^\s*````*\ze\s*$" keepend contains=@markdownHighlight'.substitute(matchstr(s:type,'[^=]*$'),'\.','','g') . s:concealends
+    let s:done_include[matchstr(s:type,'[^.]*')] = 1
   endfor
   unlet! s:type
+  unlet! s:done_include
 endif
 
 syn match markdownEscape "\\[][\\`*_{}()<>#+.!-]"
@@ -107,6 +125,12 @@ hi def link markdownH4                    htmlH4
 hi def link markdownH5                    htmlH5
 hi def link markdownH6                    htmlH6
 hi def link markdownHeadingRule           markdownRule
+hi def link markdownH1Delimiter           markdownHeadingDelimiter
+hi def link markdownH2Delimiter           markdownHeadingDelimiter
+hi def link markdownH3Delimiter           markdownHeadingDelimiter
+hi def link markdownH4Delimiter           markdownHeadingDelimiter
+hi def link markdownH5Delimiter           markdownHeadingDelimiter
+hi def link markdownH6Delimiter           markdownHeadingDelimiter
 hi def link markdownHeadingDelimiter      Delimiter
 hi def link markdownOrderedListMarker     markdownListMarker
 hi def link markdownListMarker            htmlTagName