Followup improvements to LSP (#11430)

Addressing things brought up in https://github.com/neovim/neovim/issues/11389 and beyond.

- Bugfix for empty dictionary serialization.
- Reduce markdown previews of code.
- Refactor method triggers to `lsp.buf.*` methods
- Switch to v:lua and get rid of vim interface.
- Get rid of filetype config in favor of something from https://github.com/neovim/nvim-lsp
- Switch error messages to something which doesn't require `ENTER` because if an LSP goes crazy, it'll block neovim.
- Rename `builtin_callbacks` to `default_callbacks`
- Resolve callback at time of calling using default_callbacks instead of at client creation
- Make hover/signatureHelp preview focusable so you can mess with it.
- Add vim.lsp.util.set_qflist and vim.lsp.util.set_loclist and vim.lsp.util.locations_to_items().
- Add apply_textedit and tests which enables a new class of features.
- Fix column offsets in character and bytes in vim.lsp.buf to be correct.

New methods:
- Add textDocument/references under vim.lsp.buf.references()
- Finish textDocument/rename
- Finish textDocument/rangeFormatting and textDocument/format

9 files changed, 737 insertions, 563 deletions

diff --git a/runtime/lua/vim/lsp.lua b/runtime/lua/vim/lsp.lua
index 0c1a145c04..928d19177a 100644
--- a/runtime/lua/vim/lsp.lua
+++ b/runtime/lua/vim/lsp.lua
@@ -1,9 +1,10 @@
-local builtin_callbacks = require 'vim.lsp.builtin_callbacks'
+local default_callbacks = require 'vim.lsp.default_callbacks'
 local log = require 'vim.lsp.log'
 local lsp_rpc = require 'vim.lsp.rpc'
 local protocol = require 'vim.lsp.protocol'
 local util = require 'vim.lsp.util'
 
+local vim = vim
 local nvim_err_writeln, nvim_buf_get_lines, nvim_command, nvim_buf_get_option
   = vim.api.nvim_err_writeln, vim.api.nvim_buf_get_lines, vim.api.nvim_command, vim.api.nvim_buf_get_option
 local uv = vim.loop
@@ -12,7 +13,8 @@ local validate = vim.validate
 
 local lsp = {
   protocol = protocol;
-  builtin_callbacks = builtin_callbacks;
+  default_callbacks = default_callbacks;
+  buf = require'vim.lsp.buf';
   util = util;
   -- Allow raw RPC access.
   rpc = lsp_rpc;
@@ -25,6 +27,11 @@ local lsp = {
 
 -- TODO improve handling of scratch buffers with LSP attached.
 
+local function err_message(...)
+  nvim_err_writeln(table.concat(vim.tbl_flatten{...}))
+  nvim_command("redraw")
+end
+
 local function resolve_bufnr(bufnr)
   validate { bufnr = { bufnr, 'n', true } }
   if bufnr == nil or bufnr == 0 then
@@ -92,11 +99,7 @@ local function for_each_buffer_client(bufnr, callback)
     return
   end
   for client_id in pairs(client_ids) do
-    -- This is unlikely to happen. Could only potentially happen in a race
-    -- condition between literally a single statement.
-    -- We could skip this error, but let's error for now.
     local client = active_clients[client_id]
-        -- or error(string.format("Client %d has already shut down.", client_id))
     if client then
       callback(client, client_id)
     end
@@ -154,13 +157,13 @@ local function validate_client_config(config)
     root_dir        = { config.root_dir, is_dir, "directory" };
     callbacks       = { config.callbacks, "t", true };
     capabilities    = { config.capabilities, "t", true };
-    -- cmd             = { config.cmd, "s", false };
     cmd_cwd         = { config.cmd_cwd, optional_validator(is_dir), "directory" };
     cmd_env         = { config.cmd_env, "f", true };
     name            = { config.name, 's', true };
     on_error        = { config.on_error, "f", true };
     on_exit         = { config.on_exit, "f", true };
     on_init         = { config.on_init, "f", true };
+    before_init     = { config.before_init, "f", true };
     offset_encoding = { config.offset_encoding, "s", true };
   }
   local cmd, cmd_args = validate_command(config.cmd)
@@ -261,6 +264,12 @@ end
 -- possible errors. `vim.lsp.client_errors[code]` can be used to retrieve a
 -- human understandable string.
 --
+-- before_init(initialize_params, config): A function which is called *before*
+-- the request `initialize` is completed. `initialize_params` contains
+-- the parameters we are sending to the server and `config` is the config that
+-- was passed to `start_client()` for convenience. You can use this to modify
+-- parameters before they are sent.
+--
 -- on_init(client, initialize_result): A function which is called after the
 -- request `initialize` is completed. `initialize_result` contains
 -- `capabilities` and anything else the server may send. For example, `clangd`
@@ -290,19 +299,19 @@ function lsp.start_client(config)
 
   local client_id = next_client_id()
 
-  local callbacks = tbl_extend("keep", config.callbacks or {}, builtin_callbacks)
-  -- Copy metatable if it has one.
-  if config.callbacks and config.callbacks.__metatable then
-    setmetatable(callbacks, getmetatable(config.callbacks))
-  end
+  local callbacks = config.callbacks or {}
   local name = config.name or tostring(client_id)
   local log_prefix = string.format("LSP[%s]", name)
 
   local handlers = {}
 
+  local function resolve_callback(method)
+    return callbacks[method] or default_callbacks[method]
+  end
+
   function handlers.notification(method, params)
     local _ = log.debug() and log.debug('notification', method, params)
-    local callback = callbacks[method]
+    local callback = resolve_callback(method)
     if callback then
       -- Method name is provided here for convenience.
       callback(nil, method, params, client_id)
@@ -311,7 +320,7 @@ function lsp.start_client(config)
 
   function handlers.server_request(method, params)
     local _ = log.debug() and log.debug('server_request', method, params)
-    local callback = callbacks[method]
+    local callback = resolve_callback(method)
     if callback then
       local _ = log.debug() and log.debug("server_request: found callback for", method)
       return callback(nil, method, params, client_id)
@@ -322,12 +331,12 @@ function lsp.start_client(config)
 
   function handlers.on_error(code, err)
     local _ = log.error() and log.error(log_prefix, "on_error", { code = lsp.client_errors[code], err = err })
-    nvim_err_writeln(string.format('%s: Error %s: %q', log_prefix, lsp.client_errors[code], vim.inspect(err)))
+    err_message(log_prefix, ': Error ', lsp.client_errors[code], ': ', vim.inspect(err))
     if config.on_error then
       local status, usererr = pcall(config.on_error, code, err)
       if not status then
         local _ = log.error() and log.error(log_prefix, "user on_error failed", { err = usererr })
-        nvim_err_writeln(log_prefix.." user on_error failed: "..tostring(usererr))
+        err_message(log_prefix, ' user on_error failed: ', tostring(usererr))
       end
     end
   end
@@ -335,9 +344,19 @@ function lsp.start_client(config)
   function handlers.on_exit(code, signal)
     active_clients[client_id] = nil
     uninitialized_clients[client_id] = nil
-    for _, client_ids in pairs(all_buffer_active_clients) do
+    local active_buffers = {}
+    for bufnr, client_ids in pairs(all_buffer_active_clients) do
+      if client_ids[client_id] then
+        table.insert(active_buffers, bufnr)
+      end
       client_ids[client_id] = nil
     end
+    -- Buffer level cleanup
+    vim.schedule(function()
+      for _, bufnr in ipairs(active_buffers) do
+        util.buf_clear_diagnostics(bufnr)
+      end
+    end)
     if config.on_exit then
       pcall(config.on_exit, code, signal, client_id)
     end
@@ -379,7 +398,6 @@ function lsp.start_client(config)
       -- The rootUri of the workspace. Is null if no folder is open. If both
       -- `rootPath` and `rootUri` are set `rootUri` wins.
       rootUri = vim.uri_from_fname(config.root_dir);
---      rootUri = vim.uri_from_fname(vim.fn.expand("%:p:h"));
       -- User provided initialization options.
       initializationOptions = config.init_options;
       -- The capabilities provided by the client (editor or tool)
@@ -403,11 +421,15 @@ function lsp.start_client(config)
       -- }
       workspaceFolders = nil;
     }
+    if config.before_init then
+      -- TODO(ashkan) handle errors here.
+      pcall(config.before_init, initialize_params, config)
+    end
     local _ = log.debug() and log.debug(log_prefix, "initialize_params", initialize_params)
     rpc.request('initialize', initialize_params, function(init_err, result)
       assert(not init_err, tostring(init_err))
       assert(result, "server sent empty result")
-      rpc.notify('initialized', {})
+      rpc.notify('initialized', {[vim.type_idx]=vim.types.dictionary})
       client.initialized = true
       uninitialized_clients[client_id] = nil
       client.server_capabilities = assert(result.capabilities, "initialize result doesn't contain capabilities")
@@ -439,14 +461,14 @@ function lsp.start_client(config)
   local function unsupported_method(method)
     local msg = "server doesn't support "..method
     local _ = log.warn() and log.warn(msg)
-    nvim_err_writeln(msg)
+    err_message(msg)
     return lsp.rpc_response_error(protocol.ErrorCodes.MethodNotFound, msg)
   end
 
   --- Checks capabilities before rpc.request-ing.
   function client.request(method, params, callback)
     if not callback then
-      callback = client.callbacks[method]
+      callback = resolve_callback(method)
         or error(string.format("request callback is empty and no default was found for client %s", client.name))
     end
     local _ = log.debug() and log.debug(log_prefix, "client.request", client_id, method, params, callback)
@@ -851,7 +873,7 @@ function lsp.omnifunc(findstart, base)
       position = {
         -- 0-indexed for both line and character
         line = pos[1] - 1,
-        character = pos[2],
+        character = vim.str_utfindex(line, pos[2]),
       };
       -- The completion context. This is only available if the client specifies
       -- to send this using `ClientCapabilities.textDocument.completion.contextSupport === true`
@@ -876,134 +898,8 @@ function lsp.omnifunc(findstart, base)
   end
 end
 
----
---- FileType based configuration utility
----
-
-local all_filetype_configs = {}
-
--- Lookup a filetype config client by its name.
-function lsp.get_filetype_client_by_name(name)
-  local config = all_filetype_configs[name]
-  if config.client_id then
-    return active_clients[config.client_id]
-  end
-end
-
-local function start_filetype_config(config)
-  config.client_id = lsp.start_client(config)
-  nvim_command(string.format(
-    "autocmd FileType %s silent lua vim.lsp.buf_attach_client(0, %d)",
-    table.concat(config.filetypes, ','),
-    config.client_id))
-  return config.client_id
-end
-
--- Easy configuration option for common LSP use-cases.
--- This will lazy initialize the client when the filetypes specified are
--- encountered and attach to those buffers.
---
--- The configuration options are the same as |vim.lsp.start_client()|, but
--- with a few additions and distinctions:
---
--- Additional parameters:
--- - filetype: {string} or {list} of filetypes to attach to.
--- - name: A unique string among all other servers configured with
--- |vim.lsp.add_filetype_config|.
---
--- Differences:
--- - root_dir: will default to |getcwd()|
---
-function lsp.add_filetype_config(config)
-  -- Additional defaults.
-  -- Keep a copy of the user's input for debugging reasons.
-  local user_config = config
-  config = tbl_extend("force", {}, user_config)
-  config.root_dir = config.root_dir or uv.cwd()
-  -- Validate config.
-  validate_client_config(config)
-  validate {
-    name = { config.name, 's' };
-  }
-  assert(config.filetype, "config must have 'filetype' key")
-
-  local filetypes
-  if type(config.filetype) == 'string' then
-    filetypes = { config.filetype }
-  elseif type(config.filetype) == 'table' then
-    filetypes = config.filetype
-    assert(not tbl_isempty(filetypes), "config.filetype must not be an empty table")
-  else
-    error("config.filetype must be a string or a list of strings")
-  end
-
-  if all_filetype_configs[config.name] then
-    -- If the client exists, then it is likely that they are doing some kind of
-    -- reload flow, so let's not throw an error here.
-    if all_filetype_configs[config.name].client_id then
-      -- TODO log here? It might be unnecessarily annoying.
-      return
-    end
-    error(string.format('A configuration with the name %q already exists. They must be unique', config.name))
-  end
-
-  all_filetype_configs[config.name] = tbl_extend("keep", config, {
-    client_id = nil;
-    filetypes = filetypes;
-    user_config = user_config;
-  })
-
-  nvim_command(string.format(
-    "autocmd FileType %s ++once silent lua vim.lsp._start_filetype_config_client(%q)",
-    table.concat(filetypes, ','),
-    config.name))
-end
-
--- Create a copy of an existing configuration, and override config with values
--- from new_config.
--- This is useful if you wish you create multiple LSPs with different root_dirs
--- or other use cases.
---
--- You can specify a new unique name, but if you do not, a unique name will be
--- created like `name-dup_count`.
---
--- existing_name: the name of the existing config to copy.
--- new_config: the new configuration options. @see |vim.lsp.start_client()|.
--- @returns string the new name.
-function lsp.copy_filetype_config(existing_name, new_config)
-  local config = all_filetype_configs[existing_name]
-      or error(string.format("Configuration with name %q doesn't exist", existing_name))
-  config = tbl_extend("force", config, new_config or {})
-  config.client_id = nil
-  config.original_config_name = existing_name
-
-  -- If the user didn't rename it, we will.
-  if config.name == existing_name then
-    -- Create a new, unique name.
-    local duplicate_count = 0
-    for _, conf in pairs(all_filetype_configs) do
-      if conf.original_config_name == existing_name then
-        duplicate_count = duplicate_count + 1
-      end
-    end
-    config.name = string.format("%s-%d", existing_name, duplicate_count + 1)
-  end
-  print("New config name:", config.name)
-  lsp.add_filetype_config(config)
-  return config.name
-end
-
--- Autocmd handler to actually start the client when an applicable filetype is
--- encountered.
-function lsp._start_filetype_config_client(name)
-  local config = all_filetype_configs[name]
-  -- If it exists and is running, don't make it again.
-  if config.client_id and active_clients[config.client_id] then
-    -- TODO log here?
-    return
-  end
-  lsp.buf_attach_client(0, start_filetype_config(config))
-  return config.client_id
+function lsp.client_is_stopped(client_id)
+  return active_clients[client_id] == nil
 end
 
 ---
@@ -1030,7 +926,7 @@ end
 -- Print some debug information about all LSP related things.
 -- The output of this function should not be relied upon and may change.
 function lsp.print_debug_info()
-  print(vim.inspect({ clients = active_clients, filetype_configs = all_filetype_configs }))
+  print(vim.inspect({ clients = active_clients }))
 end
 
 -- Log level dictionary with reverse lookup as well.
diff --git a/runtime/lua/vim/lsp/buf.lua b/runtime/lua/vim/lsp/buf.lua
new file mode 100644
index 0000000000..51295e5570
--- /dev/null
+++ b/runtime/lua/vim/lsp/buf.lua
@@ -0,0 +1,349 @@
+local vim = vim
+local validate = vim.validate
+local api = vim.api
+local vfn = vim.fn
+local util = require 'vim.lsp.util'
+local log = require 'vim.lsp.log'
+local list_extend = vim.list_extend
+
+local M = {}
+
+local function ok_or_nil(status, ...)
+  if not status then return end
+  return ...
+end
+local function npcall(fn, ...)
+  return ok_or_nil(pcall(fn, ...))
+end
+
+local function err_message(...)
+  api.nvim_err_writeln(table.concat(vim.tbl_flatten{...}))
+  api.nvim_command("redraw")
+end
+
+local function find_window_by_var(name, value)
+  for _, win in ipairs(api.nvim_list_wins()) do
+    if npcall(api.nvim_win_get_var, win, name) == value then
+      return win
+    end
+  end
+end
+
+local function request(method, params, callback)
+  --  TODO(ashkan) enable this.
+  --  callback = vim.lsp.default_callbacks[method] or callback
+  validate {
+    method = {method, 's'};
+    callback = {callback, 'f'};
+  }
+  return vim.lsp.buf_request(0, method, params, function(err, _, result, client_id)
+    local _ = log.debug() and log.debug("vim.lsp.buf", method, client_id, err, result)
+    if err then error(tostring(err)) end
+    return callback(err, method, result, client_id)
+  end)
+end
+
+local function focusable_preview(method, params, fn)
+  if npcall(api.nvim_win_get_var, 0, method) then
+    return api.nvim_command("wincmd p")
+  end
+
+  local bufnr = api.nvim_get_current_buf()
+  do
+    local win = find_window_by_var(method, bufnr)
+    if win then
+      api.nvim_set_current_win(win)
+      api.nvim_command("stopinsert")
+      return
+    end
+  end
+  return request(method, params, function(_, _, result, _)
+      -- TODO(ashkan) could show error in preview...
+    local lines, filetype, opts = fn(result)
+    if lines then
+      local _, winnr = util.open_floating_preview(lines, filetype, opts)
+      api.nvim_win_set_var(winnr, method, bufnr)
+    end
+  end)
+end
+
+function M.hover()
+  local params = util.make_position_params()
+  focusable_preview('textDocument/hover', params, function(result)
+    if not (result and result.contents) then return end
+
+    local markdown_lines = util.convert_input_to_markdown_lines(result.contents)
+    markdown_lines = util.trim_empty_lines(markdown_lines)
+    if vim.tbl_isempty(markdown_lines) then
+      return { 'No information available' }
+    end
+    return markdown_lines, util.try_trim_markdown_code_blocks(markdown_lines)
+  end)
+end
+
+function M.peek_definition()
+  local params = util.make_position_params()
+  request('textDocument/peekDefinition', params, function(_, _, result, _)
+    if not (result and result[1]) then return end
+    local loc = result[1]
+    local bufnr = vim.uri_to_bufnr(loc.uri) or error("couldn't find file "..tostring(loc.uri))
+    local start = loc.range.start
+    local finish = loc.range["end"]
+    util.open_floating_peek_preview(bufnr, start, finish, { offset_x = 1 })
+    local headbuf = util.open_floating_preview({"Peek:"}, nil, {
+      offset_y = -(finish.line - start.line);
+      width = finish.character - start.character + 2;
+    })
+    -- TODO(ashkan) change highlight group?
+    api.nvim_buf_add_highlight(headbuf, -1, 'Keyword', 0, -1)
+  end)
+end
+
+
+local function update_tagstack()
+  local bufnr = api.nvim_get_current_buf()
+  local line = vfn.line('.')
+  local col = vfn.col('.')
+  local tagname = vfn.expand('<cWORD>')
+  local item = { bufnr = bufnr, from = { bufnr, line, col, 0 }, tagname = tagname }
+  local winid = vfn.win_getid()
+  local tagstack = vfn.gettagstack(winid)
+  local action
+  if tagstack.length == tagstack.curidx then
+    action = 'r'
+    tagstack.items[tagstack.curidx] = item
+  elseif tagstack.length > tagstack.curidx then
+    action = 'r'
+    if tagstack.curidx > 1 then
+      tagstack.items = table.insert(tagstack.items[tagstack.curidx - 1], item)
+    else
+      tagstack.items = { item }
+    end
+  else
+    action = 'a'
+    tagstack.items = { item }
+  end
+  tagstack.curidx = tagstack.curidx + 1
+  vfn.settagstack(winid, tagstack, action)
+end
+local function handle_location(result)
+  -- We can sometimes get a list of locations, so set the first value as the
+  -- only value we want to handle
+  -- TODO(ashkan) was this correct^? We could use location lists.
+  if result[1] ~= nil then
+    result = result[1]
+  end
+  if result.uri == nil then
+    err_message('[LSP] Could not find a valid location')
+    return
+  end
+  local bufnr = vim.uri_to_bufnr(result.uri)
+  update_tagstack()
+  api.nvim_set_current_buf(bufnr)
+  local row = result.range.start.line
+  local col = result.range.start.character
+  local line = api.nvim_buf_get_lines(0, row, row+1, true)[1]
+  col = #line:sub(1, col)
+  api.nvim_win_set_cursor(0, {row + 1, col})
+  return true
+end
+local function location_callback(_, method, result)
+  if result == nil or vim.tbl_isempty(result) then
+    local _ = log.info() and log.info(method, 'No location found')
+    return nil
+  end
+  return handle_location(result)
+end
+
+function M.declaration()
+  local params = util.make_position_params()
+  request('textDocument/declaration', params, location_callback)
+end
+
+function M.definition()
+  local params = util.make_position_params()
+  request('textDocument/definition', params, location_callback)
+end
+
+function M.type_definition()
+  local params = util.make_position_params()
+  request('textDocument/typeDefinition', params, location_callback)
+end
+
+function M.implementation()
+  local params = util.make_position_params()
+  request('textDocument/implementation', params, location_callback)
+end
+
+--- Convert SignatureHelp response to preview contents.
+-- https://microsoft.github.io/language-server-protocol/specifications/specification-3-14/#textDocument_signatureHelp
+local function signature_help_to_preview_contents(input)
+  if not input.signatures then
+    return
+  end
+  --The active signature. If omitted or the value lies outside the range of
+  --`signatures` the value defaults to zero or is ignored if `signatures.length
+  --=== 0`. Whenever possible implementors should make an active decision about
+  --the active signature and shouldn't rely on a default value.
+  local contents = {}
+  local active_signature = input.activeSignature or 0
+  -- If the activeSignature is not inside the valid range, then clip it.
+  if active_signature >= #input.signatures then
+    active_signature = 0
+  end
+  local signature = input.signatures[active_signature + 1]
+  if not signature then
+    return
+  end
+  list_extend(contents, vim.split(signature.label, '\n', true))
+  if signature.documentation then
+    util.convert_input_to_markdown_lines(signature.documentation, contents)
+  end
+  if input.parameters then
+    local active_parameter = input.activeParameter or 0
+    -- If the activeParameter is not inside the valid range, then clip it.
+    if active_parameter >= #input.parameters then
+      active_parameter = 0
+    end
+    local parameter = signature.parameters and signature.parameters[active_parameter]
+    if parameter then
+      --[=[
+      --Represents a parameter of a callable-signature. A parameter can
+      --have a label and a doc-comment.
+      interface ParameterInformation {
+        --The label of this parameter information.
+        --
+        --Either a string or an inclusive start and exclusive end offsets within its containing
+        --signature label. (see SignatureInformation.label). The offsets are based on a UTF-16
+        --string representation as `Position` and `Range` does.
+        --
+        --*Note*: a label of type string should be a substring of its containing signature label.
+        --Its intended use case is to highlight the parameter label part in the `SignatureInformation.label`.
+        label: string | [number, number];
+        --The human-readable doc-comment of this parameter. Will be shown
+        --in the UI but can be omitted.
+        documentation?: string | MarkupContent;
+      }
+      --]=]
+      -- TODO highlight parameter
+      if parameter.documentation then
+        util.convert_input_to_markdown_lines(parameter.documentation, contents)
+      end
+    end
+  end
+  return contents
+end
+
+function M.signature_help()
+  local params = util.make_position_params()
+  focusable_preview('textDocument/signatureHelp', params, function(result)
+    if not (result and result.signatures and result.signatures[1]) then
+      return { 'No signature available' }
+    end
+
+    -- TODO show empty popup when signatures is empty?
+    local lines = signature_help_to_preview_contents(result)
+    lines = util.trim_empty_lines(lines)
+    if vim.tbl_isempty(lines) then
+      return { 'No signature available' }
+    end
+    return lines, util.try_trim_markdown_code_blocks(lines)
+  end)
+end
+
+-- TODO(ashkan) ?
+function M.completion(context)
+  local params = util.make_position_params()
+  params.context = context
+  return request('textDocument/completion', params, function(_, _, result)
+    if vim.tbl_isempty(result or {}) then return end
+    local row, col = unpack(api.nvim_win_get_cursor(0))
+    local line = assert(api.nvim_buf_get_lines(0, row-1, row, false)[1])
+    local line_to_cursor = line:sub(col+1)
+
+    local matches = util.text_document_completion_list_to_complete_items(result, line_to_cursor)
+    vim.fn.complete(col, matches)
+  end)
+end
+
+function M.formatting(options)
+  validate { options = {options, 't', true} }
+  options = vim.tbl_extend('keep', options or {}, {
+    tabSize = api.nvim_buf_get_option(0, 'tabstop');
+    insertSpaces = api.nvim_buf_get_option(0, 'expandtab');
+  })
+  local params = {
+    textDocument = { uri = vim.uri_from_bufnr(0) };
+    options = options;
+  }
+  return request('textDocument/formatting', params, function(_, _, result)
+    if not result then return end
+    util.apply_text_edits(result)
+  end)
+end
+
+function M.range_formatting(options, start_pos, end_pos)
+  validate {
+    options = {options, 't', true};
+    start_pos = {start_pos, 't', true};
+    end_pos = {end_pos, 't', true};
+  }
+  options = vim.tbl_extend('keep', options or {}, {
+    tabSize = api.nvim_buf_get_option(0, 'tabstop');
+    insertSpaces = api.nvim_buf_get_option(0, 'expandtab');
+  })
+  local A = list_extend({}, start_pos or api.nvim_buf_get_mark(0, '<'))
+  local B = list_extend({}, end_pos or api.nvim_buf_get_mark(0, '>'))
+  -- convert to 0-index
+  A[1] = A[1] - 1
+  B[1] = B[1] - 1
+  -- account for encoding.
+  if A[2] > 0 then
+    A = {A[1], util.character_offset(0, unpack(A))}
+  end
+  if B[2] > 0 then
+    B = {B[1], util.character_offset(0, unpack(B))}
+  end
+  local params = {
+    textDocument = { uri = vim.uri_from_bufnr(0) };
+    range = {
+      start = { line = A[1]; character = A[2]; };
+      ["end"] = { line = B[1]; character = B[2]; };
+    };
+    options = options;
+  }
+  return request('textDocument/rangeFormatting', params, function(_, _, result)
+    if not result then return end
+    util.apply_text_edits(result)
+  end)
+end
+
+function M.rename(new_name)
+  -- TODO(ashkan) use prepareRename
+  -- * result: [`Range`](#range) \| `{ range: Range, placeholder: string }` \| `null` describing the range of the string to rename and optionally a placeholder text of the string content to be renamed. If `null` is returned then it is deemed that a 'textDocument/rename' request is not valid at the given position.
+  local params = util.make_position_params()
+  new_name = new_name or npcall(vfn.input, "New Name: ")
+  if not (new_name and #new_name > 0) then return end
+  params.newName = new_name
+  request('textDocument/rename', params, function(_, _, result)
+    if not result then return end
+    util.apply_workspace_edit(result)
+  end)
+end
+
+function M.references(context)
+  validate { context = { context, 't', true } }
+  local params = util.make_position_params()
+  params.context = context or {
+    includeDeclaration = true;
+  }
+  params[vim.type_idx] = vim.types.dictionary
+  request('textDocument/references', params, function(_, _, result)
+    if not result then return end
+    util.set_qflist(result)
+    vim.api.nvim_command("copen")
+  end)
+end
+
+return M
+-- vim:sw=2 ts=2 et
diff --git a/runtime/lua/vim/lsp/builtin_callbacks.lua b/runtime/lua/vim/lsp/builtin_callbacks.lua
deleted file mode 100644
index cc739ce3ad..0000000000
--- a/runtime/lua/vim/lsp/builtin_callbacks.lua
+++ /dev/null
@@ -1,296 +0,0 @@
---- Implements the following default callbacks:
---
--- vim.api.nvim_buf_set_lines(0, 0, 0, false, vim.tbl_keys(vim.lsp.builtin_callbacks))
---
-
--- textDocument/completion
--- textDocument/declaration
--- textDocument/definition
--- textDocument/hover
--- textDocument/implementation
--- textDocument/publishDiagnostics
--- textDocument/rename
--- textDocument/signatureHelp
--- textDocument/typeDefinition
--- TODO codeLens/resolve
--- TODO completionItem/resolve
--- TODO documentLink/resolve
--- TODO textDocument/codeAction
--- TODO textDocument/codeLens
--- TODO textDocument/documentHighlight
--- TODO textDocument/documentLink
--- TODO textDocument/documentSymbol
--- TODO textDocument/formatting
--- TODO textDocument/onTypeFormatting
--- TODO textDocument/rangeFormatting
--- TODO textDocument/references
--- window/logMessage
--- window/showMessage
-
-local log = require 'vim.lsp.log'
-local protocol = require 'vim.lsp.protocol'
-local util = require 'vim.lsp.util'
-local api = vim.api
-
-local function split_lines(value)
-  return vim.split(value, '\n', true)
-end
-
-local builtin_callbacks = {}
-
--- textDocument/completion
--- https://microsoft.github.io/language-server-protocol/specification#textDocument_completion
-builtin_callbacks['textDocument/completion'] = function(_, _, result)
-  if not result or vim.tbl_isempty(result) then
-    return
-  end
-  local pos = api.nvim_win_get_cursor(0)
-  local row, col = pos[1], pos[2]
-  local line = assert(api.nvim_buf_get_lines(0, row-1, row, false)[1])
-  local line_to_cursor = line:sub(col+1)
-
-  local matches = util.text_document_completion_list_to_complete_items(result, line_to_cursor)
-  local match_result = vim.fn.matchstrpos(line_to_cursor, '\\k\\+$')
-  local match_start, match_finish = match_result[2], match_result[3]
-
-  vim.fn.complete(col + 1 - (match_finish - match_start), matches)
-end
-
--- textDocument/rename
-builtin_callbacks['textDocument/rename'] = function(_, _, result)
-  if not result then return end
-  util.workspace_apply_workspace_edit(result)
-end
-
-local function uri_to_bufnr(uri)
-  return vim.fn.bufadd((vim.uri_to_fname(uri)))
-end
-
-builtin_callbacks['textDocument/publishDiagnostics'] = function(_, _, result)
-  if not result then return end
-  local uri = result.uri
-  local bufnr = uri_to_bufnr(uri)
-  if not bufnr then
-    api.nvim_err_writeln(string.format("LSP.publishDiagnostics: Couldn't find buffer for %s", uri))
-    return
-  end
-  util.buf_clear_diagnostics(bufnr)
-  util.buf_diagnostics_save_positions(bufnr, result.diagnostics)
-  util.buf_diagnostics_underline(bufnr, result.diagnostics)
-  util.buf_diagnostics_virtual_text(bufnr, result.diagnostics)
-  -- util.buf_loclist(bufnr, result.diagnostics)
-end
-
--- textDocument/hover
--- https://microsoft.github.io/language-server-protocol/specification#textDocument_hover
--- @params MarkedString | MarkedString[] | MarkupContent
-builtin_callbacks['textDocument/hover'] = function(_, _, result)
-  if result == nil or vim.tbl_isempty(result) then
-    return
-  end
-
-  if result.contents ~= nil then
-    local markdown_lines = util.convert_input_to_markdown_lines(result.contents)
-    if vim.tbl_isempty(markdown_lines) then
-      markdown_lines = { 'No information available' }
-    end
-    util.open_floating_preview(markdown_lines, 'markdown')
-  end
-end
-
-builtin_callbacks['textDocument/peekDefinition'] = function(_, _, result)
-  if result == nil or vim.tbl_isempty(result) then return end
-  -- TODO(ashkan) what to do with multiple locations?
-  result = result[1]
-  local bufnr = uri_to_bufnr(result.uri)
-  assert(bufnr)
-  local start = result.range.start
-  local finish = result.range["end"]
-  util.open_floating_peek_preview(bufnr, start, finish, { offset_x = 1 })
-  util.open_floating_preview({"*Peek:*", string.rep(" ", finish.character - start.character + 1) }, 'markdown', { offset_y = -(finish.line - start.line) })
-end
-
---- Convert SignatureHelp response to preview contents.
--- https://microsoft.github.io/language-server-protocol/specifications/specification-3-14/#textDocument_signatureHelp
-local function signature_help_to_preview_contents(input)
-  if not input.signatures then
-    return
-  end
-  --The active signature. If omitted or the value lies outside the range of
-  --`signatures` the value defaults to zero or is ignored if `signatures.length
-  --=== 0`. Whenever possible implementors should make an active decision about
-  --the active signature and shouldn't rely on a default value.
-  local contents = {}
-  local active_signature = input.activeSignature or 0
-  -- If the activeSignature is not inside the valid range, then clip it.
-  if active_signature >= #input.signatures then
-    active_signature = 0
-  end
-  local signature = input.signatures[active_signature + 1]
-  if not signature then
-    return
-  end
-  vim.list_extend(contents, split_lines(signature.label))
-  if signature.documentation then
-    util.convert_input_to_markdown_lines(signature.documentation, contents)
-  end
-  if input.parameters then
-    local active_parameter = input.activeParameter or 0
-    -- If the activeParameter is not inside the valid range, then clip it.
-    if active_parameter >= #input.parameters then
-      active_parameter = 0
-    end
-    local parameter = signature.parameters and signature.parameters[active_parameter]
-    if parameter then
-      --[=[
-      --Represents a parameter of a callable-signature. A parameter can
-      --have a label and a doc-comment.
-      interface ParameterInformation {
-        --The label of this parameter information.
-        --
-        --Either a string or an inclusive start and exclusive end offsets within its containing
-        --signature label. (see SignatureInformation.label). The offsets are based on a UTF-16
-        --string representation as `Position` and `Range` does.
-        --
-        --*Note*: a label of type string should be a substring of its containing signature label.
-        --Its intended use case is to highlight the parameter label part in the `SignatureInformation.label`.
-        label: string | [number, number];
-        --The human-readable doc-comment of this parameter. Will be shown
-        --in the UI but can be omitted.
-        documentation?: string | MarkupContent;
-      }
-      --]=]
-      -- TODO highlight parameter
-      if parameter.documentation then
-        util.convert_input_to_markdown_lines(parameter.documentation, contents)
-      end
-    end
-  end
-  return contents
-end
-
--- textDocument/signatureHelp
--- https://microsoft.github.io/language-server-protocol/specification#textDocument_signatureHelp
-