Merge remote-tracking branch 'upstream/master' into usermarksusermarks

36 files changed, 4349 insertions, 2039 deletions

diff --git a/runtime/lua/vim/F.lua b/runtime/lua/vim/F.lua
index bca5ddf68b..3e370c0a84 100644
--- a/runtime/lua/vim/F.lua
+++ b/runtime/lua/vim/F.lua
@@ -2,8 +2,12 @@ local F = {}
 
 --- Returns {a} if it is not nil, otherwise returns {b}.
 ---
----@param a
----@param b
+---@generic A
+---@generic B
+---
+---@param a A
+---@param b B
+---@return A | B
 function F.if_nil(a, b)
   if a == nil then
     return b
diff --git a/runtime/lua/vim/_editor.lua b/runtime/lua/vim/_editor.lua
index b8a7f71145..da8764fbd4 100644
--- a/runtime/lua/vim/_editor.lua
+++ b/runtime/lua/vim/_editor.lua
@@ -12,21 +12,8 @@
 -- Guideline: "If in doubt, put it in the runtime".
 --
 -- Most functions should live directly in `vim.`, not in submodules.
--- The only "forbidden" names are those claimed by legacy `if_lua`:
---    $ vim
---    :lua for k,v in pairs(vim) do print(k) end
---    buffer
---    open
---    window
---    lastline
---    firstline
---    type
---    line
---    eval
---    dict
---    beep
---    list
---    command
+--
+-- Compatibility with Vim's `if_lua` is explicitly a non-goal.
 --
 -- Reference (#6580):
 --    - https://github.com/luafun/luafun
@@ -36,8 +23,6 @@
 --    - https://github.com/bakpakin/Fennel (pretty print, repl)
 --    - https://github.com/howl-editor/howl/tree/master/lib/howl/util
 
-local vim = assert(vim)
-
 -- These are for loading runtime modules lazily since they aren't available in
 -- the nvim binary as specified in executor.c
 for k, v in pairs({
@@ -51,6 +36,7 @@ for k, v in pairs({
   ui = true,
   health = true,
   fs = true,
+  secure = true,
 }) do
   vim._submodules[k] = v
 end
@@ -122,9 +108,7 @@ function vim._os_proc_children(ppid)
   return children
 end
 
--- TODO(ZyX-I): Create compatibility layer.
-
---- Return a human-readable representation of the given object.
+--- Gets a human-readable representation of the given object.
 ---
 ---@see https://github.com/kikito/inspect.lua
 ---@see https://github.com/mpeterv/vinspect
@@ -139,7 +123,7 @@ do
   --- (such as the |TUI|) pastes text into the editor.
   ---
   --- Example: To remove ANSI color codes when pasting:
-  --- <pre>
+  --- <pre>lua
   --- vim.paste = (function(overridden)
   ---   return function(lines, phase)
   ---     for i,line in ipairs(lines) do
@@ -152,14 +136,15 @@ do
   --- </pre>
   ---
   ---@see |paste|
+  ---@alias paste_phase -1 | 1 | 2 | 3
   ---
-  ---@param lines  |readfile()|-style list of lines to paste. |channel-lines|
-  ---@param phase  -1: "non-streaming" paste: the call contains all lines.
+  ---@param lines  string[] # |readfile()|-style list of lines to paste. |channel-lines|
+  ---@param phase paste_phase  -1: "non-streaming" paste: the call contains all lines.
   ---              If paste is "streamed", `phase` indicates the stream state:
   ---                - 1: starts the paste (exactly once)
   ---                - 2: continues the paste (zero or more times)
   ---                - 3: ends the paste (exactly once)
-  ---@returns false if client should cancel the paste.
+  ---@returns boolean # false if client should cancel the paste.
   function vim.paste(lines, phase)
     local now = vim.loop.now()
     local is_first_chunk = phase < 2
@@ -183,7 +168,8 @@ do
         local line1 = lines[1]:gsub('(%c)', '\022%1')
         -- nvim_input() is affected by mappings,
         -- so use nvim_feedkeys() with "n" flag to ignore mappings.
-        vim.api.nvim_feedkeys(line1, 'n', true)
+        -- "t" flag is also needed so the pasted text is saved in cmdline history.
+        vim.api.nvim_feedkeys(line1, 'nt', true)
       end
       return true
     end
@@ -255,6 +241,8 @@ end
 ---@see |lua-loop-callbacks|
 ---@see |vim.schedule()|
 ---@see |vim.in_fast_event()|
+---@param cb function
+---@return function
 function vim.schedule_wrap(cb)
   return function(...)
     local args = vim.F.pack_len(...)
@@ -292,7 +280,7 @@ end
 --- command.
 ---
 --- Example:
---- <pre>
+--- <pre>lua
 ---   vim.cmd('echo 42')
 ---   vim.cmd([[
 ---     augroup My_group
@@ -399,11 +387,11 @@ end
 --- Get a table of lines with start, end columns for a region marked by two points
 ---
 ---@param bufnr number of buffer
----@param pos1 (line, column) tuple marking beginning of region
----@param pos2 (line, column) tuple marking end of region
----@param regtype type of selection (:help setreg)
+---@param pos1 integer[] (line, column) tuple marking beginning of region
+---@param pos2 integer[] (line, column) tuple marking end of region
+---@param regtype string type of selection, see |setreg()|
 ---@param inclusive boolean indicating whether the selection is end-inclusive
----@return region lua table of the form {linenr = {startcol,endcol}}
+---@return table region Table of the form `{linenr = {startcol,endcol}}`
 function vim.region(bufnr, pos1, pos2, regtype, inclusive)
   if not vim.api.nvim_buf_is_loaded(bufnr) then
     vim.fn.bufload(bufnr)
@@ -430,11 +418,16 @@ function vim.region(bufnr, pos1, pos2, regtype, inclusive)
       c2 = c1 + regtype:sub(2)
       -- and adjust for non-ASCII characters
       bufline = vim.api.nvim_buf_get_lines(bufnr, l, l + 1, true)[1]
-      if c1 < #bufline then
+      local utflen = vim.str_utfindex(bufline, #bufline)
+      if c1 <= utflen then
         c1 = vim.str_byteindex(bufline, c1)
+      else
+        c1 = #bufline + 1
       end
-      if c2 < #bufline then
+      if c2 <= utflen then
         c2 = vim.str_byteindex(bufline, c2)
+      else
+        c2 = #bufline + 1
       end
     else
       c1 = (l == pos1[1]) and pos1[2] or 0
@@ -448,11 +441,11 @@ end
 --- Defers calling `fn` until `timeout` ms passes.
 ---
 --- Use to do a one-shot timer that calls `fn`
---- Note: The {fn} is |schedule_wrap|ped automatically, so API functions are
+--- Note: The {fn} is |vim.schedule_wrap()|ped automatically, so API functions are
 --- safe to call.
----@param fn Callback to call once `timeout` expires
----@param timeout Number of milliseconds to wait before calling `fn`
----@return timer luv timer object
+---@param fn function Callback to call once `timeout` expires
+---@param timeout integer Number of milliseconds to wait before calling `fn`
+---@return table timer luv timer object
 function vim.defer_fn(fn, timeout)
   vim.validate({ fn = { fn, 'c', true } })
   local timer = vim.loop.new_timer()
@@ -753,12 +746,12 @@ end
 
 ---Prints given arguments in human-readable format.
 ---Example:
----<pre>
+---<pre>lua
 ---  -- Print highlight group Normal and store it's contents in a variable.
 ---  local hl_normal = vim.pretty_print(vim.api.nvim_get_hl_by_name("Normal", true))
 ---</pre>
 ---@see |vim.inspect()|
----@return given arguments.
+---@return any # given arguments.
 function vim.pretty_print(...)
   local objects = {}
   for i = 1, select('#', ...) do
diff --git a/runtime/lua/vim/_init_packages.lua b/runtime/lua/vim/_init_packages.lua
index 7e3c73667e..0c4ee8636d 100644
--- a/runtime/lua/vim/_init_packages.lua
+++ b/runtime/lua/vim/_init_packages.lua
@@ -1,6 +1,3 @@
--- prevents luacheck from making lints for setting things on vim
-local vim = assert(vim)
-
 local pathtrails = {}
 vim._so_trails = {}
 for s in (package.cpath .. ';'):gmatch('[^;]*;') do
@@ -59,6 +56,9 @@ setmetatable(vim, {
     if vim._submodules[key] then
       t[key] = require('vim.' .. key)
       return t[key]
+    elseif key == 'inspect_pos' or key == 'show_pos' then
+      require('vim._inspector')
+      return t[key]
     elseif vim.startswith(key, 'uri_') then
       local val = require('vim.uri')[key]
       if val ~= nil then
diff --git a/runtime/lua/vim/_inspector.lua b/runtime/lua/vim/_inspector.lua
new file mode 100644
index 0000000000..f46a525910
--- /dev/null
+++ b/runtime/lua/vim/_inspector.lua
@@ -0,0 +1,238 @@
+---@class InspectorFilter
+---@field syntax boolean include syntax based highlight groups (defaults to true)
+---@field treesitter boolean include treesitter based highlight groups (defaults to true)
+---@field extmarks boolean|"all" include extmarks. When `all`, then extmarks without a `hl_group` will also be included (defaults to true)
+---@field semantic_tokens boolean include semantic tokens (defaults to true)
+local defaults = {
+  syntax = true,
+  treesitter = true,
+  extmarks = true,
+  semantic_tokens = true,
+}
+
+---Get all the items at a given buffer position.
+---
+---Can also be pretty-printed with `:Inspect!`. *:Inspect!*
+---
+---@param bufnr? number defaults to the current buffer
+---@param row? number row to inspect, 0-based. Defaults to the row of the current cursor
+---@param col? number col to inspect, 0-based. Defaults to the col of the current cursor
+---@param filter? InspectorFilter (table|nil) a table with key-value pairs to filter the items
+---               - syntax (boolean): include syntax based highlight groups (defaults to true)
+---               - treesitter (boolean): include treesitter based highlight groups (defaults to true)
+---               - extmarks (boolean|"all"): include extmarks. When `all`, then extmarks without a `hl_group` will also be included (defaults to true)
+---               - semantic_tokens (boolean): include semantic tokens (defaults to true)
+---@return {treesitter:table,syntax:table,extmarks:table,semantic_tokens:table,buffer:number,col:number,row:number} (table) a table with the following key-value pairs. Items are in "traversal order":
+---               - treesitter: a list of treesitter captures
+---               - syntax: a list of syntax groups
+---               - semantic_tokens: a list of semantic tokens
+---               - extmarks: a list of extmarks
+---               - buffer: the buffer used to get the items
+---               - row: the row used to get the items
+---               - col: the col used to get the items
+function vim.inspect_pos(bufnr, row, col, filter)
+  filter = vim.tbl_deep_extend('force', defaults, filter or {})
+
+  bufnr = bufnr or 0
+  if row == nil or col == nil then
+    -- get the row/col from the first window displaying the buffer
+    local win = bufnr == 0 and vim.api.nvim_get_current_win() or vim.fn.bufwinid(bufnr)
+    if win == -1 then
+      error('row/col is required for buffers not visible in a window')
+    end
+    local cursor = vim.api.nvim_win_get_cursor(win)
+    row, col = cursor[1] - 1, cursor[2]
+  end
+  bufnr = bufnr == 0 and vim.api.nvim_get_current_buf() or bufnr
+
+  local results = {
+    treesitter = {},
+    syntax = {},
+    extmarks = {},
+    semantic_tokens = {},
+    buffer = bufnr,
+    row = row,
+    col = col,
+  }
+
+  -- resolve hl links
+  ---@private
+  local function resolve_hl(data)
+    if data.hl_group then
+      local hlid = vim.api.nvim_get_hl_id_by_name(data.hl_group)
+      local name = vim.fn.synIDattr(vim.fn.synIDtrans(hlid), 'name')
+      data.hl_group_link = name
+    end
+    return data
+  end
+
+  -- treesitter
+  if filter.treesitter then
+    for _, capture in pairs(vim.treesitter.get_captures_at_pos(bufnr, row, col)) do
+      capture.hl_group = '@' .. capture.capture
+      table.insert(results.treesitter, resolve_hl(capture))
+    end
+  end
+
+  -- syntax
+  if filter.syntax then
+    for _, i1 in ipairs(vim.fn.synstack(row + 1, col + 1)) do
+      table.insert(results.syntax, resolve_hl({ hl_group = vim.fn.synIDattr(i1, 'name') }))
+    end
+  end
+
+  -- semantic tokens
+  if filter.semantic_tokens then
+    for _, token in ipairs(vim.lsp.semantic_tokens.get_at_pos(bufnr, row, col) or {}) do
+      token.hl_groups = {
+        type = resolve_hl({ hl_group = '@' .. token.type }),
+        modifiers = vim.tbl_map(function(modifier)
+          return resolve_hl({ hl_group = '@' .. modifier })
+        end, token.modifiers or {}),
+      }
+      table.insert(results.semantic_tokens, token)
+    end
+  end
+
+  -- extmarks
+  if filter.extmarks then
+    for ns, nsid in pairs(vim.api.nvim_get_namespaces()) do
+      if ns:find('vim_lsp_semantic_tokens') ~= 1 then
+        local extmarks = vim.api.nvim_buf_get_extmarks(bufnr, nsid, 0, -1, { details = true })
+        for _, extmark in ipairs(extmarks) do
+          extmark = {
+            ns_id = nsid,
+            ns = ns,
+            id = extmark[1],
+            row = extmark[2],
+            col = extmark[3],
+            opts = resolve_hl(extmark[4]),
+          }
+          local end_row = extmark.opts.end_row or extmark.row -- inclusive
+          local end_col = extmark.opts.end_col or (extmark.col + 1) -- exclusive
+          if
+            (filter.extmarks == 'all' or extmark.opts.hl_group) -- filter hl_group
+            and (row >= extmark.row and row <= end_row) -- within the rows of the extmark
+            and (row > extmark.row or col >= extmark.col) -- either not the first row, or in range of the col
+            and (row < end_row or col < end_col) -- either not in the last row or in range of the col
+          then
+            table.insert(results.extmarks, extmark)
+          end
+        end
+      end
+    end
+  end
+  return results
+end
+
+---Show all the items at a given buffer position.
+---
+---Can also be shown with `:Inspect`. *:Inspect*
+---
+---@param bufnr? number defaults to the current buffer
+---@param row? number row to inspect, 0-based. Defaults to the row of the current cursor
+---@param col? number col to inspect, 0-based. Defaults to the col of the current cursor
+---@param filter? InspectorFilter (table|nil) see |vim.inspect_pos()|
+function vim.show_pos(bufnr, row, col, filter)
+  local items = vim.inspect_pos(bufnr, row, col, filter)
+
+  local lines = { {} }
+
+  ---@private
+  local function append(str, hl)
+    table.insert(lines[#lines], { str, hl })
+  end
+
+  ---@private
+  local function nl()
+    table.insert(lines, {})
+  end
+
+  ---@private
+  local function item(data, comment)
+    append('  - ')
+    append(data.hl_group, data.hl_group)
+    append(' ')
+    if data.hl_group ~= data.hl_group_link then
+      append('links to ', 'MoreMsg')
+      append(data.hl_group_link, data.hl_group_link)
+      append(' ')
+    end
+    if comment then
+      append(comment, 'Comment')
+    end
+    nl()
+  end
+
+  -- treesitter
+  if #items.treesitter > 0 then
+    append('Treesitter', 'Title')
+    nl()
+    for _, capture in ipairs(items.treesitter) do
+      item(capture, capture.lang)
+    end
+    nl()
+  end
+
+  if #items.semantic_tokens > 0 then
+    append('Semantic Tokens', 'Title')
+    nl()
+    for _, token in ipairs(items.semantic_tokens) do
+      local client = vim.lsp.get_client_by_id(token.client_id)
+      client = client and (' (' .. client.name .. ')') or ''
+      item(token.hl_groups.type, 'type' .. client)
+      for _, modifier in ipairs(token.hl_groups.modifiers) do
+        item(modifier, 'modifier' .. client)
+      end
+    end
+    nl()
+  end
+
+  -- syntax
+  if #items.syntax > 0 then
+    append('Syntax', 'Title')
+    nl()
+    for _, syn in ipairs(items.syntax) do
+      item(syn)
+    end
+    nl()
+  end
+  -- extmarks
+  if #items.extmarks > 0 then
+    append('Extmarks', 'Title')
+    nl()
+    for _, extmark in ipairs(items.extmarks) do
+      if extmark.opts.hl_group then
+        item(extmark.opts, extmark.ns)
+      else
+        append('  - ')
+        append(extmark.ns, 'Comment')
+        nl()
+      end
+    end
+    nl()
+  end
+
+  if #lines[#lines] == 0 then
+    table.remove(lines)
+  end
+
+  local chunks = {}
+  for _, line in ipairs(lines) do
+    vim.list_extend(chunks, line)
+    table.insert(chunks, { '\n' })
+  end
+  if #chunks == 0 then
+    chunks = {
+      {
+        'No items found at position '
+          .. items.row
+          .. ','
+          .. items.col
+          .. ' in buffer '
+          .. items.buffer,
+      },
+    }
+  end
+  vim.api.nvim_echo(chunks, false, {})
+end
diff --git a/runtime/lua/vim/_meta.lua b/runtime/lua/vim/_meta.lua
index f1652718ee..104f29c4c0 100644
--- a/runtime/lua/vim/_meta.lua
+++ b/runtime/lua/vim/_meta.lua
@@ -1,173 +1,113 @@
--- prevents luacheck from making lints for setting things on vim
-local vim = assert(vim)
-
 local a = vim.api
-local validate = vim.validate
-
-local SET_TYPES = setmetatable({
-  SET = 0,
-  LOCAL = 1,
-  GLOBAL = 2,
-}, { __index = error })
-
-local options_info = nil
-local buf_options = nil
-local glb_options = nil
-local win_options = nil
-
-local function _setup()
-  if options_info ~= nil then
-    return
-  end
-  options_info = {}
-  for _, v in pairs(a.nvim_get_all_options_info()) do
-    options_info[v.name] = v
-    if v.shortname ~= '' then
-      options_info[v.shortname] = v
-    end
-  end
 
-  local function get_scoped_options(scope)
-    local result = {}
-    for name, option_info in pairs(options_info) do
-      if option_info.scope == scope then
-        result[name] = true
+-- TODO(tjdevries): Improve option metadata so that this doesn't have to be hardcoded.
+--                  Can be done in a separate PR.
+local key_value_options = {
+  fillchars = true,
+  fcs = true,
+  listchars = true,
+  lcs = true,
+  winhighlight = true,
+  winhl = true,
+}
+
+--- Convert a vimoption_T style dictionary to the correct OptionType associated with it.
+---@return string
+local function get_option_metatype(name, info)
+  if info.type == 'string' then
+    if info.flaglist then
+      return 'set'
+    elseif info.commalist then
+      if key_value_options[name] then
+        return 'map'
       end
+      return 'array'
     end
-
-    return result
+    return 'string'
   end
-
-  buf_options = get_scoped_options('buf')
-  glb_options = get_scoped_options('global')
-  win_options = get_scoped_options('win')
+  return info.type
 end
 
-local function make_meta_accessor(get, set, del, validator)
-  validator = validator or function()
-    return true
-  end
-
-  validate({
-    get = { get, 'f' },
-    set = { set, 'f' },
-    del = { del, 'f', true },
-    validator = { validator, 'f' },
-  })
+local options_info = setmetatable({}, {
+  __index = function(t, k)
+    local info = a.nvim_get_option_info(k)
+    info.metatype = get_option_metatype(k, info)
+    rawset(t, k, info)
+    return rawget(t, k)
+  end,
+})
 
-  local mt = {}
-  function mt:__newindex(k, v)
-    if not validator(k) then
-      return
+vim.env = setmetatable({}, {
+  __index = function(_, k)
+    local v = vim.fn.getenv(k)
+    if v == vim.NIL then
+      return nil
     end
+    return v
+  end,
 
-    if del and v == nil then
-      return del(k)
-    end
-    return set(k, v)
-  end
-  function mt:__index(k)
-    if not validator(k) then
-      return
-    end
+  __newindex = function(_, k, v)
+    vim.fn.setenv(k, v)
+  end,
+})
 
-    return get(k)
+local function opt_validate(option_name, target_scope)
+  local scope = options_info[option_name].scope
+  if scope ~= target_scope then
+    local scope_to_string = { buf = 'buffer', win = 'window' }
+    error(
+      string.format(
+        [['%s' is a %s option, not a %s option. See ":help %s"]],
+        option_name,
+        scope_to_string[scope] or scope,
+        scope_to_string[target_scope] or target_scope,
+        option_name
+      )
+    )
   end
-  return setmetatable({}, mt)
 end
 
-vim.env = make_meta_accessor(function(k)
-  local v = vim.fn.getenv(k)
-  if v == vim.NIL then
-    return nil
-  end
-  return v
-end, vim.fn.setenv)
-
-do -- buffer option accessor
-  local function new_buf_opt_accessor(bufnr)
-    local function get(k)
-      if bufnr == nil and type(k) == 'number' then
-        return new_buf_opt_accessor(k)
-      end
-
-      return a.nvim_get_option_value(k, { buf = bufnr or 0 })
-    end
-
-    local function set(k, v)
-      return a.nvim_set_option_value(k, v, { buf = bufnr or 0 })
-    end
-
-    return make_meta_accessor(get, set, nil, function(k)
-      if type(k) == 'string' then
-        _setup()
-        if win_options[k] then
-          error(
-            string.format([['%s' is a window option, not a buffer option. See ":help %s"]], k, k)
-          )
-        elseif glb_options[k] then
-          error(
-            string.format([['%s' is a global option, not a buffer option. See ":help %s"]], k, k)
-          )
-        end
+local function new_opt_accessor(handle, scope)
+  return setmetatable({}, {
+    __index = function(_, k)
+      if handle == nil and type(k) == 'number' then
+        return new_opt_accessor(k, scope)
       end
+      opt_validate(k, scope)
+      return a.nvim_get_option_value(k, { [scope] = handle or 0 })
+    end,
 
-      return true
-    end)
-  end
-
-  vim.bo = new_buf_opt_accessor(nil)
+    __newindex = function(_, k, v)
+      opt_validate(k, scope)
+      return a.nvim_set_option_value(k, v, { [scope] = handle or 0 })
+    end,
+  })
 end
 
-do -- window option accessor
-  local function new_win_opt_accessor(winnr)
-    local function get(k)
-      if winnr == nil and type(k) == 'number' then
-        return new_win_opt_accessor(k)
-      end
-      return a.nvim_get_option_value(k, { win = winnr or 0 })
-    end
-
-    local function set(k, v)
-      return a.nvim_set_option_value(k, v, { win = winnr or 0 })
-    end
-
-    return make_meta_accessor(get, set, nil, function(k)
-      if type(k) == 'string' then
-        _setup()
-        if buf_options[k] then
-          error(
-            string.format([['%s' is a buffer option, not a window option. See ":help %s"]], k, k)
-          )
-        elseif glb_options[k] then
-          error(
-            string.format([['%s' is a global option, not a window option. See ":help %s"]], k, k)
-          )
-        end
-      end
-
-      return true
-    end)
-  end
-
-  vim.wo = new_win_opt_accessor(nil)
-end
+vim.bo = new_opt_accessor(nil, 'buf')
+vim.wo = new_opt_accessor(nil, 'win')
 
 -- vim global option
 --  this ONLY sets the global option. like `setglobal`
-vim.go = make_meta_accessor(function(k)
-  return a.nvim_get_option_value(k, { scope = 'global' })
-end, function(k, v)
-  return a.nvim_set_option_value(k, v, { scope = 'global' })
-end)
+vim.go = setmetatable({}, {
+  __index = function(_, k)
+    return a.nvim_get_option_value(k, { scope = 'global' })
+  end,
+  __newindex = function(_, k, v)
+    return a.nvim_set_option_value(k, v, { scope = 'global' })
+  end,
+})
 
 -- vim `set` style options.
 --  it has no additional metamethod magic.
-vim.o = make_meta_accessor(function(k)
-  return a.nvim_get_option_value(k, {})
-end, function(k, v)
-  return a.nvim_set_option_value(k, v, {})
-end)
+vim.o = setmetatable({}, {
+  __index = function(_, k)
+    return a.nvim_get_option_value(k, {})
+  end,
+  __newindex = function(_, k, v)
+    return a.nvim_set_option_value(k, v, {})
+  end,
+})
 
 ---@brief [[
 --- vim.opt, vim.opt_local and vim.opt_global implementation
@@ -178,11 +118,8 @@ end)
 ---@brief ]]
 
 --- Preserves the order and does not mutate the original list
-local remove_duplicate_values = function(t)
+local function remove_duplicate_values(t)
   local result, seen = {}, {}
-  if type(t) == 'function' then
-    error(debug.traceback('asdf'))
-  end
   for _, v in ipairs(t) do
     if not seen[v] then
       table.insert(result, v)
@@ -194,61 +131,6 @@ local remove_duplicate_values = function(t)
   return result
 end
 
--- TODO(tjdevries): Improve option metadata so that this doesn't have to be hardcoded.
---                  Can be done in a separate PR.
-local key_value_options = {
-  fillchars = true,
-  listchars = true,
-  winhl = true,
-}
-
----@class OptionTypes
---- Option Type Enum
-local OptionTypes = setmetatable({
-  BOOLEAN = 0,
-  NUMBER = 1,
-  STRING = 2,
-  ARRAY = 3,
-  MAP = 4,
-  SET = 5,
-}, {
-  __index = function(_, k)
-    error('Not a valid OptionType: ' .. k)
-  end,
-  __newindex = function(_, k)
-    error('Cannot set a new OptionType: ' .. k)
-  end,
-})
-
---- Convert a vimoption_T style dictionary to the correct OptionType associated with it.
----@return OptionType
-local get_option_type = function(name, info)
-  if info.type == 'boolean' then
-    return OptionTypes.BOOLEAN
-  elseif info.type == 'number' then
-    return OptionTypes.NUMBER
-  elseif info.type == 'string' then
-    if not info.commalist and not info.flaglist then
-      return OptionTypes.STRING
-    end
-
-    if key_value_options[name] then
-      assert(info.commalist, 'Must be a comma list to use key:value style')
-      return OptionTypes.MAP
-    end
-
-    if info.flaglist then
-      return OptionTypes.SET
-    elseif info.commalist then
-      return OptionTypes.ARRAY
-    end
-
-    error('Fallthrough in OptionTypes')
-  else
-    error('Not a known info.type:' .. info.type)
-  end
-end
-
 -- Check whether the OptionTypes is allowed for vim.opt
 -- If it does not match, throw an error which indicates which option causes the error.
 local function assert_valid_value(name, value, types)
@@ -269,373 +151,323 @@ local function assert_valid_value(name, value, types)
   )
 end
 
+local function passthrough(_, x)
+  return x
+end
+
+local function tbl_merge(left, right)
+  return vim.tbl_extend('force', left, right)
+end
+
+local function tbl_remove(t, value)
+  if type(value) == 'string' then
+    t[value] = nil
+  else
+    for _, v in ipairs(value) do
+      t[v] = nil
+    end
+  end
+
+  return t
+end
+
 local valid_types = {
-  [OptionTypes.BOOLEAN] = { 'boolean' },
-  [OptionTypes.NUMBER] = { 'number' },
-  [OptionTypes.STRING] = { 'string' },
-  [OptionTypes.SET] = { 'string', 'table' },
-  [OptionTypes.ARRAY] = { 'string', 'table' },
-  [OptionTypes.MAP] = { 'string', 'table' },
+  boolean = { 'boolean' },
+  number = { 'number' },
+  string = { 'string' },
+  set = { 'string', 'table' },
+  array = { 'string', 'table' },
+  map = { 'string', 'table' },
 }
 
---- Convert a lua value to a vimoption_T value
-local convert_value_to_vim = (function()
-  -- Map of functions to take a Lua style value and convert to vimoption_T style value.
-  -- Each function takes (info, lua_value) -> vim_value
-  local to_vim_value = {
-    [OptionTypes.BOOLEAN] = function(_, value)
-      return value
-    end,
-    [OptionTypes.NUMBER] = function(_, value)
-      return value
-    end,
-    [OptionTypes.STRING] = function(_, value)
-      return value
-    end,
-
-    [OptionTypes.SET] = function(info, value)
-      if type(value) == 'string' then
-        return value
-      end
+-- Map of functions to take a Lua style value and convert to vimoption_T style value.
+-- Each function takes (info, lua_value) -> vim_value
+local to_vim_value = {
+  boolean = passthrough,
+  number = passthrough,
+  string = passthrough,
 
-      if info.flaglist and info.commalist then
-        local keys = {}
-        for k, v in pairs(value) do
-          if v then
-            table.insert(keys, k)
-          end
-        end
+  set = function(info, value)
+    if type(value) == 'string' then
+      return value
+    end
 
-        table.sort(keys)
-        return table.concat(keys, ',')
-      else
-        local result = ''
-        for k, v in pairs(value) do
-          if v then
-            result = result .. k
-          end
+    if info.flaglist and info.commalist then
+      local keys = {}
+      for k, v in pairs(value) do
+        if v then
+          table.insert(keys, k)
         end
-
-        return result
       end
-    end,
 
-    [OptionTypes.ARRAY] = function(info, value)
-      if type(value) == 'string' then
-        return value
-      end
-      if not info.allows_duplicates then
-        value = remove_duplicate_values(value)
+      table.sort(keys)
+      return table.concat(keys, ',')
+    else
+      local result = ''
+      for k, v in pairs(value) do
+        if v then
+          result = result .. k
+        end
       end
-      return table.concat(value, ',')
-    end,
 
-    [OptionTypes.MAP] = function(_, value)
-      if type(value) == 'string' then
-        return value
-      end
+      return result
+    end
+  end,
 
-      local result = {}
-      for opt_key, opt_value in pairs(value) do
-        table.insert(result, string.format('%s:%s', opt_key, opt_value))
-      end
+  array = function(info, value)
+    if type(value) == 'string' then
+      return value
+    end
+    if not info.allows_duplicates then
+      value = remove_duplicate_values(value)
+    end
+    return table.concat(value, ',')
+  end,
 
-      table.sort(result)
-      return table.concat(result, ',')
-    end,
-  }
+  map = function(_, value)
+    if type(value) == 'string' then
+      return value
+    end
 
-  return function(name, info, value)
-    if value == nil then
-      return vim.NIL
+    local result = {}
+    for opt_key, opt_value in pairs(value) do
+      table.insert(result, string.format('%s:%s', opt_key, opt_value))
     end
 
-    local option_type = get_option_type(name, info)
-    assert_valid_value(name, value, valid_types[option_type])
+    table.sort(result)
+    return table.concat(result, ',')
+  end,
+}
 
-    return to_vim_value[option_type](info, value)
+--- Convert a lua value to a vimoption_T value
+local function convert_value_to_vim(name, info, value)
+  if value == nil then
+    return vim.NIL
   end
-end)()
 
---- Converts a vimoption_T style value to a Lua value
-local convert_value_to_lua = (function()
-  -- Map of OptionType to functions that take vimoption_T values and convert to lua values.
-  -- Each function takes (info, vim_value) -> lua_value
-  local to_lua_value = {
-    [OptionTypes.BOOLEAN] = function(_, value)
-      return value
-    end,
-    [OptionTypes.NUMBER] = function(_, value)
-      return value
-    end,
-    [OptionTypes.STRING] = function(_, value)
-      return value
-    end,
+  assert_valid_value(name, value, valid_types[info.metatype])
 
-    [OptionTypes.ARRAY] = function(info, value)
-      if type(value) == 'table' then
-        if not info.allows_duplicates then
-          value = remove_duplicate_values(value)
-        end
+  return to_vim_value[info.metatype](info, value)
+end
 
-        return value
-      end
+-- Map of OptionType to functions that take vimoption_T values and convert to lua values.
+-- Each function takes (info, vim_value) -> lua_value
+local to_lua_value = {
+  boolean = passthrough,
+  number = passthrough,
+  string = passthrough,
 
-      -- Empty strings mean that there is nothing there,
-      -- so empty table should be returned.
-      if value == '' then
-        return {}
+  array = function(info, value)
+    if type(value) == 'table' then
+      if not info.allows_duplicates then
+        value = remove_duplicate_values(value)
       end
 
-      -- Handles unescaped commas in a list.
-      if string.find(value, ',,,') then
-        local comma_split = vim.split(value, ',,,')
-        local left = comma_split[1]
-        local right = comma_split[2]
+      return value
+    end
 
-        local result = {}
-        vim.list_extend(result, vim.split(left, ','))
-        table.insert(result, ',')
-        vim.list_extend(result, vim.split(right, ','))
+    -- Empty strings mean that there is nothing there,
+    -- so empty table should be returned.
+    if value == '' then
+      return {}
+    end
 
-        table.sort(result)
+    -- Handles unescaped commas in a list.
+    if string.find(value, ',,,') then
+      local left, right = unpack(vim.split(value, ',,,'))
 
-        return result
-      end
+      local result = {}
+      vim.list_extend(result, vim.split(left, ','))
+      table.insert(result, ',')
+      vim.list_extend(result, vim.split(right, ','))
 
-      if string.find(value, ',^,,', 1, true) then
-        local comma_split = vim.split(value, ',^,,', true)
-        local left = comma_split[1]
-        local right = comma_split[2]
+      table.sort(result)
 
-        local result = {}
-        vim.list_extend(result, vim.split(left, ','))
-        table.insert(result, '^,')
-        vim.list_extend(result, vim.split(right, ','))
+      return result
+    end
 
-        table.sort(result)
+    if string.find(value, ',^,,', 1, true) then
+      local left, right = unpack(vim.split(value, ',^,,', true))
 
-        return result
-      end
+      local result = {}
+      vim.list_extend(result, vim.split(left, ','))
+      table.insert(result, '^,')
+      vim.list_extend(result, vim.split(right, ','))
 
-      return vim.split(value, ',')
-    end,
+      table.sort(result)
 
-    [OptionTypes.SET] = function(info, value)
-      if type(value) == 'table' then
-        return value
-      end
+      return result
+    end
 
-      -- Empty strings mean that there is nothing there,
-      -- so empty table should be returned.
-      if value == '' then
-        return {}
-      end
+    return vim.split(value, ',')
+  end,
 
-      assert(info.flaglist, 'That is the only one I know how to handle')
+  set = function(info, value)
+    if type(value) == 'table' then
+      return value
+    end
 
-      if info.flaglist and info.commalist then
-        local split_value = vim.split(value, ',')
-        local result = {}
-        for _, v in ipairs(split_value) do
-          result[v] = true
-        end
+    -- Empty strings mean that there is nothing there,
+    -- so empty table should be returned.
+    if value == '' then
+      return {}
+    end
 
-        return result
-      else
-        local result = {}
-        for i = 1, #value do
-          result[value:sub(i, i)] = true
-        end
+    assert(info.flaglist, 'That is the only one I know how to handle')
 
-        return result
+    if info.flaglist and info.commalist then
+      local split_value = vim.split(value, ',')
+      local result = {}
+      for _, v in ipairs(split_value) do
+        result[v] = true
       end
-    end,
 
-    [OptionTypes.MAP] = function(info, raw_value)
-      if type(raw_value) == 'table' then
-        return raw_value
+      return result
+    else
+      local result = {}
+      for i = 1, #value do
+        result[value:sub(i, i)] = true
       end
 
-      assert(info.commalist, 'Only commas are supported currently')
-
-      local result = {}
+      return result
+    end
+  end,
 
-      local comma_split = vim.split(raw_value, ',')
-      for _, key_value_str in ipairs(comma_split) do
-        local key, value = unpack(vim.split(key_value_str, ':'))
-        key = vim.trim(key)
+  map = function(info, raw_value)
+    if type(raw_value) == 'table' then
+      return raw_value
+    end
 
-        result[key] = value
-      end
+    assert(info.commalist, 'Only commas are supported currently')
 
-      return result
-    end,
-  }
+    local result = {}
 
-  return function(name, info, option_value)
-    return to_lua_value[get_option_type(name, info)](info, option_value)
-  end
-end)()
+    local comma_split = vim.split(raw_value, ',')
+    for _, key_value_str in ipairs(comma_split) do
+      local key, value = unpack(vim.split(key_value_str, ':'))
+      key = vim.trim(key)
 
---- Handles the mutation of various different values.
-local value_mutator = function(name, info, current, new, mutator)
-  return mutator[get_option_type(name, info)](current, new)
-end
+      result[key] = value
+    end
 
---- Handles the '^' operator
-local prepend_value = (function()
-  local methods = {
-    [OptionTypes.NUMBER] = function()
-      error("The '^' operator is not currently supported for")
-    end,
+    return result
+  end,
+}
 
-    [OptionTypes.STRING] = function(left, right)
-      return right .. left
-    end,
+--- Converts a vimoption_T style value to a Lua value
+local function convert_value_to_lua(info, option_value)
+  return to_lua_value[info.metatype](info, option_value)
+end
 
-    [OptionTypes.ARRAY] = function(left, right)
-      for i = #right, 1, -1 do
-        table.insert(left, 1, right[i])
-      end
+local prepend_methods = {
+  number = function()
+    error("The '^' operator is not currently supported for")
+  end,
 
-      return left
-    end,
+  string = function(left, right)
+    return right .. left
+  end,
 
-    [OptionTypes.MAP] = function(left, right)
-      return vim.tbl_extend('force', left, right)
-    end,
+  array = function(left, right)
+    for i = #right, 1, -1 do
+      table.insert(left, 1, right[i])
+    end
 
-    [OptionTypes.SET] = function(left, right)
-      return vim.tbl_extend('force', left, right)
-    end,
-  }
+    return left
+  end,
 
-  return function(name, info, current, new)
-    return value_mutator(
-      name,
-      info,
-      convert_value_to_lua(name, info, current),
-      convert_value_to_lua(name, info, new),
-      methods
-    )
-  end
-end)()
+  map = tbl_merge,
+  set = tbl_merge,
+}
 
---- Handles the '+' operator
-local add_value = (function()
-  local methods = {
-    [OptionTypes.NUMBER] = function(left, right)
-      return left + right
-    end,
+--- Handles the '^' operator
+local function prepend_value(info, current, new)
+  return prepend_methods[info.metatype](
+    convert_value_to_lua(info, current),
+    convert_value_to_lua(info, new)
+  )
+end
 
-    [OptionTypes.STRING] = function(left, right)
-      return left .. right
-    end,
+local add_methods = {
+  number = function(left, right)
+    return left + right
+  end,
 
-    [OptionTypes.ARRAY] = function(left, right)
-      for _, v in ipairs(right) do
-        table.insert(left, v)
-      end
+  string = function(left, right)
+    return left .. right
+  end,
 
-      return left
-    end,
+  array = function(left, right)
+    for _, v in ipairs(right) do
+      table.insert(left, v)
+    end
 
-    [OptionTypes.MAP] = function(left, right)
-      return vim.tbl_extend('force', left, right)
-    end,
+    return left
+  end,
 
-    [OptionTypes.SET] = function(left, right)
-      return vim.tbl_extend('force', left, right)
-    end,
-  }
+  map = tbl_merge,
+  set = tbl_merge,
+}
 
-  return function(name, info, current, new)
-    return value_mutator(
-      name,
-      info,
-      convert_value_to_lua(name, info, current),
-      convert_value_to_lua(name, info, new),
-      methods
-    )
-  end
-end)()
+--- Handles the '+' operator
+local function add_value(info, current, new)
+  return add_methods[info.metatype](
+    convert_value_to_lua(info, current),
+    convert_value_to_lua(info, new)
+  )
+end
 
---- Handles the '-' operator
-local remove_value = (function()
-  local remove_one_item = function(t, val)
-    if vim.tbl_islist(t) then
-      local remove_index = nil
-      for i, v in ipairs(t) do
-        if v == val then
-          remove_index = i
-        end
+local function remove_one_item(t, val)
+  if vim.tbl_islist(t) then
+    local remove_index = nil
+    for i, v in ipairs(t) do
+      if v == val then
+        remove_index = i
       end
+    end
 
-      if remove_index then
-        table.remove(t, remove_index)
-      end
-    else
-      t[val] = nil
+    if remove_index then
+      table.remove(t, remove_index)
     end
+  else
+    t[val] = nil
   end
+end
 
-  local methods = {
-    [OptionTypes.NUMBER] = function(left, right)
-      return left - right
-    end,
-
-    [OptionTypes.STRING] = function()
-      error('Subtraction not supported for strings.')
-    end,
-
-    [OptionTypes.ARRAY] = function(left, right)
-      if type(right) == 'string' then
-        remove_one_item(left, right)
-      else
-        for _, v in ipairs(right) do
-          remove_one_item(left, v)
-        end
-      end
+local remove_methods = {
+  number = function(left, right)
+    return left - right
+  end,
 
-      return left
-    end,
+  string = function()
+    error('Subtraction not supported for strings.')
+  end,
 
-    [OptionTypes.MAP] = function(left, right)
-      if type(right) == 'string' then
-        left[right] = nil
-      else
-        for _, v in ipairs(right) do
-          left[v] = nil
-        end
+  array = function(left, right)
+    if type(right) == 'string' then
+      remove_one_item(left, right)
+    else
+      for _, v in ipairs(right) do
+        remove_one_item(left, v)
       end
+    end
 
-      return left
-    end,
-
-    [OptionTypes.SET] = function(left, right)
-      if type(right) == 'string' then
-        left[right] = nil
-      else
-        for _, v in ipairs(right) do
-          left[v] = nil
-        end
-      end
+    return left
+  end,
 
-      return left
-    end,
-  }
+  map = tbl_remove,
+  set = tbl_remove,
+}
 
-  return function(name, info, current, new)
-    return value_mutator(name, info, convert_value_to_lua(name, info, current), new, methods)
-  end
-end)()
+--- Handles the '-' operator
+local function remove_value(info, current, new)
+  return remove_methods[info.metatype](convert_value_to_lua(info, current), new)
+end
 
-local create_option_metatable = function(set_type)
-  local set_mt, option_mt
+local function create_option_accessor(scope)
+  local option_mt
 
-  local make_option = function(name, value)
-    _setup()
+  local function make_option(name, value)
     local info = assert(options_info[name], 'Not a valid option name: ' .. name)
 
     if type(value) == 'table' and getmetatable(value) == option_mt then
@@ -651,67 +483,58 @@ local create_option_metatable = function(set_type)
     }, option_mt)
   end
 
-  local scope
-  if set_type == SET_TYPES.GLOBAL then
-    scope = 'global'
-  elseif set_type == SET_TYPES.LOCAL then
-    scope = 'local'
-  end
-
   option_mt = {
     -- To set a value, instead use:
     --  opt[my_option] = value
     _set = function(self)
       local value = convert_value_to_vim(self._name, self._info, self._value)
       a.nvim_set_option_value(self._name, value, { scope = scope })
-
-      return self
     end,
 
     get = function(self)
-      return convert_value_to_lua(self._name, self._info, self._value)
+      return convert_value_to_lua(self._info, self._value)
     end,
 
     append = function(self, right)
-      return self:__add(right):_set()
+      self._value = add_value(self._info, self._value, right)
+      self:_set()
     end,
 
     __add = function(self, right)
-      return make_option(self._name, add_value(self._name, self._info, self._value, right))
+      return make_option(self._name, add_value(self._info, self._value, right))
     end,
 
     prepend = function(self, right)
-      return self:__pow(right):_set()
+      self._value = prepend_value(self._info, self._value, right)
+      self:_set()
     end,
 
     __pow = function(self, right)
-      return make_option(self._name, prepend_value(self._name, self._info, self._value, right))
+      return make_option(self._name, prepend_value(self._info, self._value, right))
     end,
 
     remove = function(self, right)
-      return self:__sub(right):_set()
+      self._value = remove_value(self._info, self._value, right)
+      self:_set()
     end,
 
     __sub = function(self, right)
-      return make_option(self._name, remove_value(self._name, self._info, self._value, right))
+      return make_option(self._name, remove_value(self._info, self._value, right))
     end,
   }
   option_mt.__index = option_mt
 
-  set_mt = {
+  return setmetatable({}, {
     __index = function(_, k)
-      return make_option(k, a.nvim_get_option_value(k, { scope = scope }))
+      return make_option(k, a.nvim_get_option_value(k, {}))
     end,
 
     __newindex = function(_, k, v)
-      local opt = make_option(k, v)
-      opt:_set()
+      make_option(k, v):_set()
     end,
-  }
-
-  return set_mt
+  })
 end
 
-vim.opt = setmetatable({}, create_option_metatable(SET_TYPES.SET))
-vim.opt_local = setmetatable({}, create_option_metatable(SET_TYPES.LOCAL))
-vim.opt_global = setmetatable({}, create_option_metatable(SET_TYPES.GLOBAL))
+vim.opt = create_option_accessor()
+vim.opt_local = create_option_accessor('local')
+vim.opt_global = create_option_accessor('global')
diff --git a/runtime/lua/vim/compat.lua b/runtime/lua/vim/compat.lua
deleted file mode 100644
index 2c9786d491..0000000000
--- a/runtime/lua/vim/compat.lua
+++ /dev/null
@@ -1,12 +0,0 @@
--- Lua 5.1 forward-compatibility layer.
--- For background see https://github.com/neovim/neovim/pull/9280
---
--- Reference the lua-compat-5.2 project for hints:
---    https://github.com/keplerproject/lua-compat-5.2/blob/c164c8f339b95451b572d6b4b4d11e944dc7169d/compat52/mstrict.lua
---    https://github.com/keplerproject/lua-compat-5.2/blob/c164c8f339b95451b572d6b4b4d11e944dc7169d/tests/test.lua
-
-local lua_version = _VERSION:sub(-3)
-
-if lua_version >= '5.2' then
-  unpack = table.unpack -- luacheck: ignore 121 143
-end
diff --git a/runtime/lua/vim/diagnostic.lua b/runtime/lua/vim/diagnostic.lua
index 3f71d4f70d..6fd000a029 100644
--- a/runtime/lua/vim/diagnostic.lua
+++ b/runtime/lua/vim/diagnostic.lua
@@ -1,7 +1,8 @@
-local if_nil = vim.F.if_nil
+local api, if_nil = vim.api, vim.F.if_nil
 
 local M = {}
 
+---@enum DiagnosticSeverity
 M.severity = {
   ERROR = 1,
   WARN = 2,
@@ -45,18 +46,24 @@ local bufnr_and_namespace_cacher_mt = {
   end,
 }
 
-local diagnostic_cache = setmetatable({}, {
-  __index = function(t, bufnr)
-    assert(bufnr > 0, 'Invalid buffer number')
-    vim.api.nvim_buf_attach(bufnr, false, {
-      on_detach = function()
-        rawset(t, bufnr, nil) -- clear cache
-      end,
-    })
-    t[bufnr] = {}
-    return t[bufnr]
-  end,
-})
+local diagnostic_cache
+do
+  local group = api.nvim_create_augroup('DiagnosticBufWipeout', {})
+  diagnostic_cache = setmetatable({}, {
+    __index = function(t, bufnr)
+      assert(bufnr > 0, 'Invalid buffer number')
+      api.nvim_create_autocmd('BufWipeout', {
+        group = group,
+        buffer = bufnr,
+        callback = function()
+          rawset(t, bufnr, nil)
+        end,
+      })
+      t[bufnr] = {}
+      return t[bufnr]
+    end,
+  })
+end
 
 local diagnostic_cache_extmarks = setmetatable({}, bufnr_and_namespace_cacher_mt)
 local diagnostic_attached_buffers = {}
@@ -239,25 +246,12 @@ end)()
 ---@private
 local function get_bufnr(bufnr)
   if not bufnr or bufnr == 0 then
-    return vim.api.nvim_get_current_buf()
+    return api.nvim_get_current_buf()
   end
   return bufnr
 end
 
 ---@private
-local function is_disabled(namespace, bufnr)
-  local ns = M.get_namespace(namespace)
-  if ns.disabled then
-    return true
-  end
-
-  if type(diagnostic_disabled[bufnr]) == 'table' then
-    return diagnostic_disabled[bufnr][namespace]
-  end
-  return diagnostic_disabled[bufnr]
-end
-
----@private
 local function diagnostic_lines(diagnostics)
   if not diagnostics then
     return {}
@@ -293,7 +287,7 @@ end
 ---@private
 local function restore_extmarks(bufnr, last)
   for ns, extmarks in pairs(diagnostic_cache_extmarks[bufnr]) do
-    local extmarks_current = vim.api.nvim_buf_get_extmarks(bufnr, ns, 0, -1, { details = true })
+    local extmarks_current = api.nvim_buf_get_extmarks(bufnr, ns, 0, -1, { details = true })
     local found = {}
     for _, extmark in ipairs(extmarks_current) do
       -- nvim_buf_set_lines will move any extmark to the line after the last
@@ -306,7 +300,7 @@ local function restore_extmarks(bufnr, last)
       if not found[extmark[1]] then
         local opts = extmark[4]
         opts.id = extmark[1]
-        pcall(vim.api.nvim_buf_set_extmark, bufnr, ns, extmark[2], extmark[3], opts)
+        pcall(api.nvim_buf_set_extmark, bufnr, ns, extmark[2], extmark[3], opts)
       end
     end
   end
@@ -316,7 +310,7 @@ end
 local function save_extmarks(namespace, bufnr)
   bufnr = get_bufnr(bufnr)
   if not diagnostic_attached_buffers[bufnr] then
-    vim.api.nvim_buf_attach(bufnr, false, {
+    api.nvim_buf_attach(bufnr, false, {
       on_lines = function(_, _, _, _, _, last)
         restore_extmarks(bufnr, last - 1)
       end,
@@ -327,7 +321,7 @@ local function save_extmarks(namespace, bufnr)
     diagnostic_attached_buffers[bufnr] = true
   end
   diagnostic_cache_extmarks[bufnr][namespace] =
-    vim.api.nvim_buf_get_extmarks(bufnr, namespace, 0, -1, { details = true })
+    api.nvim_buf_get_extmarks(bufnr, namespace, 0, -1, { details = true })
 end
 
 local registered_autocmds = {}
@@ -351,19 +345,6 @@ local function execute_scheduled_display(namespace, bufnr)
   M.show(namespace, bufnr, nil, args)
 end
 
---- @deprecated
---- Callback scheduled when leaving Insert mode.
----
---- called from the Vimscript autocommand.
----
---- See @ref schedule_display()
----
----@private
-function M._execute_scheduled_display(namespace, bufnr)
-  vim.deprecate('vim.diagnostic._execute_scheduled_display', nil, '0.9')
-  execute_scheduled_display(namespace, bufnr)
-end
-
 --- Table of autocmd events to fire the update for displaying new diagnostic information
 local insert_leave_auto_cmds = { 'InsertLeave', 'CursorHoldI' }
 
@@ -373,8 +354,8 @@ local function schedule_display(namespace, bufnr, args)
 
   local key = make_augroup_key(namespace, bufnr)
   if not registered_autocmds[key] then
-    local group = vim.api.nvim_create_augroup(key, { clear = true })
-    vim.api.nvim_create_autocmd(insert_leave_auto_cmds, {
+    local group = api.nvim_create_augroup(key, { clear = true })
+    api.nvim_create_autocmd(insert_leave_auto_cmds, {
       group = group,
       buffer = bufnr,
       callback = function()
@@ -391,7 +372,7 @@ local function clear_scheduled_display(namespace, bufnr)
   local key = make_augroup_key(namespace, bufnr)
 
   if registered_autocmds[key] then
-    vim.api.nvim_del_augroup_by_name(key)
+    api.nvim_del_augroup_by_name(key)
     registered_autocmds[key] = nil
   end
 end
@@ -406,7 +387,7 @@ local function get_diagnostics(bufnr, opts, clamp)
   -- Memoized results of buf_line_count per bufnr
   local buf_line_count = setmetatable({}, {
     __index = function(t, k)
-      t[k] = vim.api.nvim_buf_line_count(k)
+      t[k] = api.nvim_buf_line_count(k)
       return rawget(t, k)
     end,
   })
@@ -414,7 +395,7 @@ local function get_diagnostics(bufnr, opts, clamp)
   ---@private
   local function add(b, d)
     if not opts.lnum or d.lnum == opts.lnum then
-      if clamp and vim.api.nvim_buf_is_loaded(b) then
+      if clamp and api.nvim_buf_is_loaded(b) then
         local line_count = buf_line_count[b] - 1
         if
           d.lnum > line_count
@@ -435,32 +416,31 @@ local function get_diagnostics(bufnr, opts, clamp)
     end
   end
 
+  ---@private
+  local function add_all_diags(buf, diags)
+    for _, diagnostic in pairs(diags) do
+      add(buf, diagnostic)
+    end
+  end
+
   if namespace == nil and bufnr == nil then
     for b, t in pairs(diagnostic_cache) do
       for _, v in pairs(t) do
-        for _, diagnostic in pairs(v) do
-          add(b, diagnostic)
-        end
+        add_all_diags(b, v)
       end
     end
   elseif namespace == nil then
     bufnr = get_bufnr(bufnr)
     for iter_namespace in pairs(diagnostic_cache[bufnr]) do
-      for _, diagnostic in pairs(diagnostic_cache[bufnr][iter_namespace]) do
-        add(bufnr, diagnostic)
-      end
+      add_all_diags(bufnr, diagnostic_cache[bufnr][iter_namespace])
     end
   elseif bufnr == nil then
     for b, t in pairs(diagnostic_cache) do
-      for _, diagnostic in pairs(t[namespace] or {}) do
-        add(b, diagnostic)
-      end
+      add_all_diags(b, t[namespace] or {})
     end
   else
     bufnr = get_bufnr(bufnr)
-    for _, diagnostic in pairs(diagnostic_cache[bufnr][namespace] or {}) do
-      add(bufnr, diagnostic)
-    end
+    add_all_diags(bufnr, diagnostic_cache[bufnr][namespace] or {})
   end
 
   if opts.severity then
@@ -478,7 +458,7 @@ local function set_list(loclist, opts)
   local winnr = opts.winnr or 0
   local bufnr
   if loclist then
-    bufnr = vim.api.nvim_win_get_buf(winnr)
+    bufnr = api.nvim_win_get_buf(winnr)
   end
   -- Don't clamp line numbers since the quickfix list can already handle line
   -- numbers beyond the end of the buffer
@@ -490,7 +470,7 @@ local function set_list(loclist, opts)
     vim.fn.setqflist({}, ' ', { title = title, items = items })
   end
   if open then
-    vim.api.nvim_command(loclist and 'lopen' or 'botright copen')
+    api.nvim_command(loclist and 'lwindow' or 'botright cwindow')
   end
 end
 
@@ -499,7 +479,7 @@ local function next_diagnostic(position, search_forward, bufnr, opts, namespace)
   position[1] = position[1] - 1
   bufnr = get_bufnr(bufnr)
   local wrap = vim.F.if_nil(opts.wrap, true)
-  local line_count = vim.api.nvim_buf_line_count(bufnr)
+  local line_count = api.nvim_buf_line_count(bufnr)
   local diagnostics =
     get_diagnostics(bufnr, vim.tbl_extend('keep', opts, { namespace = namespace }), true)
   local line_diagnostics = diagnostic_lines(diagnostics)
@@ -513,7 +493,7 @@ local function next_diagnostic(position, search_forward, bufnr, opts, namespace)
       lnum = (lnum + line_count) % line_count
     end
     if line_diagnostics[lnum] and not vim.tbl_isempty(line_diagnostics[lnum]) then
-      local line_length = #vim.api.nvim_buf_get_lines(bufnr, lnum, lnum + 1, true)[1]
+      local line_length = #api.nvim_buf_get_lines(bufnr, lnum, lnum + 1, true)[1]
       local sort_diagnostics, is_next
       if search_forward then
         sort_diagnostics = function(a, b)
@@ -549,17 +529,17 @@ local function diagnostic_move_pos(opts, pos)
   opts = opts or {}
 
   local float = vim.F.if_nil(opts.float, true)
-  local win_id = opts.win_id or vim.api.nvim_get_current_win()
+  local win_id = opts.win_id or api.nvim_get_current_win()
 
   if not pos then
-    vim.api.nvim_echo({ { 'No more valid diagnostics to move to', 'WarningMsg' } }, true, {})
+    api.nvim_echo({ { 'No more valid diagnostics to move to', 'WarningMsg' } }, true, {})
     return
   end
 
-  vim.api.nvim_win_call(win_id, function()
+  api.nvim_win_call(win_id, function()
     -- Save position in the window's jumplist
     vim.cmd("normal! m'")
-    vim.api.nvim_win_set_cursor(win_id, { pos[1] + 1, pos[2] })
+    api.nvim_win_set_cursor(win_id, { pos[1] + 1, pos[2] })
     -- Open folds under the cursor
     vim.cmd('normal! zv')
   end)
@@ -568,7 +548,7 @@ local function diagnostic_move_pos(opts, pos)
     local float_opts = type(float) == 'table' and float or {}
     vim.schedule(function()
       M.open_float(vim.tbl_extend('keep', float_opts, {
-        bufnr = vim.api.nvim_win_get_buf(win_id),
+        bufnr = api.nvim_win_get_buf(win_id),
         scope = 'cursor',
         focus = false,
       }))
@@ -585,12 +565,12 @@ end
 --- followed by namespace configuration, and finally global configuration.
 ---
 --- For example, if a user enables virtual text globally with
---- <pre>
+--- <pre>lua
 ---   vim.diagnostic.config({ virtual_text = true })
 --- </pre>
 ---
 --- and a diagnostic producer sets diagnostics with
---- <pre>
+--- <pre>lua
 ---   vim.diagnostic.set(ns, 0, diagnostics, { virtual_text = false })
 --- </pre>
 ---
@@ -620,16 +600,20 @@ end
 ---                       * spacing: (number) Amount of empty spaces inserted at the beginning
 ---                                  of the virtual text.
 ---                       * prefix: (string) Prepend diagnostic message with prefix.
+---                       * suffix: (string or function) Append diagnostic message with suffix.
+---                                 If a function, it must have the signature (diagnostic) ->
+---                                 string, where {diagnostic} is of type |diagnostic-structure|.
+---                                 This can be used to render an LSP diagnostic error code.
 ---                       * format: (function) A function that takes a diagnostic as input and
 ---                                 returns a string. The return value is the text used to display
 ---                                 the diagnostic. Example:
----                       <pre>
----                       function(diagnostic)
----                         if diagnostic.severity == vim.diagnostic.severity.ERROR then
----                           return string.format("E: %s", diagnostic.message)
+---                       <pre>lua
+---                         function(diagnostic)
+---                           if diagnostic.severity == vim.diagnostic.severity.ERROR then
+---                             return string.format("E: %s", diagnostic.message)
+---                           end
+---                           return diagnostic.message
 ---                         end
----                         return diagnostic.message
----                       end
 ---                       </pre>
 ---       - signs: (default true) Use signs for diagnostics. Options:
 ---                * severity: Only show signs for diagnostics matching the given severity
@@ -673,13 +657,13 @@ function M.config(opts, namespace)
 
   if namespace then
     for bufnr, v in pairs(diagnostic_cache) do
-      if vim.api.nvim_buf_is_loaded(bufnr) and v[namespace] then
+      if api.nvim_buf_is_loaded(bufnr) and v[namespace] then
         M.show(namespace, bufnr)
       end
     end
   else
     for bufnr, v in pairs(diagnostic_cache) do
-      if vim.api.nvim_buf_is_loaded(bufnr) then
+      if api.nvim_buf_is_loaded(bufnr) then
         for ns in pairs(v) do
           M.show(ns, bufnr)
         end
@@ -714,13 +698,14 @@ function M.set(namespace, bufnr, diagnostics, opts)
     set_diagnostic_cache(namespace, bufnr, diagnostics)
   end
 
-  if vim.api.nvim_buf_is_loaded(bufnr) then
+  if api.nvim_buf_is_loaded(bufnr) then
     M.show(namespace, bufnr, nil, opts)
   end
 
-  vim.api.nvim_exec_autocmds('DiagnosticChanged', {
+  api.nvim_exec_autocmds('DiagnosticChanged', {
     modeline = false,
     buffer = bufnr,
+    data = { diagnostics = diagnostics },
   })
 end
 
@@ -732,7 +717,7 @@ function M.get_namespace(namespace)
   vim.validate({ namespace = { namespace, 'n' } })
   if not all_namespaces[namespace] then
     local name
-    for k, v in pairs(vim.api.nvim_get_namespaces()) do
+    for k, v in pairs(api.nvim_get_namespaces()) do
       if namespace == v then
         name = k
         break
@@ -757,6 +742,18 @@ function M.get_namespaces()
   return vim.deepcopy(all_namespaces)
 end
 
+---@class Diagnostic
+---@field buffer number
+---@field lnum number 0-indexed
+---@field end_lnum nil|number 0-indexed
+---@field col number 0-indexed
+---@field end_col nil|number 0-indexed
+---@field severity DiagnosticSeverity
+---@field message string
+---@field source nil|string
+---@field code nil|string
+---@field user_data nil|any arbitrary data plugins can add
+
 --- Get current diagnostics.
 ---
 ---@param bufnr number|nil Buffer number to get diagnostics from. Use 0 for
@@ -765,7 +762,7 @@ end
 ---                        - namespace: (number) Limit diagnostics to the given namespace.
 ---                        - lnum: (number) Limit diagnostics to the given line number.
 ---                        - severity: See |diagnostic-severity|.
----@return table A list of diagnostic items |diagnostic-structure|.
+---@return Diagnostic[] table A list of diagnostic items |diagnostic-structure|.
 function M.get(bufnr, opts)
   vim.validate({
     bufnr = { bufnr, 'n', true },
@@ -777,22 +774,23 @@ end
 
 --- Get the previous diagnostic closest to the cursor position.
 ---
----@param opts table See |vim.diagnostic.goto_next()|
----@return table Previous diagnostic
+---@param opts nil|table See |vim.diagnostic.goto_next()|
+---@return Diagnostic|nil Previous diagnostic
 function M.get_prev(opts)
   opts = opts or {}
 
-  local win_id = opts.win_id or vim.api.nvim_get_current_win()
-  local bufnr = vim.api.nvim_win_get_buf(win_id)
-  local cursor_position = opts.cursor_position or vim.api.nvim_win_get_cursor(win_id)
+  local win_id = opts.win_id or api.nvim_get_current_win()
+  local bufnr = api.nvim_win_get_buf(win_id)
+  local cursor_position = opts.cursor_position or api.nvim_win_get_cursor(win_id)
 
   return next_diagnostic(cursor_position, false, bufnr, opts, opts.namespace)
 end
 
 --- Return the position of the previous diagnostic in the current buffer.
 ---
----@param opts table See |vim.diagnostic.goto_next()|
----@return table Previous diagnostic position as a (row, col) tuple.
+---@param opts table|nil See |vim.diagnostic.goto_next()|
+---@return table|false Previous diagnostic position as a (row, col) tuple or false if there is no
+---                    prior diagnostic
 function M.get_prev_pos(opts)
   local prev = M.get_prev(opts)
   if not prev then
@@ -803,29 +801,30 @@ function M.get_prev_pos(opts)
 end
 
 --- Move to the previous diagnostic in the current buffer.
----@param opts table See |vim.diagnostic.goto_next()|
+---@param opts table|nil See |vim.diagnostic.goto_next()|
 function M.goto_prev(opts)
   return diagnostic_move_pos(opts, M.get_prev_pos(opts))
 end
 
 --- Get the next diagnostic closest to the cursor position.
 ---
----@param opts table See |vim.diagnostic.goto_next()|
----@return table Next diagnostic
+---@param opts table|nil See |vim.diagnostic.goto_next()|
+---@return Diagnostic|nil Next diagnostic
 function M.get_next(opts)
   opts = opts or {}
 
-  local win_id = opts.win_id or vim.api.nvim_get_current_win()
-  local bufnr = vim.api.nvim_win_get_buf(win_id)
-  local cursor_position = opts.cursor_position or vim.api.nvim_win_get_cursor(win_id)
+  local win_id = opts.win_id or api.nvim_get_current_win()
+  local bufnr = api.nvim_win_get_buf(win_id)
+  local cursor_position = opts.cursor_position or api.nvim_win_get_cursor(win_id)
 
   return next_diagnostic(cursor_position, true, bufnr, opts, opts.namespace)
 end
 
 --- Return the position of the next diagnostic in the current buffer.
 ---
----@param opts table See |vim.diagnostic.goto_next()|
----@return table Next diagnostic position as a (row, col) tuple.
+---@param opts table|nil See |vim.diagnostic.goto_next()|
+---@return table|false Next diagnostic position as a (row, col) tuple or false if no next
+---                    diagnostic.
 function M.get_next_pos(opts)
   local next = M.get_next(opts)
   if not next then
@@ -909,7 +908,7 @@ M.handlers.signs = {
   end,
   hide = function(namespace, bufnr)
     local ns = M.get_namespace(namespace)
-    if ns.user_data.sign_group then
+    if ns.user_data.sign_group and api.nvim_buf_is_valid(bufnr) then
       vim.fn.sign_unplace(ns.user_data.sign_group, { buffer = bufnr })
     end
   end,
@@ -937,7 +936,7 @@ M.handlers.underline = {
 
     local ns = M.get_namespace(namespace)
     if not ns.user_data.underline_ns then
-      ns.user_data.underline_ns = vim.api.nvim_create_namespace('')
+      ns.user_data.underline_ns = api.nvim_create_namespace('')
     end
 
     local underline_ns = ns.user_data.underline_ns
@@ -964,7 +963,9 @@ M.handlers.underline = {
     local ns = M.get_namespace(namespace)
     if ns.user_data.underline_ns then
       diagnostic_cache_extmarks[bufnr][ns.user_data.underline_ns] = {}
-      vim.api.nvim_buf_clear_namespace(bufnr, ns.user_data.underline_ns, 0, -1)
+      if api.nvim_buf_is_valid(bufnr) then
+        api.nvim_buf_clear_namespace(bufnr, ns.user_data.underline_ns, 0, -1)
+      end
     end
   end,
 }
@@ -1003,7 +1004,7 @@ M.handlers.virtual_text = {
 
     local ns = M.get_namespace(namespace)
     if not ns.user_data.virt_text_ns then
-      ns.user_data.virt_text_ns = vim.api.nvim_create_namespace('')
+      ns.user_data.virt_text_ns = api.nvim_create_namespace('')
     end
 
     local virt_text_ns = ns.user_data.virt_text_ns
@@ -1015,7 +1016,7 @@ M.handlers.virtual_text = {
       local virt_texts = M._get_virt_text_chunks(line_diagnostics, opts.virtual_text)
 
       if virt_texts then
-        vim.api.nvim_buf_set_extmark(bufnr, virt_text_ns, line, 0, {
+        api.nvim_buf_set_extmark(bufnr, virt_text_ns, line, 0, {
           hl_mode = 'combine',
           virt_text = virt_texts,
         })
@@ -1027,7 +1028,9 @@ M.handlers.virtual_text = {
     local ns = M.get_namespace(namespace)
     if ns.user_data.virt_text_ns then
       diagnostic_cache_extmarks[bufnr][ns.user_data.virt_text_ns] = {}
-      vim.api.nvim_buf_clear_namespace(bufnr, ns.user_data.virt_text_ns, 0, -1)
+      if api.nvim_buf_is_valid(bufnr) then
+        api.nvim_buf_clear_namespace(bufnr, ns.user_data.virt_text_ns, 0, -1)
+      end
     end
   end,
 }
@@ -1045,6 +1048,7 @@ function M._get_virt_text_chunks(line_diags, opts)
 
   opts = opts or {}
   local prefix = opts.prefix or '■'
+  local suffix = opts.suffix or ''
   local spacing = opts.spacing or 4
 
   -- Create a little more space between virtual text and contents
@@ -1058,8 +1062,11 @@ function M._get_virt_text_chunks(line_diags, opts)
   -- TODO(tjdevries): Allow different servers to be shown first somehow?
   -- TODO(tjdevries): Display server name associated with these?
   if last.message then
+    if type(suffix) == 'function' then
+      suffix = suffix(last) or ''
+    end
     table.insert(virt_texts, {
-      string.format('%s %s', prefix, last.message:gsub('\r', ''):gsub('\n', '  ')),
+      string.format('%s %s%s', prefix, last.message:gsub('\r', ''):gsub('\n', '  '), suffix),
       virtual_text_highlight_map[last.severity],
     })
 
@@ -1099,6 +1106,27 @@ function M.hide(namespace, bufnr)
   end
 end
 
+--- Check whether diagnostics are disabled in a given buffer.
+---
+---@param bufnr number|nil Buffer number, or 0 for current buffer.
+---@param namespace number|nil Diagnostic namespace. When omitted, checks if
+---                            all diagnostics are disabled in {bufnr}.
+---                            Otherwise, only checks if diagnostics from
+---                            {namespace} are disabled.
+---@return boolean
+function M.is_disabled(bufnr, namespace)
+  bufnr = get_bufnr(bufnr)
+  if namespace and M.get_namespace(namespace).disabled then
+    return true
+  end
+
+  if type(diagnostic_disabled[bufnr]) == 'table' then
+    return diagnostic_disabled[bufnr][namespace]
+  end
+
+  return diagnostic_disabled[bufnr] ~= nil
+end
+
 --- Display diagnostics for the given namespace and buffer.
 ---
 ---@param namespace number|nil Diagnostic namespace. When omitted, show
@@ -1142,7 +1170,7 @@ function M.show(namespace, bufnr, diagnostics, opts)
     return
   end
 
-  if is_disabled(namespace, bufnr) then
+  if M.is_disabled(bufnr, namespace) then
     return
   end
 
@@ -1159,7 +1187,7 @@ function M.show(namespace, bufnr, diagnostics, opts)
   if opts.update_in_insert then
     clear_scheduled_display(namespace, bufnr)
   else
-    local mode = vim.api.nvim_get_mode()
+    local mode = api.nvim_get_mode()
     if string.sub(mode.mode, 1, 1) == 'i' then
       schedule_display(namespace, bufnr, opts)
       return
@@ -1226,7 +1254,9 @@ end
 ---                      string, it is prepended to each diagnostic in the window with no
 ---                      highlight.
 ---                      Overrides the setting from |vim.diagnostic.config()|.
----@return tuple ({float_bufnr}, {win_id})
+---            - suffix: Same as {prefix}, but appends the text to the diagnostic instead of
+---                      prepending it. Overrides the setting from |vim.diagnostic.config()|.
+---@return number|nil, number|nil: ({float_bufnr}, {win_id})
 function M.open_float(opts, ...)
   -- Support old (bufnr, opts) signature
   local bufnr
@@ -1257,7 +1287,7 @@ function M.open_float(opts, ...)
   local lnum, col
   if scope == 'line' or scope == 'cursor' then
     if not opts.pos then
-      local pos = vim.api.nvim_win_get_cursor(0)
+      local pos = api.nvim_win_get_cursor(0)
       lnum = pos[1] - 1
       col = pos[2]
     elseif type(opts.pos) == 'number' then
@@ -1279,7 +1309,7 @@ function M.open_float(opts, ...)
     end, diagnostics)
   elseif scope == 'cursor' then
     -- LSP servers can send diagnostics with `end_col` past the length of the line
-    local line_length = #vim.api.nvim_buf_get_lines(bufnr, lnum, lnum + 1, true)[1]
+    local line_length = #api.nvim_buf_get_lines(bufnr, lnum, lnum + 1, true)[1]
     diagnostics = vim.tbl_filter(function(d)
       return d.lnum == lnum
         and math.min(d.col, line_length - 1) <= col
@@ -1311,9 +1341,7 @@ function M.open_float(opts, ...)
     vim.validate({
       header = {
         header,
-        function(v)
-          return type(v) == 'string' or type(v) == 'table'
-        end,
+        { 'string', 'table' },
         "'string' or 'table'",
       },
     })
@@ -1321,11 +1349,11 @@ function M.open_float(opts, ...)
       -- Don't insert any lines for an empty string
       if string.len(if_nil(header[1], '')) > 0 then
         table.insert(lines, header[1])
-        table.insert(highlights, { 0, header[2] or 'Bold' })
+        table.insert(highlights, { hlname = header[2] or 'Bold' })
       end
     elseif #header > 0 then
       table.insert(lines, header)
-      table.insert(highlights, { 0, 'Bold' })
+      table.insert(highlights, { hlname = 'Bold' })
     end
   end
 
@@ -1347,9 +1375,7 @@ function M.open_float(opts, ...)
     vim.validate({
       prefix = {
         prefix_opt,
-        function(v)
-          return type(v) == 'string' or type(v) == 'table' or type(v) == 'function'
-        end,
+        { 'string', 'table', 'function' },
         "'string' or 'table' or 'function'",
       },
     })
@@ -1360,18 +1386,52 @@ function M.open_float(opts, ...)
     end
   end
 
+  local suffix_opt = if_nil(opts.suffix, function(diagnostic)
+    return diagnostic.code and string.format(' [%s]', diagnostic.code) or ''
+  end)
+
+  local suffix, suffix_hl_group
+  if suffix_opt then
+    vim.validate({
+      suffix = {
+        suffix_opt,
+        { 'string', 'table', 'function' },
+        "'string' or 'table' or 'function'",
+      },
+    })
+    if type(suffix_opt) == 'string' then
+      suffix, suffix_hl_group = suffix_opt, 'NormalFloat'
+    elseif type(suffix_opt) == 'table' then
+      suffix, suffix_hl_group = suffix_opt[1] or '', suffix_opt[2] or 'NormalFloat'
+    end
+  end
+
   for i, diagnostic in ipairs(diagnostics) do
     if prefix_opt and type(prefix_opt) == 'function' then
       prefix, prefix_hl_group = prefix_opt(diagnostic, i, #diagnostics)
       prefix, prefix_hl_group = prefix or '', prefix_hl_group or 'NormalFloat'
     end
+    if suffix_opt and type(suffix_opt) == 'function' then
+      suffix, suffix_hl_group = suffix_opt(diagnostic, i, #diagnostics)
+      suffix, suffix_hl_group = suffix or '', suffix_hl_group or 'NormalFloat'
+    end
     local hiname = floating_highlight_map[diagnostic.severity]
     local message_lines = vim.split(diagnostic.message, '\n')
-    table.insert(lines, prefix .. message_lines[1])
-    table.insert(highlights, { #prefix, hiname, prefix_hl_group })
-    for j = 2, #message_lines do
-      table.insert(lines, string.rep(' ', #prefix) .. message_lines[j])
-      table.insert(highlights, { 0, hiname })
+    for j = 1, #message_lines do
+      local pre = j == 1 and prefix or string.rep(' ', #prefix)
+      local suf = j == #message_lines and suffix or ''
+      table.insert(lines, pre .. message_lines[j] .. suf)
+      table.insert(highlights, {
+        hlname = hiname,
+        prefix = {
+          length = j == 1 and #prefix or 0,
+          hlname = prefix_hl_group,
+        },
+        suffix = {
+          length = j == #message_lines and #suffix or 0,
+          hlname = suffix_hl_group,
+        },
+      })
     end
   end
 
@@ -1380,12 +1440,17 @@ function M.open_float(opts, ...)
     opts.focus_id = scope
   end
   local float_bufnr, winnr = require('vim.lsp.util').open_floating_preview(lines, 'plaintext', opts)
-  for i, hi in ipairs(highlights) do
-    local prefixlen, hiname, prefix_hiname = unpack(hi)
-    if prefix_hiname then
-      vim.api.nvim_buf_add_highlight(float_bufnr, -1, prefix_hiname, i - 1, 0, prefixlen)
+  for i, hl in ipairs(highlights) do
+    local line = lines[i]
+    local prefix_len = hl.prefix and hl.prefix.length or 0
+    local suffix_len = hl.suffix and hl.suffix.length or 0
+    if prefix_len > 0 then
+      api.nvim_buf_add_highlight(float_bufnr, -1, hl.prefix.hlname, i - 1, 0, prefix_len)
+    end
+    api.nvim_buf_add_highlight(float_bufnr, -1, hl.hlname, i - 1, prefix_len, #line - suffix_len)
+    if suffix_len > 0 then
+      api.nvim_buf_add_highlight(float_bufnr, -1, hl.suffix.hlname, i - 1, #line - suffix_len, -1)
     end
-    vim.api.nvim_buf_add_highlight(float_bufnr, -1, hiname, i - 1, prefixlen, -1)
   end
 
   return float_bufnr, winnr
@@ -1416,10 +1481,15 @@ function M.reset(namespace, bufnr)
       M.hide(iter_namespace, iter_bufnr)
     end
 
-    vim.api.nvim_exec_autocmds('DiagnosticChanged', {
-      modeline = false,
-      buffer = iter_bufnr,
-    })
+    if api.nvim_buf_is_valid(iter_bufnr) then
+      api.nvim_exec_autocmds('DiagnosticChanged', {
+        modeline = false,
+        buffer = iter_bufnr,
+        data = { diagnostics = {} },
+      })
+    else
+      diagnostic_cache[iter_bufnr] = nil
+    end
   end
 end
 
@@ -1522,11 +1592,11 @@ end
 ---
 --- This can be parsed into a diagnostic |diagnostic-structure|
 --- with:
---- <pre>
---- local s = "WARNING filename:27:3: Variable 'foo' does not exist"
---- local pattern = "^(%w+) %w+:(%d+):(%d+): (.+)$"
---- local groups = { "severity", "lnum", "col", "message" }
---- vim.diagnostic.match(s, pattern, groups, { WARNING = vim.diagnostic.WARN })
+--- <pre>lua
+---   local s = "WARNING filename:27:3: Variable 'foo' does not exist"
+---   local pattern = "^(%w+) %w+:(%d+):(%d+): (.+)$"
+---   local groups = { "severity", "lnum", "col", "message" }
+---   vim.diagnostic.match(s, pattern, groups, { WARNING = vim.diagnostic.WARN })
 --- </pre>
 ---
 ---@param str string String to parse diagnostics from.
@@ -1538,7 +1608,7 @@ end
 ---@param defaults table|nil Table of default values for any fields not listed in {groups}.
 ---                          When omitted, numeric values default to 0 and "severity" defaults to
 ---                          ERROR.
----@return diagnostic |diagnostic-structure| or `nil` if {pat} fails to match {str}.
+---@return Diagnostic|nil: |diagnostic-structure| or `nil` if {pat} fails to match {str}.
 function M.match(str, pat, groups, severity_map, defaults)
   vim.validate({
     str = { str, 's' },
@@ -1585,7 +1655,7 @@ local errlist_type_map = {
 --- passed to |setqflist()| or |setloclist()|.
 ---
 ---@param diagnostics table List of diagnostics |diagnostic-structure|.
----@return array of quickfix list items |setqflist-what|
+---@return table[] of quickfix list items |setqflist-what|
 function M.toqflist(diagnostics)
   vim.validate({
     diagnostics = {
@@ -1610,7 +1680,11 @@ function M.toqflist(diagnostics)
   end
   table.sort(list, function(a, b)
     if a.bufnr == b.bufnr then
-      return a.lnum < b.lnum
+      if a.lnum == b.lnum then
+        return a.col < b.col
+      else
+        return a.lnum < b.lnum
+      end
     else
       return a.bufnr < b.bufnr
     end
@@ -1622,7 +1696,7 @@ end
 ---
 ---@param list table A list of quickfix items from |getqflist()| or
 ---            |getloclist()|.
----@return array of diagnostics |diagnostic-structure|
+---@return Diagnostic[] array of |diagnostic-structure|
 function M.fromqflist(list)
   vim.validate({
     list = {
diff --git a/runtime/lua/vim/filetype.lua b/runtime/lua/vim/filetype.lua
index 99c98764dd..9293c828b8 100644
--- a/runtime/lua/vim/filetype.lua
+++ b/runtime/lua/vim/filetype.lua
@@ -170,13 +170,14 @@ local extension = {
     return require('vim.filetype.detect').bindzone(bufnr, 'dcl')
   end,
   db = function(path, bufnr)
-    return require('vim.filetype.detect').bindzone(bufnr, '')
+    return require('vim.filetype.detect').bindzone(bufnr)
   end,
   bicep = 'bicep',
   bb = 'bitbake',
   bbappend = 'bitbake',
   bbclass = 'bitbake',
   bl = 'blank',
+  blp = 'blueprint',
   bsd = 'bsdl',
   bsdl = 'bsdl',
   bst = 'bst',
@@ -189,6 +190,7 @@ local extension = {
   BUILD = 'bzl',
   qc = 'c',
   cabal = 'cabal',
+  capnp = 'capnp',
   cdl = 'cdl',
   toc = 'cdrtoc',
   cfc = 'cf',
@@ -201,6 +203,7 @@ local extension = {
     return require('vim.filetype.detect').change(bufnr)
   end,
   chs = 'chaskell',
+  chatito = 'chatito',
   chopro = 'chordpro',
   crd = 'chordpro',
   crdpro = 'chordpro',
@@ -255,6 +258,7 @@ local extension = {
   cc = function(path, bufnr)
     return vim.g.cynlib_syntax_for_cc and 'cynlib' or 'cpp'
   end,
+  cql = 'cqlang',
   crm = 'crm',
   csx = 'cs',
   cs = 'cs',
@@ -324,11 +328,7 @@ local extension = {
   end,
   eex = 'eelixir',
   leex = 'eelixir',
-  am = function(path, bufnr)
-    if not path:lower():find('makefile%.am$') then
-      return 'elf'
-    end
-  end,
+  am = 'elf',
   exs = 'elixir',
   elm = 'elm',
   elv = 'elvish',
@@ -416,15 +416,18 @@ local extension = {
   fs = function(path, bufnr)
     return require('vim.filetype.detect').fs(bufnr)
   end,
+  fsh = 'fsh',
   fsi = 'fsharp',
   fsx = 'fsharp',
   fusion = 'fusion',
   gdb = 'gdb',
   gdmo = 'gdmo',
   mo = 'gdmo',
-  tres = 'gdresource',
   tscn = 'gdresource',
+  tres = 'gdresource',
   gd = 'gdscript',
+  gdshader = 'gdshader',
+  shader = 'gdshader',
   ged = 'gedcom',
   gmi = 'gemtext',
   gemini = 'gemtext',
@@ -444,6 +447,8 @@ local extension = {
   gsp = 'gsp',
   gjs = 'javascript.glimmer',
   gts = 'typescript.glimmer',
+  gyp = 'gyp',
+  gypi = 'gyp',
   hack = 'hack',
   hackpartial = 'hack',
   haml = 'haml',
@@ -469,6 +474,8 @@ local extension = {
   hex = 'hex',
   ['h32'] = 'hex',
   hjson = 'hjson',
+  m3u = 'hlsplaylist',
+  m3u8 = 'hlsplaylist',
   hog = 'hog',
   hws = 'hollywood',
   hoon = 'hoon',
@@ -538,6 +545,7 @@ local extension = {
   mjs = 'javascript',
   javascript = 'javascript',
   js = 'javascript',
+  jsm = 'javascript',
   cjs = 'javascript',
   jsx = 'javascriptreact',
   clp = 'jess',
@@ -546,6 +554,7 @@ local extension = {
   jov = 'jovial',
   jovial = 'jovial',
   properties = 'jproperties',
+  jq = 'jq',
   slnf = 'json',
   json = 'json',
   jsonp = 'json',
@@ -554,6 +563,8 @@ local extension = {
   ['json-patch'] = 'json',
   json5 = 'json5',
   jsonc = 'jsonc',
+  jsonnet = 'jsonnet',
+  libsonnet = 'jsonnet',
   jsp = 'jsp',
   jl = 'julia',
   kv = 'kivy',
@@ -600,11 +611,14 @@ local extension = {
   c = function(path, bufnr)
     return require('vim.filetype.detect').lpc(bufnr)
   end,
-  lsl = 'lsl',
+  lsl = function(path, bufnr)
+    return require('vim.filetype.detect').lsl(bufnr)
+  end,
   lss = 'lss',
   nse = 'lua',
   rockspec = 'lua',
   lua = 'lua',
+  lrc = 'lyrics',
   m = function(path, bufnr)
     return require('vim.filetype.detect').m(bufnr)
   end,
@@ -644,6 +658,9 @@ local extension = {
   dmt = 'maxima',
   wxm = 'maxima',
   mel = 'mel',
+  mmd = 'mermaid',
+  mmdc = 'mermaid',
+  mermaid = 'mermaid',
   mf = 'mf',
   mgl = 'mgl',
   mgp = 'mgp',
@@ -662,6 +679,7 @@ local extension = {
   DEF = 'modula2',
   ['m2'] = 'modula2',
   mi = 'modula2',
+  lm3 = 'modula3',
   ssc = 'monk',
   monk = 'monk',
   tsc = 'monk',
@@ -695,6 +713,9 @@ local extension = {
   nanorc = 'nanorc',
   ncf = 'ncf',
   nginx = 'nginx',
+  nim = 'nim',
+  nims = 'nim',
+  nimble = 'nim',
   ninja = 'ninja',
   nix = 'nix',
   nqc = 'nqc',
@@ -707,6 +728,10 @@ local extension = {
   nsi = 'nsis',
   nsh = 'nsis',
   obj = 'obj',
+  obl = 'obse',
+  obse = 'obse',
+  oblivion = 'obse',
+  obscript = 'obse',
   mlt = 'ocaml',
   mly = 'ocaml',
   mll = 'ocaml',
@@ -720,6 +745,7 @@ local extension = {
   opam = 'opam',
   ['or'] = 'openroad',
   scad = 'openscad',
+  ovpn = 'openvpn',
   ora = 'ora',
   org = 'org',
   org_archive = 'org',
@@ -741,6 +767,7 @@ local extension = {
   php = 'php',
   phpt = 'php',
   phtml = 'php',
+  theme = 'php',
   pike = 'pike',
   pmod = 'pike',
   rcp = 'pilrc',
@@ -758,6 +785,7 @@ local extension = {
   po = 'po',
   pot = 'po',
   pod = 'pod',
+  filter = 'poefilter',
   pk = 'poke',
   ps = 'postscr',
   epsi = 'postscr',
@@ -900,7 +928,9 @@ local extension = {
   sig = function(path, bufnr)
     return require('vim.filetype.detect').sig(bufnr)
   end,
-  sil = 'sil',
+  sil = function(path, bufnr)
+    return require('vim.filetype.detect').sil(bufnr)
+  end,
   sim = 'simula',
   ['s85'] = 'sinda',
   sin = 'sinda',
@@ -916,12 +946,14 @@ local extension = {
   ice = 'slice',
   score = 'slrnsc',
   sol = 'solidity',
+  smali = 'smali',
   tpl = 'smarty',
   ihlp = 'smcl',
   smcl = 'smcl',
   hlp = 'smcl',
   smith = 'smith',
   smt = 'smith',
+  smithy = 'smithy',
   sml = 'sml',
   spt = 'snobol4',
   sno = 'snobol4',
@@ -950,6 +982,9 @@ local extension = {
   srec = 'srec',
   mot = 'srec',
   ['s19'] = 'srec',
+  srt = 'srt',
+  ssa = 'ssa',
+  ass = 'ssa',
   st = 'st',
   imata = 'stata',
   ['do'] = 'stata',
@@ -987,6 +1022,7 @@ local extension = {
   texinfo = 'texinfo',
   text = 'text',
   tfvars = 'terraform-vars',
+  thrift = 'thrift',
   tla = 'tla',
   tli = 'tli',
   toml = 'toml',
@@ -1002,6 +1038,8 @@ local extension = {
   ts = function(path, bufnr)
     return M.getlines(bufnr, 1):find('<%?xml') and 'xml' or 'typescript'
   end,
+  mts = 'typescript',
+  cts = 'typescript',
   tsx = 'typescriptreact',
   uc = 'uc',
   uit = 'uil',
@@ -1011,6 +1049,12 @@ local extension = {
   dsm = 'vb',
   ctl = 'vb',
   vbs = 'vb',
+  vdf = 'vdf',
+  vdmpp = 'vdmpp',
+  vpp = 'vdmpp',
+  vdmrt = 'vdmrt',
+  vdmsl = 'vdmsl',
+  vdm = 'vdmsl',
   vr = 'vera',
   vri = 'vera',
   vrh = 'vera',
@@ -1023,6 +1067,7 @@ local extension = {
   hdl = 'vhdl',
   vho = 'vhdl',
   vbe = 'vhdl',
+  tape = 'vhs',
   vim = 'vim',
   vba = 'vim',
   mar = 'vmasm',
@@ -1032,6 +1077,7 @@ local extension = {
   vue = 'vue',
   wat = 'wast',
   wast = 'wast',
+  wdl = 'wdl',
   wm = 'webmacro',
   wbt = 'winbatch',
   wml = 'wml',
@@ -1078,6 +1124,7 @@ local extension = {
   yang = 'yang',
   ['z8a'] = 'z8a',
   zig = 'zig',
+  zir = 'zir',
   zu = 'zimbu',
   zut = 'zimbutempl',
   zsh = 'zsh',
@@ -1276,9 +1323,9 @@ local filename = {
   ['GNUmakefile.am'] = 'automake',
   ['named.root'] = 'bindzone',
   WORKSPACE = 'bzl',
+  ['WORKSPACE.bzlmod'] = 'bzl',
   BUILD = 'bzl',
   ['cabal.project'] = 'cabalproject',
-  [vim.env.HOME .. '/cabal.config'] = 'cabalconfig',
   ['cabal.config'] = 'cabalconfig',
   calendar = 'calendar',
   catalog = 'catalog',
@@ -1332,13 +1379,13 @@ local filename = {
   npmrc = 'dosini',
   ['/etc/yum.conf'] = 'dosini',
   ['.npmrc'] = 'dosini',
-  ['.editorconfig'] = 'dosini',
   ['/etc/pacman.conf'] = 'confini',
   ['mpv.conf'] = 'confini',
   dune = 'dune',
   jbuild = 'dune',
   ['dune-workspace'] = 'dune',
   ['dune-project'] = 'dune',
+  ['.editorconfig'] = 'editorconfig',
   ['elinks.conf'] = 'elinks',
   ['mix.lock'] = 'elixir',
   ['filter-rules'] = 'elmfilt',
@@ -1371,6 +1418,8 @@ local filename = {
   ['EDIT_DESCRIPTION'] = 'gitcommit',
   ['.gitconfig'] = 'gitconfig',
   ['.gitmodules'] = 'gitconfig',
+  ['.gitattributes'] = 'gitattributes',
+  ['.gitignore'] = 'gitignore',
   ['gitolite.conf'] = 'gitolite',
   ['git-rebase-todo'] = 'gitrebase',
   gkrellmrc = 'gkrellmrc',
@@ -1379,6 +1428,7 @@ local filename = {
   gnashpluginrc = 'gnash',
   gnashrc = 'gnash',
   ['.gnuplot'] = 'gnuplot',
+  ['go.sum'] = 'gosum',
   ['go.work'] = 'gowork',
   ['.gprc'] = 'gp',
   ['/.gnupg/gpg.conf'] = 'gpg',
@@ -1408,11 +1458,15 @@ local filename = {
   ['ipf.conf'] = 'ipfilter',
   ['ipf6.conf'] = 'ipfilter',
   ['ipf.rules'] = 'ipfilter',
-  ['.eslintrc'] = 'json',
-  ['.babelrc'] = 'json',
   ['Pipfile.lock'] = 'json',
   ['.firebaserc'] = 'json',
   ['.prettierrc'] = 'json',
+  ['.babelrc'] = 'jsonc',
+  ['.eslintrc'] = 'jsonc',
+  ['.hintrc'] = 'jsonc',
+  ['.jsfmtrc'] = 'jsonc',
+  ['.jshintrc'] = 'jsonc',
+  ['.swrc'] = 'jsonc',
   Kconfig = 'kconfig',
   ['Kconfig.debug'] = 'kconfig',
   ['lftp.conf'] = 'lftp',
@@ -1427,6 +1481,10 @@ local filename = {
   ['.sawfishrc'] = 'lisp',
   ['/etc/login.access'] = 'loginaccess',
   ['/etc/login.defs'] = 'logindefs',
+  ['.lsl'] = function(path, bufnr)
+    return require('vim.filetype.detect').lsl(bufnr)
+  end,
+  ['.luacheckrc'] = 'lua',
   ['lynx.cfg'] = 'lynx',
   ['m3overrides'] = 'm3build',
   ['m3makefile'] = 'm3build',
@@ -1472,8 +1530,11 @@ local filename = {
   ['/etc/shadow-'] = 'passwd',
   ['/etc/shadow'] = 'passwd',
   ['/etc/passwd.edit'] = 'passwd',
+  ['latexmkrc'] = 'perl',
+  ['.latexmkrc'] = 'perl',
   ['pf.conf'] = 'pf',
   ['main.cf'] = 'pfmain',
+  ['main.cf.proto'] = 'pfmain',
   pinerc = 'pine',
   ['.pinercex'] = 'pine',
   ['.pinerc'] = 'pine',
@@ -1506,6 +1567,9 @@ local filename = {
   ['.pythonstartup'] = 'python',
   ['.pythonrc'] = 'python',
   SConstruct = 'python',
+  ['.Rprofile'] = 'r',
+  ['Rprofile'] = 'r',
+  ['Rprofile.site'] = 'r',
   ratpoisonrc = 'ratpoison',
   ['.ratpoisonrc'] = 'ratpoison',
   inputrc = 'readline',
@@ -1632,6 +1696,8 @@ local filename = {
   fglrxrc = 'xml',
   ['/etc/blkid.tab'] = 'xml',
   ['/etc/blkid.tab.old'] = 'xml',
+  ['.clang-format'] = 'yaml',
+  ['.clang-tidy'] = 'yaml',
   ['/etc/zprofile'] = 'zsh',
   ['.zlogin'] = 'zsh',
   ['.zlogout'] = 'zsh',
@@ -1684,6 +1750,7 @@ local pattern = {
   ['.*/meta%-.*/conf/.*%.conf'] = 'bitbake',
   ['bzr_log%..*'] = 'bzr',
   ['.*enlightenment/.*%.cfg'] = 'c',
+  ['${HOME}/cabal%.config'] = 'cabalconfig',
   ['cabal%.project%..*'] = starsetf('cabalproject'),
   ['.*/%.calendar/.*'] = starsetf('calendar'),
   ['.*/share/calendar/.*/calendar%..*'] = starsetf('calendar'),
@@ -1702,7 +1769,7 @@ local pattern = {
     { priority = -1 },
   },
   ['[cC]hange[lL]og.*'] = starsetf(function(path, bufnr)
-    require('vim.filetype.detect').changelog(bufnr)
+    return require('vim.filetype.detect').changelog(bufnr)
   end),
   ['.*%.%.ch'] = 'chill',
   ['.*%.cmake%.in'] = 'cmake',
@@ -1744,6 +1811,8 @@ local pattern = {
   ['.*/etc/DIR_COLORS'] = 'dircolors',
   ['.*/etc/dnsmasq%.conf'] = 'dnsmasq',
   ['php%.ini%-.*'] = 'dosini',
+  ['.*/%.aws/config'] = 'confini',
+  ['.*/%.aws/credentials'] = 'confini',
   ['.*/etc/pacman%.conf'] = 'confini',
   ['.*/etc/yum%.conf'] = 'dosini',
   ['.*lvs'] = 'dracula',
@@ -1808,26 +1877,21 @@ local pattern = {
   ['.*/%.config/git/config'] = 'gitconfig',
   ['.*%.git/config%.worktree'] = 'gitconfig',
   ['.*%.git/worktrees/.*/config%.worktree'] = 'gitconfig',
-  ['.*/git/config'] = function(path, bufnr)
-    if vim.env.XDG_CONFIG_HOME and path:find(vim.env.XDG_CONFIG_HOME .. '/git/config') then
-      return 'gitconfig'
-    end
-  end,
+  ['${XDG_CONFIG_HOME}/git/config'] = 'gitconfig',
+  ['.*%.git/info/attributes'] = 'gitattributes',
+  ['.*/etc/gitattributes'] = 'gitattributes',
+  ['.*/%.config/git/attributes'] = 'gitattributes',
+  ['${XDG_CONFIG_HOME}/git/attributes'] = 'gitattributes',
+  ['.*%.git/info/exclude'] = 'gitignore',
+  ['.*/%.config/git/ignore'] = 'gitignore',
+  ['${XDG_CONFIG_HOME}/git/ignore'] = 'gitignore',
   ['%.gitsendemail%.msg%.......'] = 'gitsendemail',
   ['gkrellmrc_.'] = 'gkrellmrc',
   ['.*/usr/.*/gnupg/options%.skel'] = 'gpg',
   ['.*/%.gnupg/options'] = 'gpg',
   ['.*/%.gnupg/gpg%.conf'] = 'gpg',
-  ['.*/options'] = function(path, bufnr)
-    if vim.env.GNUPGHOME and path:find(vim.env.GNUPGHOME .. '/options') then
-      return 'gpg'
-    end
-  end,
-  ['.*/gpg%.conf'] = function(path, bufnr)
-    if vim.env.GNUPGHOME and path:find(vim.env.GNUPGHOME .. '/gpg%.conf') then
-      return 'gpg'
-    end
-  end,
+  ['${GNUPGHOME}/options'] = 'gpg',
+  ['${GNUPGHOME}/gpg%.conf'] = 'gpg',
   ['.*/etc/group'] = 'group',
   ['.*/etc/gshadow'] = 'group',
   ['.*/etc/group%.edit'] = 'group',
@@ -1841,7 +1905,7 @@ local pattern = {
   ['.*/etc/grub%.conf'] = 'grub',
   -- gtkrc* and .gtkrc*
   ['%.?gtkrc.*'] = starsetf('gtkrc'),
-  [vim.env.VIMRUNTIME .. '/doc/.*%.txt'] = 'help',
+  ['${VIMRUNTIME}/doc/.*%.txt'] = 'help',
   ['hg%-editor%-.*%.txt'] = 'hgcommit',
   ['.*/etc/host%.conf'] = 'hostconf',
   ['.*/etc/hosts%.deny'] = 'hostsaccess',
@@ -1855,7 +1919,9 @@ local pattern = {
   ['Prl.*%..*'] = starsetf('jam'),
   ['.*%.properties_..'] = 'jproperties',
   ['.*%.properties_.._..'] = 'jproperties',
+  ['org%.eclipse%..*%.prefs'] = 'jproperties',
   ['.*%.properties_.._.._.*'] = starsetf('jproperties'),
+  ['[jt]sconfig.*%.json'] = 'jsonc',
   ['Kconfig%..*'] = starsetf('kconfig'),
   ['.*%.[Ss][Uu][Bb]'] = 'krl',
   ['lilo%.conf.*'] = starsetf('lilo'),
@@ -2009,6 +2075,7 @@ local pattern = {
   ['.*%.ml%.cppo'] = 'ocaml',
   ['.*%.mli%.cppo'] = 'ocaml',
   ['.*%.opam%.template'] = 'opam',
+  ['.*/openvpn/.*/.*%.conf'] = 'openvpn',
   ['.*%.[Oo][Pp][Ll]'] = 'opl',
   ['.*/etc/pam%.conf'] = 'pamconf',
   ['.*/etc/pam%.d/.*'] = starsetf('pamconf'),
@@ -2245,13 +2312,14 @@ end
 --- Filename patterns can specify an optional priority to resolve cases when a
 --- file path matches multiple patterns. Higher priorities are matched first.
 --- When omitted, the priority defaults to 0.
+--- A pattern can contain environment variables of the form "${SOME_VAR}" that will
+--- be automatically expanded. If the environment variable is not set, the pattern
+--- won't be matched.
 ---
 --- See $VIMRUNTIME/lua/vim/filetype.lua for more examples.
 ---
---- Note that Lua filetype detection is disabled when |g:do_legacy_filetype| is set.
----
 --- Example:
---- <pre>
+--- <pre>lua
 ---  vim.filetype.add({
 ---    extension = {
 ---      foo = 'fooscript',
@@ -2273,6 +2341,8 @@ end
 ---      ['.*/etc/foo/.*'] = 'fooscript',
 ---      -- Using an optional priority
 ---      ['.*/etc/foo/.*%.conf'] = { 'dosini', { priority = 10 } },
+---      -- A pattern containing an environment variable
+---      ['${XDG_CONFIG_HOME}/foo/git'] = 'git',
 ---      ['README.(%a+)$'] = function(path, bufnr, ext)
 ---        if ext == 'md' then
 ---          return 'markdown'
@@ -2284,8 +2354,8 @@ end
 ---  })
 --- </pre>
 ---
---- To add a fallback match on contents (see |new-filetype-scripts|), use
---- <pre>
+--- To add a fallback match on contents, use
+--- <pre>lua
 --- vim.filetype.add {
 ---   pattern = {
 ---     ['.*'] = {
@@ -2346,8 +2416,28 @@ local function dispatch(ft, path, bufnr, ...)
   end
 end
 
+-- Lookup table/cache for patterns that contain an environment variable pattern, e.g. ${SOME_VAR}.
+local expand_env_lookup = {}
+
 ---@private
 local function match_pattern(name, path, tail, pat)
+  if expand_env_lookup[pat] == nil then
+    expand_env_lookup[pat] = pat:find('%${') ~= nil
+  end
+  if expand_env_lookup[pat] then
+    local return_early
+    pat = pat:gsub('%${(%S-)}', function(env)
+      -- If an environment variable is present in the pattern but not set, there is no match
+      if not vim.env[env] then
+        return_early = true
+        return nil
+      end
+      return vim.env[env]
+    end)
+    if return_early then
+      return false
+    end
+  end
   -- If the pattern contains a / match against the full path, otherwise just the tail
   local fullpat = '^' .. pat .. '$'
   local matches
@@ -2377,7 +2467,7 @@ end
 --- Each of the three options is specified using a key to the single argument of this function.
 --- Example:
 ---
---- <pre>
+--- <pre>lua
 ---   -- Using a buffer number
 ---   vim.filetype.match({ buf = 42 })
 ---
diff --git a/runtime/lua/vim/filetype/detect.lua b/runtime/lua/vim/filetype/detect.lua
index 2be9dcff88..edffdde9c7 100644
--- a/runtime/lua/vim/filetype/detect.lua
+++ b/runtime/lua/vim/filetype/detect.lua
@@ -181,7 +181,7 @@ function M.cls(bufnr)
     return vim.g.filetype_cls
   end
   local line = getlines(bufnr, 1)
-  if line:find('^%%') then
+  if line:find('^[%%\\]') then
     return 'tex'
   elseif line:find('^#') and line:lower():find('rexx') then
     return 'rexx'
@@ -634,6 +634,19 @@ function M.lpc(bufnr)
   return 'c'
 end
 
+function M.lsl(bufnr)
+  if vim.g.filetype_lsl then
+    return vim.g.filetype_lsl
+  end
+
+  local line = nextnonblank(bufnr, 1)
+  if findany(line, { '^%s*%%', ':%s*trait%s*$' }) then
+    return 'larch'
+  else
+    return 'lsl'
+  end
+end
+
 function M.m(bufnr)
   if vim.g.filetype_m then
     return vim.g.filetype_m
@@ -1084,11 +1097,10 @@ function M.sc(bufnr)
   for _, line in ipairs(getlines(bufnr, 1, 25)) do
     if
       findany(line, {
-        '[A-Za-z0-9]*%s:%s[A-Za-z0-9]',
         'var%s<',
         'classvar%s<',
         '%^this.*',
-        '|%w*|',
+        '|%w+|',
         '%+%s%w*%s{',
         '%*ar%s',
       })
@@ -1153,13 +1165,14 @@ function M.sh(path, contents, name)
       vim.b[b].is_bash = nil
       vim.b[b].is_sh = nil
     end
-  elseif vim.g.bash_is_sh or matchregex(name, [[\<bash\>]]) or matchregex(name, [[\<bash2\>]]) then
+  elseif vim.g.bash_is_sh or matchregex(name, [[\<\(bash\|bash2\)\>]]) then
     on_detect = function(b)
       vim.b[b].is_bash = 1
       vim.b[b].is_kornshell = nil
       vim.b[b].is_sh = nil
     end
-  elseif matchregex(name, [[\<sh\>]]) then
+    -- Ubuntu links sh to dash
+  elseif matchregex(name, [[\<\(sh\|dash\)\>]]) then
     on_detect = function(b)
       vim.b[b].is_sh = 1
       vim.b[b].is_kornshell = nil
@@ -1194,6 +1207,19 @@ function M.shell(path, contents, name)
   return name
 end
 
+-- Swift Intermediate Language or SILE
+function M.sil(bufnr)
+  for _, line in ipairs(getlines(bufnr, 1, 100)) do
+    if line:find('^%s*[\\%%]') then
+      return 'sile'
+    elseif line:find('^%s*%S') then
+      return 'sil'
+    end
+  end
+  -- No clue, default to "sil"
+  return 'sil'
+end
+
 -- SMIL or SNMP MIB file
 function M.smi(bufnr)
   local line = getlines(bufnr, 1)
@@ -1230,17 +1256,15 @@ end
 -- 2. Check the first 1000 non-comment lines for LaTeX or ConTeXt keywords.
 -- 3. Default to "plain" or to g:tex_flavor, can be set in user's vimrc.
 function M.tex(path, bufnr)
-  local format = getlines(bufnr, 1):find('^%%&%s*(%a+)')
-  if format then
+  local matched, _, format = getlines(bufnr, 1):find('^%%&%s*(%a+)')
+  if matched then
     format = format:lower():gsub('pdf', '', 1)
-    if format == 'tex' then
-      return 'tex'
-    elseif format == 'plaintex' then
-      return 'plaintex'
-    end
   elseif path:lower():find('tex/context/.*/.*%.tex') then
     return 'context'
   else
+    -- Default value, may be changed later:
+    format = vim.g.tex_flavor or 'plaintex'
+
     local lpat = [[documentclass\>\|usepackage\>\|begin{\|newcommand\>\|renewcommand\>]]
     local cpat =
       [[start\a\+\|setup\a\+\|usemodule\|enablemode\|enableregime\|setvariables\|useencoding\|usesymbols\|stelle\a\+\|verwende\a\+\|stel\a\+\|gebruik\a\+\|usa\a\+\|imposta\a\+\|regle\a\+\|utilisemodule\>]]
@@ -1249,26 +1273,25 @@ function M.tex(path, bufnr)
       -- Find first non-comment line
       if not l:find('^%s*%%%S') then
         -- Check the next thousand lines for a LaTeX or ConTeXt keyword.
-        for _, line in ipairs(getlines(bufnr, i + 1, i + 1000)) do
-          local lpat_match, cpat_match =
-            matchregex(line, [[\c^\s*\\\%(]] .. lpat .. [[\)\|^\s*\\\(]] .. cpat .. [[\)]])
-          if lpat_match then
+        for _, line in ipairs(getlines(bufnr, i, i + 1000)) do
+          if matchregex(line, [[\c^\s*\\\%(]] .. lpat .. [[\)]]) then
             return 'tex'
-          elseif cpat_match then
+          elseif matchregex(line, [[\c^\s*\\\%(]] .. cpat .. [[\)]]) then
             return 'context'
           end
         end
       end
     end
-    -- TODO: add AMSTeX, RevTex, others?
-    if not vim.g.tex_flavor or vim.g.tex_flavor == 'plain' then
-      return 'plaintex'
-    elseif vim.g.tex_flavor == 'context' then
-      return 'context'
-    else
-      -- Probably LaTeX
-      return 'tex'
-    end
+  end -- if matched
+
+  -- Translation from formats to file types.  TODO:  add AMSTeX, RevTex, others?
+  if format == 'plain' then
+    return 'plaintex'
+  elseif format == 'plaintex' or format == 'context' then
+    return format
+  else
+    -- Probably LaTeX
+    return 'tex'
   end
 end
 
@@ -1434,8 +1457,8 @@ local function match_from_hashbang(contents, path)
     name = 'wish'
   end
 
-  if matchregex(name, [[^\(bash\d*\|\|ksh\d*\|sh\)\>]]) then
-    -- Bourne-like shell scripts: bash bash2 ksh ksh93 sh
+  if matchregex(name, [[^\(bash\d*\|dash\|ksh\d*\|sh\)\>]]) then
+    -- Bourne-like shell scripts: bash bash2 dash ksh ksh93 sh
     return require('vim.filetype.detect').sh(path, contents, first_line)
   elseif matchregex(name, [[^csh\>]]) then
     return require('vim.filetype.detect').shell(path, contents, vim.g.filetype_csh or 'csh')
diff --git a/runtime/lua/vim/fs.lua b/runtime/lua/vim/fs.lua
index ce845eda15..a0d2c4c339 100644
--- a/runtime/lua/vim/fs.lua
+++ b/runtime/lua/vim/fs.lua
@@ -1,9 +1,11 @@
 local M = {}
 
+local iswin = vim.loop.os_uname().sysname == 'Windows_NT'
+
 --- Iterate over all the parents of the given file or directory.
 ---
 --- Example:
---- <pre>
+--- <pre>lua
 --- local root_dir
 --- for dir in vim.fs.parents(vim.api.nvim_buf_get_name(0)) do
 ---   if vim.fn.isdirectory(dir .. "/.git") == 1 then
@@ -40,7 +42,19 @@ function M.dirname(file)
   if file == nil then
     return nil
   end
-  return vim.fn.fnamemodify(file, ':h')
+  vim.validate({ file = { file, 's' } })
+  if iswin and file:match('^%w:[\\/]?$') then
+    return (file:gsub('\\', '/'))
+  elseif not file:match('[\\/]') then
+    return '.'
+  elseif file == '/' or file:match('^/[^/]+$') then
+    return '/'
+  end
+  local dir = file:match('[/\\]$') and file:sub(1, #file - 1) or file:match('^([/\\]?.+)[/\\]')
+  if iswin and dir:match('^%w:$') then
+    return dir .. '/'
+  end
+  return (dir:gsub('\\', '/'))
 end
 
 --- Return the basename of the given file or directory
@@ -48,21 +62,75 @@ end
 ---@param file (string) File or directory
 ---@return (string) Basename of {file}
 function M.basename(file)
-  return vim.fn.fnamemodify(file, ':t')
+  if file == nil then
+    return nil
+  end
+  vim.validate({ file = { file, 's' } })
+  if iswin and file:match('^%w:[\\/]?$') then
+    return ''
+  end
+  return file:match('[/\\]$') and '' or (file:match('[^\\/]*$'):gsub('\\', '/'))
+end
+
+---@private
+local function join_paths(...)
+  return (table.concat({ ... }, '/'):gsub('//+', '/'))
 end
 
 --- Return an iterator over the files and directories located in {path}
 ---
 ---@param path (string) An absolute or relative path to the directory to iterate
 ---            over. The path is first normalized |vim.fs.normalize()|.
+--- @param opts table|nil Optional keyword arguments:
+---             - depth: integer|nil How deep the traverse (default 1)
+---             - skip: (fun(dir_name: string): boolean)|nil Predicate
+---               to control traversal. Return false to stop searching the current directory.
+---               Only useful when depth > 1
+---
 ---@return Iterator over files and directories in {path}. Each iteration yields
 ---        two values: name and type. Each "name" is the basename of the file or
 ---        directory relative to {path}. Type is one of "file" or "directory".
-function M.dir(path)
-  return function(fs)
-    return vim.loop.fs_scandir_next(fs)
-  end,
-    vim.loop.fs_scandir(M.normalize(path))
+function M.dir(path, opts)
+  opts = opts or {}
+
+  vim.validate({
+    path = { path, { 'string' } },
+    depth = { opts.depth, { 'number' }, true },
+    skip = { opts.skip, { 'function' }, true },
+  })
+
+  if not opts.depth or opts.depth == 1 then
+    return function(fs)
+      return vim.loop.fs_scandir_next(fs)
+    end,
+      vim.loop.fs_scandir(M.normalize(path))
+  end
+
+  --- @async
+  return coroutine.wrap(function()
+    local dirs = { { path, 1 } }
+    while #dirs > 0 do
+      local dir0, level = unpack(table.remove(dirs, 1))
+      local dir = level == 1 and dir0 or join_paths(path, dir0)
+      local fs = vim.loop.fs_scandir(M.normalize(dir))
+      while fs do
+        local name, t = vim.loop.fs_scandir_next(fs)
+        if not name then
+          break
+        end
+        local f = level == 1 and name or join_paths(dir0, name)
+        coroutine.yield(f, t)
+        if
+          opts.depth
+          and level < opts.depth
+          and t == 'directory'
+          and (not opts.skip or opts.skip(f) ~= false)
+        then
+          dirs[#dirs + 1] = { f, level + 1 }
+        end
+      end
+    end
+  end)
 end
 
 --- Find files or directories in the given path.
@@ -73,14 +141,18 @@ end
 --- searches are recursive and may search through many directories! If {stop}
 --- is non-nil, then the search stops when the directory given in {stop} is
 --- reached. The search terminates when {limit} (default 1) matches are found.
---- The search can be narrowed to find only files or or only directories by
+--- The search can be narrowed to find only files or only directories by
 --- specifying {type} to be "file" or "directory", respectively.
 ---
----@param names (string|table) Names of the files and directories to find. Must
----             be base names, paths and globs are not supported.
+---@param names (string|table|fun(name: string): boolean) Names of the files
+---             and directories to find.
+---             Must be base names, paths and globs are not supported.
+---             The function is called per file and directory within the
+---             traversed directories to test if they match {names}.
+---
 ---@param opts (table) Optional keyword arguments:
 ---                       - path (string): Path to begin searching from. If
----                              omitted, the current working directory is used.
+---                              omitted, the |current-directory| is used.
 ---                       - upward (boolean, default false): If true, search
 ---                                upward through parent directories. Otherwise,
 ---                                search through child directories
@@ -89,16 +161,16 @@ end
 ---                              reached. The directory itself is not searched.
 ---                       - type (string): Find only files ("file") or
 ---                              directories ("directory"). If omitted, both
----                              files and directories that match {name} are
+---                              files and directories that match {names} are
 ---                              included.
 ---                       - limit (number, default 1): Stop the search after
 ---                               finding this many matches. Use `math.huge` to
 ---                               place no limit on the number of matches.
----@return (table) The paths of all matching files or directories
+---@return (table) Normalized paths |vim.fs.normalize()| of all matching files or directories
 function M.find(names, opts)
   opts = opts or {}
   vim.validate({
-    names = { names, { 's', 't' } },
+    names = { names, { 's', 't', 'f' } },
     path = { opts.path, 's', true },
     upward = { opts.upward, 'b', true },
     stop = { opts.stop, 's', true },
@@ -123,18 +195,31 @@ function M.find(names, opts)
   end
 
   if opts.upward then
-    ---@private
-    local function test(p)
-      local t = {}
-      for _, name in ipairs(names) do
-        local f = p .. '/' .. name
-        local stat = vim.loop.fs_stat(f)
-        if stat and (not opts.type or opts.type == stat.type) then
-          t[#t + 1] = f
+    local test
+
+    if type(names) == 'function' then
+      test = function(p)
+        local t = {}
+        for name, type in M.dir(p) do
+          if names(name) and (not opts.type or opts.type == type) then
+            table.insert(t, join_paths(p, name))
+          end
         end
+        return t
       end
+    else
+      test = function(p)
+        local t = {}
+        for _, name in ipairs(names) do
+          local f = join_paths(p, name)
+          local stat = vim.loop.fs_stat(f)
+          if stat and (not opts.type or opts.type == stat.type) then
+            t[#t + 1] = f
+          end
+        end
 
-      return t
+        return t
+      end
     end
 
     for _, match in ipairs(test(path)) do
@@ -162,17 +247,25 @@ function M.find(names, opts)
         break
       end
 
-      for other, type in M.dir(dir) do
-        local f = dir .. '/' .. other
-        for _, name in ipairs(names) do
-          if name == other and (not opts.type or opts.type == type) then
+      for other, type_ in M.dir(dir) do
+        local f = join_paths(dir, other)
+        if type(names) == 'function' then
+          if names(other) and (not opts.type or opts.type == type_) then
             if add(f) then
               return matches
             end
           end
+        else
+          for _, name in ipairs(names) do
+            if name == other and (not opts.type or opts.type == type_) then
+              if add(f) then
+                return matches
+              end
+            end
+          end
         end
 
-        if type == 'directory' then
+        if type_ == 'directory' then
           dirs[#dirs + 1] = f
         end
       end
@@ -187,23 +280,29 @@ end
 --- backslash (\\) characters are converted to forward slashes (/). Environment
 --- variables are also expanded.
 ---
---- Example:
---- <pre>
---- vim.fs.normalize('C:\\Users\\jdoe')
---- => 'C:/Users/jdoe'
+--- Examples:
+--- <pre>lua
+---   vim.fs.normalize('C:\\\\Users\\\\jdoe')
+---   --> 'C:/Users/jdoe'
 ---
---- vim.fs.normalize('~/src/neovim')
---- => '/home/jdoe/src/neovim'
+---   vim.fs.normalize('~/src/neovim')
+---   --> '/home/jdoe/src/neovim'
 ---
---- vim.fs.normalize('$XDG_CONFIG_HOME/nvim/init.vim')
---- => '/Users/jdoe/.config/nvim/init.vim'
+---   vim.fs.normalize('$XDG_CONFIG_HOME/nvim/init.vim')
+---   --> '/Users/jdoe/.config/nvim/init.vim'
 --- </pre>
 ---
 ---@param path (string) Path to normalize
 ---@return (string) Normalized path
 function M.normalize(path)
   vim.validate({ path = { path, 's' } })
-  return (path:gsub('^~/', vim.env.HOME .. '/'):gsub('%$([%w_]+)', vim.env):gsub('\\', '/'))
+  return (
+    path
+      :gsub('^~$', vim.loop.os_homedir())
+      :gsub('^~/', vim.loop.os_homedir() .. '/')
+      :gsub('%$([%w_]+)', vim.loop.os_getenv)
+      :gsub('\\', '/')
+  )
 end
 
 return M
diff --git a/runtime/lua/vim/health.lua b/runtime/lua/vim/health.lua
index b875da0abc..044880e076 100644
--- a/runtime/lua/vim/health.lua
+++ b/runtime/lua/vim/health.lua
@@ -23,7 +23,20 @@ end
 local path2name = function(path)
   if path:match('%.lua$') then
     -- Lua: transform "../lua/vim/lsp/health.lua" into "vim.lsp"
-    return path:gsub('.-lua[%\\%/]', '', 1):gsub('[%\\%/]', '.'):gsub('%.health.-$', '')
+
+    -- Get full path, make sure all slashes are '/'
+    path = vim.fs.normalize(path)
+
+    -- Remove everything up to the last /lua/ folder
+    path = path:gsub('^.*/lua/', '')
+
+    -- Remove the filename (health.lua)
+    path = vim.fn.fnamemodify(path, ':h')
+
+    -- Change slashes to dots
+    path = path:gsub('/', '.')
+
+    return path
   else
     -- Vim: transform "../autoload/health/provider.vim" into "provider"
     return vim.fn.fnamemodify(path, ':t:r')
diff --git a/runtime/lua/vim/highlight.lua b/runtime/lua/vim/highlight.lua
index ddd504a0e0..20ad48dd27 100644
--- a/runtime/lua/vim/highlight.lua
+++ b/runtime/lua/vim/highlight.lua
@@ -5,6 +5,7 @@ local M = {}
 M.priorities = {
   syntax = 50,
   treesitter = 100,
+  semantic_tokens = 125,
   diagnostics = 150,
   user = 200,
 }
@@ -41,7 +42,7 @@ end
 ---@param start first position (tuple {line,col})
 ---@param finish second position (tuple {line,col})
 ---@param opts table with options:
---             - regtype type of range (:help setreg, default charwise)
+--             - regtype type of range (see |setreg()|, default charwise)
 --             - inclusive boolean indicating whether the range is end-inclusive (default false)
 --             - priority number indicating priority of highlight (default priorities.user)
 function M.range(bufnr, ns, higroup, start, finish, opts)
diff --git a/runtime/lua/vim/inspect.lua b/runtime/lua/vim/inspect.lua
index 0a53fb203b..c232f69590 100644
--- a/runtime/lua/vim/inspect.lua
+++ b/runtime/lua/vim/inspect.lua
@@ -89,8 +89,38 @@ local function escape(str)
   )
 end
 
+-- List of lua keywords
+local luaKeywords = {
+  ['and'] = true,
+  ['break'] = true,
+  ['do'] = true,
+  ['else'] = true,
+  ['elseif'] = true,
+  ['end'] = true,
+  ['false'] = true,
+  ['for'] = true,
+  ['function'] = true,
+  ['goto'] = true,
+  ['if'] = true,
+  ['in'] = true,
+  ['local'] = true,
+  ['nil'] = true,
+  ['not'] = true,
+  ['or'] = true,
+  ['repeat'] = true,
+  ['return'] = true,
+  ['then'] = true,
+  ['true'] = true,
+  ['until'] = true,
+  ['while'] = true,
+}
+
 local function isIdentifier(str)
-  return type(str) == 'string' and not not str:match('^[_%a][_%a%d]*$')
+  return type(str) == 'string'
+    -- identifier must start with a letter and underscore, and be followed by letters, numbers, and underscores
+    and not not str:match('^[_%a][_%a%d]*$')
+    -- lua keywords are not valid identifiers
+    and not luaKeywords[str]
 end
 
 local flr = math.floor
diff --git a/runtime/lua/vim/keymap.lua b/runtime/lua/vim/keymap.lua
index 219de16b5c..ef1c66ea20 100644
--- a/runtime/lua/vim/keymap.lua
+++ b/runtime/lua/vim/keymap.lua
@@ -2,7 +2,7 @@ local keymap = {}
 
 --- Add a new |mapping|.
 --- Examples:
---- <pre>
+--- <pre>lua
 ---   -- Can add mapping to Lua functions
 ---   vim.keymap.set('n', 'lhs', function() print("real lua function") end)
 ---
@@ -21,13 +21,13 @@ local keymap = {}
 --- </pre>
 ---
 --- Note that in a mapping like:
---- <pre>
+--- <pre>lua
 ---    vim.keymap.set('n', 'asdf', require('jkl').my_fun)
 --- </pre>
 ---
 --- the ``require('jkl')`` gets evaluated during this call in order to access the function.
 --- If you want to avoid this cost at startup you can wrap it in a function, for example:
---- <pre>
+--- <pre>lua
 ---    vim.keymap.set('n', 'asdf', function() return require('jkl').my_fun() end)
 --- </pre>
 ---
@@ -36,17 +36,17 @@ local keymap = {}
 ---@param lhs string           Left-hand side |{lhs}| of the mapping.
 ---@param rhs string|function  Right-hand side |{rhs}| of the mapping. Can also be a Lua function.
 --
----@param opts table A table of |:map-arguments|.
----                  + Accepts options accepted by the {opts} parameter in |nvim_set_keymap()|,
----                    with the following notable differences:
----                    - replace_keycodes: Defaults to `true` if "expr" is `true`.
----                    - noremap: Always overridden with the inverse of "remap" (see below).
----                  + In addition to those options, the table accepts the following keys:
----                    - buffer: (number or boolean) Add a mapping to the given buffer.
----                    When `0` or `true`, use the current buffer.
----                    - remap: (boolean) Make the mapping recursive.
----                    This is the inverse of the "noremap" option from |nvim_set_keymap()|.
----                    Defaults to `false`.
+---@param opts table|nil A table of |:map-arguments|.
+---                      + Accepts options accepted by the {opts} parameter in |nvim_set_keymap()|,
+---                        with the following notable differences:
+---                        - replace_keycodes: Defaults to `true` if "expr" is `true`.
+---                        - noremap: Always overridden with the inverse of "remap" (see below).
+---                      + In addition to those options, the table accepts the following keys:
+---                        - buffer: (number or boolean) Add a mapping to the given buffer.
+---                        When `0` or `true`, use the current buffer.
+---                        - remap: (boolean) Make the mapping recursive.
+---                        This is the inverse of the "noremap" option from |nvim_set_keymap()|.
+---                        Defaults to `false`.
 ---@see |nvim_set_keymap()|
 function keymap.set(mode, lhs, rhs, opts)
   vim.validate({
@@ -57,7 +57,6 @@ function keymap.set(mode, lhs, rhs, opts)
   })
 
   opts = vim.deepcopy(opts) or {}
-  local is_rhs_luaref = type(rhs) == 'function'
   mode = type(mode) == 'string' and { mode } or mode
 
   if opts.expr and opts.replace_keycodes ~= false then
@@ -73,7 +72,7 @@ function keymap.set(mode, lhs, rhs, opts)
     opts.remap = nil
   end
 
-  if is_rhs_luaref then
+  if type(rhs) == 'function' then
     opts.callback = rhs
     rhs = ''
   end
@@ -94,14 +93,14 @@ end
 
 --- Remove an existing mapping.
 --- Examples:
---- <pre>
+--- <pre>lua
 ---   vim.keymap.del('n', 'lhs')
 ---
 ---   vim.keymap.del({'n', 'i', 'v'}, '<leader>w', { buffer = 5 })
 --- </pre>
----@param opts table A table of optional arguments:
----                  - buffer: (number or boolean) Remove a mapping from the given buffer.
----                  When "true" or 0, use the current buffer.
+---@param opts table|nil A table of optional arguments:
+---                      - buffer: (number or boolean) Remove a mapping from the given buffer.
+---                      When "true" or 0, use the current buffer.
 ---@see |vim.keymap.set()|
 ---
 function keymap.del(modes, lhs, opts)
diff --git a/runtime/lua/vim/lsp.lua b/runtime/lua/vim/lsp.lua
index fd64c1a495..c5392ac154 100644
--- a/runtime/lua/vim/lsp.lua
+++ b/runtime/lua/vim/lsp.lua
@@ -4,8 +4,8 @@ local lsp_rpc = require('vim.lsp.rpc')
 local protocol = require('vim.lsp.protocol')
 local util = require('vim.lsp.util')
 local sync = require('vim.lsp.sync')
+local semantic_tokens = require('vim.lsp.semantic_tokens')
 
-local vim = vim
 local api = vim.api
 local nvim_err_writeln, nvim_buf_get_lines, nvim_command, nvim_buf_get_option, nvim_exec_autocmds =
   api.nvim_err_writeln,
@@ -26,6 +26,7 @@ local lsp = {
   buf = require('vim.lsp.buf'),
   diagnostic = require('vim.lsp.diagnostic'),
   codelens = require('vim.lsp.codelens'),
+  semantic_tokens = semantic_tokens,
   util = util,
 
   -- Allow raw RPC access.
@@ -57,6 +58,8 @@ lsp._request_name_to_capability = {
   ['textDocument/formatting'] = { 'documentFormattingProvider' },
   ['textDocument/completion'] = { 'completionProvider' },
   ['textDocument/documentHighlight'] = { 'documentHighlightProvider' },
+  ['textDocument/semanticTokens/full'] = { 'semanticTokensProvider' },
+  ['textDocument/semanticTokens/full/delta'] = { 'semanticTokensProvider' },
 }
 
 -- TODO improve handling of scratch buffers with LSP attached.
@@ -64,7 +67,7 @@ lsp._request_name_to_capability = {
 ---@private
 --- Concatenates and writes a list of strings to the Vim error buffer.
 ---
----@param {...} (List of strings) List to write to the buffer
+---@param {...} table[] List to write to the buffer
 local function err_message(...)
   nvim_err_writeln(table.concat(vim.tbl_flatten({ ... })))
   nvim_command('redraw')
@@ -73,7 +76,7 @@ end
 ---@private
 --- Returns the buffer number for the given {bufnr}.
 ---
----@param bufnr (number) Buffer number to resolve. Defaults to the current
+---@param bufnr (number|nil) Buffer number to resolve. Defaults to the current
 ---buffer if not given.
 ---@returns bufnr (number) Number of requested buffer
 local function resolve_bufnr(bufnr)
@@ -241,9 +244,9 @@ end
 ---@private
 --- Augments a validator function with support for optional (nil) values.
 ---
----@param fn (function(v)) The original validator function; should return a
+---@param fn (fun(v)) The original validator function; should return a
 ---bool.
----@returns (function(v)) The augmented function. Also returns true if {v} is
+---@returns (fun(v)) The augmented function. Also returns true if {v} is
 ---`nil`.
 local function optional_validator(fn)
   return function(v)
@@ -289,7 +292,12 @@ local function validate_client_config(config)
     'flags.debounce_text_changes must be a number with the debounce time in milliseconds'
   )
 
-  local cmd, cmd_args = lsp._cmd_parts(config.cmd)
+  local cmd, cmd_args
+  if type(config.cmd) == 'function' then
+    cmd = config.cmd
+  else
+    cmd, cmd_args = lsp._cmd_parts(config.cmd)
+  end
   local offset_encoding = valid_encodings.UTF16
   if config.offset_encoding then
     offset_encoding = validate_encoding(config.offset_encoding)
@@ -809,8 +817,7 @@ end
 --- Attaches the current buffer to the client.
 ---
 --- Example:
----
---- <pre>
+--- <pre>lua
 --- vim.lsp.start({
 ---    name = 'my-server-name',
 ---    cmd = {'name-of-language-server-executable'},
@@ -818,25 +825,18 @@ end
 --- })
 --- </pre>
 ---
---- See |lsp.start_client| for all available options. The most important are:
----
---- `name` is an arbitrary name for the LSP client. It should be unique per
---- language server.
+--- See |vim.lsp.start_client()| for all available options. The most important are:
 ---
---- `cmd` the command as list - used to start the language server.
---- The command must be present in the `$PATH` environment variable or an
---- absolute path to the executable. Shell constructs like `~` are *NOT* expanded.
----
---- `root_dir` path to the project root.
---- By default this is used to decide if an existing client should be re-used.
---- The example above uses |vim.fs.find| and |vim.fs.dirname| to detect the
---- root by traversing the file system upwards starting
---- from the current directory until either a `pyproject.toml` or `setup.py`
---- file is found.
----
---- `workspace_folders` a list of { uri:string, name: string } tables.
---- The project root folders used by the language server.
---- If `nil` the property is derived from the `root_dir` for convenience.
+--- - `name` arbitrary name for the LSP client. Should be unique per language server.
+--- - `cmd` command (in list form) used to start the language server. Must be absolute, or found on
+---   `$PATH`. Shell constructs like `~` are not expanded.
+--- - `root_dir` path to the project root. By default this is used to decide if an existing client
+---   should be re-used. The example above uses |vim.fs.find()| and |vim.fs.dirname()| to detect the
+---   root by traversing the file system upwards starting from the current directory until either
+---   a `pyproject.toml` or `setup.py` file is found.
+--- - `workspace_folders` list of `{ uri:string, name: string }` tables specifying the project root
+---   folders used by the language server. If `nil` the property is derived from `root_dir` for
+---   convenience.
 ---
 --- Language servers use this information to discover metadata like the
 --- dependencies of your project and they tend to index the contents within the
@@ -844,26 +844,35 @@ end
 ---
 ---
 --- To ensure a language server is only started for languages it can handle,
---- make sure to call |vim.lsp.start| within a |FileType| autocmd.
+--- make sure to call |vim.lsp.start()| within a |FileType| autocmd.
 --- Either use |:au|, |nvim_create_autocmd()| or put the call in a
 --- `ftplugin/<filetype_name>.lua` (See |ftplugin-name|)
 ---
----@param config table Same configuration as documented in |lsp.start_client()|
+---@param config table Same configuration as documented in |vim.lsp.start_client()|
 ---@param opts nil|table Optional keyword arguments:
 ---             - reuse_client (fun(client: client, config: table): boolean)
 ---                            Predicate used to decide if a client should be re-used.
 ---                            Used on all running clients.
 ---                            The default implementation re-uses a client if name
 ---                            and root_dir matches.
----@return number client_id
+---             - bufnr (number)
+---                     Buffer handle to attach to if starting or re-using a
+---                     client (0 for current).
+---@return number|nil client_id
 function lsp.start(config, opts)
   opts = opts or {}
   local reuse_client = opts.reuse_client
     or function(client, conf)
       return client.config.root_dir == conf.root_dir and client.name == conf.name
     end
-  config.name = config.name or (config.cmd[1] and vim.fs.basename(config.cmd[1])) or nil
-  local bufnr = api.nvim_get_current_buf()
+  config.name = config.name
+  if not config.name and type(config.cmd) == 'table' then
+    config.name = config.cmd[1] and vim.fs.basename(config.cmd[1]) or nil
+  end
+  local bufnr = opts.bufnr
+  if bufnr == nil or bufnr == 0 then
+    bufnr = api.nvim_get_current_buf()
+  end
   for _, clients in ipairs({ uninitialized_clients, lsp.get_active_clients() }) do
     for _, client in pairs(clients) do
       if reuse_client(client, config) then
@@ -888,110 +897,114 @@ end
 --
 --- Starts and initializes a client with the given configuration.
 ---
---- Parameter `cmd` is required.
----
---- The following parameters describe fields in the {config} table.
----
----
----@param cmd: (required, string or list treated like |jobstart()|) Base command
---- that initiates the LSP client.
----
----@param cmd_cwd: (string, default=|getcwd()|) Directory to launch
---- the `cmd` process. Not related to `root_dir`.
----
----@param cmd_env: (table) Environment flags to pass to the LSP on
---- spawn.  Can be specified using keys like a map or as a list with `k=v`
---- pairs or both. Non-string values are coerced to string.
---- Example:
---- <pre>
---- { "PRODUCTION=true"; "TEST=123"; PORT = 8080; HOST = "0.0.0.0"; }
---- </pre>
----
----@param detached: (boolean, default true) Daemonize the server process so that it runs in a
---- separate process group from Nvim. Nvim will shutdown the process on exit, but if Nvim fails to
---- exit cleanly this could leave behind orphaned server processes.
----
----@param workspace_folders (table) List of workspace folders passed to the
---- language server. For backwards compatibility rootUri and rootPath will be
---- derived from the first workspace folder in this list. See `workspaceFolders` in
---- the LSP spec.
----
----@param capabilities Map overriding the default capabilities defined by
---- |vim.lsp.protocol.make_client_capabilities()|, passed to the language
---- server on initialization. Hint: use make_client_capabilities() and modify
---- its result.
---- - Note: To send an empty dictionary use
----   `{[vim.type_idx]=vim.types.dictionary}`, else it will be encoded as an
----   array.
+--- Field `cmd` in {config} is required.
+---
+---@param config (table) Configuration for the server:
+--- - cmd: (table|string|fun(dispatchers: table):table) command string or
+---       list treated like |jobstart()|. The command must launch the language server
+---       process. `cmd` can also be a function that creates an RPC client.
+---       The function receives a dispatchers table and must return a table with the
+---       functions `request`, `notify`, `is_closing` and `terminate`
+---       See |vim.lsp.rpc.request()| and |vim.lsp.rpc.notify()|
+---       For TCP there is a built-in rpc client factory: |vim.lsp.rpc.connect()|
+---
+--- - cmd_cwd: (string, default=|getcwd()|) Directory to launch
+---       the `cmd` process. Not related to `root_dir`.
+---
+--- - cmd_env: (table) Environment flags to pass to the LSP on
+---       spawn.  Can be specified using keys like a map or as a list with `k=v`
+---       pairs or both. Non-string values are coerced to string.
+---       Example:
+---       <pre>
+---                   { "PRODUCTION=true"; "TEST=123"; PORT = 8080; HOST = "0.0.0.0"; }
+---       </pre>
+---
+--- - detached: (boolean, default true) Daemonize the server process so that it runs in a
+---       separate process group from Nvim. Nvim will shutdown the process on exit, but if Nvim fails to
+---       exit cleanly this could leave behind orphaned server processes.
+---
+--- - workspace_folders: (table) List of workspace folders passed to the
+---       language server. For backwards compatibility rootUri and rootPath will be
+---       derived from the first workspace folder in this list. See `workspaceFolders` in
+---       the LSP spec.
+---
+--- - capabilities: Map overriding the default capabilities defined by
+---       |vim.lsp.protocol.make_client_capabilities()|, passed to the language
+---       server on initialization. Hint: use make_client_capabilities() and modify
+---       its result.
+---       - Note: To send an empty dictionary use
+---         `{[vim.type_idx]=vim.types.dictionary}`, else it will be encoded as an
+---         array.
+---
+--- - handlers: Map of language server method names to |lsp-handler|
+---
+--- - settings: Map with language server specific settings. These are
+---       returned to the language server if requested via `workspace/configuration`.
+---       Keys are case-sensitive.
+---
+--- - commands: table Table that maps string of clientside commands to user-defined functions.
+---       Commands passed to start_client take precedence over the global command registry. Each key
+---       must be a unique command name, and the value is a function which is called if any LSP action
+---       (code action, code lenses, ...) triggers the command.
+---
+--- - init_options Values to pass in the initialization request
+---       as `initializationOptions`. See `initialize` in the LSP spec.
+---
+--- - name: (string, default=client-id) Name in log messages.
+---
+--- - get_language_id: function(bufnr, filetype) -> language ID as string.
+---       Defaults to the filetype.
+---
+--- - offset_encoding: (default="utf-16") One of "utf-8", "utf-16",
+---       or "utf-32" which is the encoding that the LSP server expects. Client does
+---       not verify this is correct.
+---
+--- - on_error: Callback with parameters (code, ...), invoked
+---       when the client operation throws an error. `code` is a number describing
+---       the error. Other arguments may be passed depending on the error kind.  See
+---       `vim.lsp.rpc.client_errors` for possible errors.
+---       Use `vim.lsp.rpc.client_errors[code]` to get human-friendly name.
+---
+--- - before_init: Callback with parameters (initialize_params, config)
+---       invoked before the LSP "initialize" phase, where `params` contains the
+---       parameters being sent to the server and `config` is the config that was
+---       passed to |vim.lsp.start_client()|. You can use this to modify parameters before
+---       they are sent.
+---
+--- - on_init: Callback (client, initialize_result) invoked after LSP
+---       "initialize", where `result` is a table of `capabilities` and anything else
+---       the server may send. For example, clangd sends
+---       `initialize_result.offsetEncoding` if `capabilities.offsetEncoding` was
+---       sent to it. You can only modify the `client.offset_encoding` here before
+---       any notifications are sent. Most language servers expect to be sent client specified settings after
+---       initialization. Neovim does not make this assumption. A
+---       `workspace/didChangeConfiguration` notification should be sent
+---        to the server during on_init.
+---
+--- - on_exit Callback (code, signal, client_id) invoked on client
+--- exit.
+---       - code: exit code of the process
+---       - signal: number describing the signal used to terminate (if any)
+---       - client_id: client handle
 ---
----@param handlers Map of language server method names to |lsp-handler|
+--- - on_attach: Callback (client, bufnr) invoked when client
+---       attaches to a buffer.
 ---
----@param settings Map with language server specific settings. These are
---- returned to the language server if requested via `workspace/configuration`.
---- Keys are case-sensitive.
+--- - trace: ("off" | "messages" | "verbose" | nil) passed directly to the language
+---       server in the initialize request. Invalid/empty values will default to "off"
 ---
----@param commands table Table that maps string of clientside commands to user-defined functions.
---- Commands passed to start_client take precedence over the global command registry. Each key
---- must be a unique command name, and the value is a function which is called if any LSP action
---- (code action, code lenses, ...) triggers the command.
+--- - flags: A table with flags for the client. The current (experimental) flags are:
+---       - allow_incremental_sync (bool, default true): Allow using incremental sync for buffer edits
+---       - debounce_text_changes (number, default 150): Debounce didChange
+---             notifications to the server by the given number in milliseconds. No debounce
+---             occurs if nil
+---       - exit_timeout (number|boolean, default false): Milliseconds to wait for server to
+---             exit cleanly after sending the "shutdown" request before sending kill -15.
+---             If set to false, nvim exits immediately after sending the "shutdown" request to the server.
 ---
----@param init_options Values to pass in the initialization request
---- as `initializationOptions`. See `initialize` in the LSP spec.
----
----@param name (string, default=client-id) Name in log messages.
----
----@param get_language_id function(bufnr, filetype) -> language ID as string.
---- Defaults to the filetype.
----
----@param offset_encoding (default="utf-16") One of "utf-8", "utf-16",
---- or "utf-32" which is the encoding that the LSP server expects. Client does
---- not verify this is correct.
----
----@param on_error Callback with parameters (code, ...), invoked
---- when the client operation throws an error. `code` is a number describing
---- the error. Other arguments may be passed depending on the error kind.  See
---- |vim.lsp.rpc.client_errors| for possible errors.
---- Use `vim.lsp.rpc.client_errors[code]` to get human-friendly name.
----
----@param before_init Callback with parameters (initialize_params, config)
---- invoked before the LSP "initialize" phase, where `params` contains the
---- parameters being sent to the server and `config` is the config that was
---- passed to |vim.lsp.start_client()|. You can use this to modify parameters before
---- they are sent.
----
----@param on_init Callback (client, initialize_result) invoked after LSP
---- "initialize", where `result` is a table of `capabilities` and anything else
---- the server may send. For example, clangd sends
---- `initialize_result.offsetEncoding` if `capabilities.offsetEncoding` was
---- sent to it. You can only modify the `client.offset_encoding` here before
---- any notifications are sent. Most language servers expect to be sent client specified settings after
---- initialization. Neovim does not make this assumption. A
---- `workspace/didChangeConfiguration` notification should be sent
----  to the server during on_init.
----
----@param on_exit Callback (code, signal, client_id) invoked on client
---- exit.
---- - code: exit code of the process
---- - signal: number describing the signal used to terminate (if any)
---- - client_id: client handle
----
----@param on_attach Callback (client, bufnr) invoked when client
---- attaches to a buffer.
----
----@param trace:  "off" | "messages" | "verbose" | nil passed directly to the language
---- server in the initialize request. Invalid/empty values will default to "off"
----@param flags: A table with flags for the client. The current (experimental) flags are:
---- - allow_incremental_sync (bool, default true): Allow using incremental sync for buffer edits
---- - debounce_text_changes (number, default 150): Debounce didChange
----       notifications to the server by the given number in milliseconds. No debounce
----       occurs if nil
---- - exit_timeout (number|boolean, default false): Milliseconds to wait for server to
----       exit cleanly after sending the 'shutdown' request before sending kill -15.
----       If set to false, nvim exits immediately after sending the 'shutdown' request to the server.
----
----@param root_dir string Directory where the LSP
---- server will base its workspaceFolders, rootUri, and rootPath
---- on initialization.
+--- - root_dir: (string) Directory where the LSP
+---       server will base its workspaceFolders, rootUri, and rootPath
+---       on initialization.
 ---
 ---@returns Client id. |vim.lsp.get_client_by_id()| Note: client may not be
 --- fully initialized. Use `on_init` to do any actions once
@@ -1065,7 +1078,7 @@ function lsp.start_client(config)
   ---
   ---@param code (number) Error code
   ---@param err (...) Other arguments may be passed depending on the error kind
-  ---@see |vim.lsp.rpc.client_errors| for possible errors. Use
+  ---@see `vim.lsp.rpc.client_errors` for possible errors. Use
   ---`vim.lsp.rpc.client_errors[code]` to get a human-friendly name.
   function dispatch.on_error(code, err)
     local _ = log.error()
@@ -1134,41 +1147,47 @@ function lsp.start_client(config)
 
           local namespace = vim.lsp.diagnostic.get_namespace(client_id)
           vim.diagnostic.reset(namespace, bufnr)
-        end)
 
-        client_ids[client_id] = nil
-      end
-      if vim.tbl_isempty(client_ids) then
-        vim.schedule(function()
-          unset_defaults(bufnr)
+          client_ids[client_id] = nil
+          if vim.tbl_isempty(client_ids) then
+            unset_defaults(bufnr)
+          end
         end)
       end
     end
-    local client = active_clients[client_id] and active_clients[client_id]
-      or uninitialized_clients[client_id]
-    active_clients[client_id] = nil
-    uninitialized_clients[client_id] = nil
 
-    -- Client can be absent if executable starts, but initialize fails
-    -- init/attach won't have happened
-    if client then
-      changetracking.reset(client)
-    end
-    if code ~= 0 or (signal ~= 0 and signal ~= 15) then
-      local msg =
-        string.format('Client %s quit with exit code %s and signal %s', client_id, code, signal)
-      vim.schedule(function()
+    -- Schedule the deletion of the client object so that it exists in the execution of LspDetach
+    -- autocommands
+    vim.schedule(function()
+      local client = active_clients[client_id] and active_clients[client_id]
+        or uninitialized_clients[client_id]
+      active_clients[client_id] = nil
+      uninitialized_clients[client_id] = nil
+
+      -- Client can be absent if executable starts, but initialize fails
+      -- init/attach won't have happened
+      if client then
+        changetracking.reset(client)
+      end
+      if code ~= 0 or (signal ~= 0 and signal ~= 15) then
+        local msg =
+          string.format('Client %s quit with exit code %s and signal %s', client_id, code, signal)
         vim.notify(msg, vim.log.levels.WARN)
-      end)
-    end
+      end
+    end)
   end
 
   -- Start the RPC client.
-  local rpc = lsp_rpc.start(cmd, cmd_args, dispatch, {
-    cwd = config.cmd_cwd,
-    env = config.cmd_env,
-    detached = config.detached,
-  })
+  local rpc
+  if type(cmd) == 'function' then
+    rpc = cmd(dispatch)
+  else
+    rpc = lsp_rpc.start(cmd, cmd_args, dispatch, {
+      cwd = config.cmd_cwd,
+      env = config.cmd_env,
+      detached = config.detached,
+    })
+  end
 
   -- Return nil if client fails to start
   if not rpc then
@@ -1269,8 +1288,6 @@ function lsp.start_client(config)
       client.initialized = true
       uninitialized_clients[client_id] = nil
       client.workspace_folders = workspace_folders
-      -- TODO(mjlbach): Backwards compatibility, to be removed in 0.7
-      client.workspaceFolders = client.workspace_folders
 
       -- These are the cleaned up capabilities we use for dynamically deciding
       -- when to send certain events to clients.
@@ -1348,7 +1365,7 @@ function lsp.start_client(config)
   ---
   ---@param method (string) LSP method name.
   ---@param params (table) LSP request params.
-  ---@param handler (function, optional) Response |lsp-handler| for this method.
+  ---@param handler (function|nil) Response |lsp-handler| for this method.
   ---@param bufnr (number) Buffer handle (0 for current).
   ---@returns ({status}, [request_id]): {status} is a bool indicating
   ---whether the request was successful. If it is `false`, then it will
@@ -1359,8 +1376,10 @@ function lsp.start_client(config)
   ---@see |vim.lsp.buf_request()|
   function client.request(method, params, handler, bufnr)
     if not handler then
-      handler = resolve_handler(method)
-        or error(string.format('not found: %q request handler for client %q.', method, client.name))
+      handler = assert(
+        resolve_handler(method),
+        string.format('not found: %q request handler for client %q.', method, client.name)
+      )
     end
     -- Ensure pending didChange notifications are sent so that the server doesn't operate on a stale state
     changetracking.flush(client, bufnr)
@@ -1378,7 +1397,7 @@ function lsp.start_client(config)
       nvim_exec_autocmds('User', { pattern = 'LspRequest', modeline = false })
     end)
 
-    if success then
+    if success and request_id then
       client.requests[request_id] = { type = 'pending', bufnr = bufnr, method = method }
       nvim_exec_autocmds('User', { pattern = 'LspRequest', modeline = false })
     end
@@ -1393,8 +1412,8 @@ function lsp.start_client(config)
   ---
   ---@param method (string) LSP method name.
   ---@param params (table) LSP request params.
-  ---@param timeout_ms (number, optional, default=1000) Maximum time in
-  ---milliseconds to wait for a result.
+  ---@param timeout_ms (number|nil) Maximum time in milliseconds to wait for
+  ---                               a result. Defaults to 1000
   ---@param bufnr (number) Buffer handle (0 for current).
   ---@returns { err=err, result=result }, a dictionary, where `err` and `result` come from the |lsp-handler|.
   ---On timeout, cancel or error, returns `(nil, err)` where `err` is a
@@ -1417,7 +1436,9 @@ function lsp.start_client(config)
     end, 10)
 
     if not wait_result then
-      client.cancel_request(request_id)
+      if request_id then
+        client.cancel_request(request_id)
+      end
       return nil, wait_result_reason[reason]
     end
     return request_result
@@ -1464,14 +1485,13 @@ function lsp.start_client(config)
   --- you request to stop a client which has previously been requested to
   --- shutdown, it will automatically escalate and force shutdown.
   ---
-  ---@param force (bool, optional)
+  ---@param force boolean|nil
   function client.stop(force)
-    local handle = rpc.handle
-    if handle:is_closing() then
+    if rpc.is_closing() then
       return
     end
     if force or not client.initialized or graceful_shutdown_failed then
-      handle:kill(15)
+      rpc.terminate()
       return
     end
     -- Sending a signal after a process has exited is acceptable.
@@ -1480,7 +1500,7 @@ function lsp.start_client(config)
         rpc.notify('exit')
       else
         -- If there was an error in the shutdown request, then term to be safe.
-        handle:kill(15)
+        rpc.terminate()
         graceful_shutdown_failed = true
       end
     end)
@@ -1492,7 +1512,7 @@ function lsp.start_client(config)
   ---@returns (bool) true if client is stopped or in the process of being
   ---stopped; false otherwise
   function client.is_stopped()
-    return rpc.handle:is_closing()
+    return rpc.is_closing()
   end
 
   ---@private
@@ -1513,6 +1533,16 @@ function lsp.start_client(config)
       -- TODO(ashkan) handle errors.
       pcall(config.on_attach, client, bufnr)
     end
+
+    -- schedule the initialization of semantic tokens to give the above
+    -- on_attach and LspAttach callbacks the ability to schedule wrap the
+    -- opt-out (deleting the semanticTokensProvider from capabilities)
+    vim.schedule(function()
+      if vim.tbl_get(client.server_capabilities, 'semanticTokensProvider', 'full') then
+        semantic_tokens.start(bufnr, client.id)
+      end
+    end)
+
     client.attached_buffers[bufnr] = true
   end
 
@@ -1526,15 +1556,21 @@ end
 --- Notify all attached clients that a buffer has changed.
 local text_document_did_change_handler
 do
-  text_document_did_change_handler =
-    function(_, bufnr, changedtick, firstline, lastline, new_lastline)
-      -- Detach (nvim_buf_attach) via returning True to on_lines if no clients are attached
-      if tbl_isempty(all_buffer_active_clients[bufnr] or {}) then
-        return true
-      end
-      util.buf_versions[bufnr] = changedtick
-      changetracking.send_changes(bufnr, firstline, lastline, new_lastline)
+  text_document_did_change_handler = function(
+    _,
+    bufnr,
+    changedtick,
+    firstline,
+    lastline,
+    new_lastline
+  )
+    -- Detach (nvim_buf_attach) via returning True to on_lines if no clients are attached
+    if tbl_isempty(all_buffer_active_clients[bufnr] or {}) then
+      return true
     end
+    util.buf_versions[bufnr] = changedtick
+    changetracking.send_changes(bufnr, firstline, lastline, new_lastline)
+  end
 end
 
 ---@private
@@ -1598,9 +1634,37 @@ function lsp.buf_attach_client(bufnr, client_id)
     all_buffer_active_clients[bufnr] = buffer_client_ids
 
     local uri = vim.uri_from_bufnr(bufnr)
-    local augroup = ('lsp_c_%d_b_%d_did_save'):format(client_id, bufnr)
+    local augroup = ('lsp_c_%d_b_%d_save'):format(client_id, bufnr)
+    local group = api.nvim_create_augroup(augroup, { clear = true })
+    api.nvim_create_autocmd('BufWritePre', {
+      group = group,
+      buffer = bufnr,
+      desc = 'vim.lsp: textDocument/willSave',
+      callback = function(ctx)
+        for_each_buffer_client(ctx.buf, function(client)
+          local params = {
+            textDocument = {
+              uri = uri,
+            },
+            reason = protocol.TextDocumentSaveReason.Manual,
+          }
+          if vim.tbl_get(client.server_capabilities, 'textDocumentSync', 'willSave') then
+            client.notify('textDocument/willSave', params)
+          end
+          if vim.tbl_get(client.server_capabilities, 'textDocumentSync', 'willSaveWaitUntil') then
+            local result, err =
+              client.request_sync('textDocument/willSaveWaitUntil', params, 1000, ctx.buf)
+            if result and result.result then
+              util.apply_text_edits(result.result, ctx.buf, client.offset_encoding)
+            elseif err then
+              log.error(vim.inspect(err))
+            end
+          end
+        end)
+      end,
+    })
     api.nvim_create_autocmd('BufWritePost', {
-      group = api.nvim_create_augroup(augroup, { clear = true }),
+      group = group,
       buffer = bufnr,
       desc = 'vim.lsp: textDocument/didSave handler',
       callback = function(ctx)
@@ -1627,6 +1691,7 @@ function lsp.buf_attach_client(bufnr, client_id)
           if vim.tbl_get(client.server_capabilities, 'textDocumentSync', 'openClose') then
             client.notify('textDocument/didClose', params)
           end
+          client.attached_buffers[bufnr] = nil
         end)
         util.buf_versions[bufnr] = nil
         all_buffer_active_clients[bufnr] = nil
@@ -1737,16 +1802,15 @@ end
 ---
 --- You can also use the `stop()` function on a |vim.lsp.client| object.
 --- To stop all clients:
----
---- <pre>
+--- <pre>lua
 --- vim.lsp.stop_client(vim.lsp.get_active_clients())
 --- </pre>
 ---
 --- By default asks the server to shutdown, unless stop was requested
 --- already for this client, then force-shutdown is attempted.
 ---
----@param client_id client id or |vim.lsp.client| object, or list thereof
----@param force boolean (optional) shutdown forcefully
+---@param client_id number|table id or |vim.lsp.client| object, or list thereof
+---@param force boolean|nil shutdown forcefully
 function lsp.stop_client(client_id, force)
   local ids = type(client_id) == 'table' and client_id or { client_id }
   for _, id in ipairs(ids) do
@@ -1760,10 +1824,16 @@ function lsp.stop_client(client_id, force)
   end
 end
 
+---@class vim.lsp.get_active_clients.filter
+---@field id number|nil Match clients by id
+---@field bufnr number|nil match clients attached to the given buffer
+---@field name string|nil match clients by name
+
 --- Get active clients.
 ---
----@param filter (table|nil) A table with key-value pairs used to filter the
----              returned clients. The available keys are:
+---@param filter vim.lsp.get_active_clients.filter|nil (table|nil) A table with
+---              key-value pairs used to filter the returned clients.
+---              The available keys are:
 ---               - id (number): Only return clients with the given id
 ---               - bufnr (number): Only return clients attached to this buffer
 ---               - name (string): Only return clients with the given name
@@ -1780,7 +1850,8 @@ function lsp.get_active_clients(filter)
   for client_id in pairs(t) do
     local client = active_clients[client_id]
     if
-      (filter.id == nil or client.id == filter.id)
+      client
+      and (filter.id == nil or client.id == filter.id)
       and (filter.name == nil or client.name == filter.name)
     then
       clients[#clients + 1] = client
@@ -1911,7 +1982,7 @@ end
 ---
 ---@param bufnr (number) Buffer handle, or 0 for current.
 ---@param method (string) LSP method name
----@param params (optional, table) Parameters to send to the server
+---@param params (table|nil) Parameters to send to the server
 ---@param callback (function) The callback to call when all requests are finished.
 --  Unlike `buf_request`, this will collect all the responses from each server instead of handling them.
 --  A map of client_id:request_result will be provided to the callback
@@ -1953,9 +2024,9 @@ end
 ---
 ---@param bufnr (number) Buffer handle, or 0 for current.
 ---@param method (string) LSP method name
----@param params (optional, table) Parameters to send to the server
----@param timeout_ms (optional, number, default=1000) Maximum time in
----      milliseconds to wait for a result.
+---@param params (table|nil) Parameters to send to the server
+---@param timeout_ms (number|nil) Maximum time in milliseconds to wait for a
+---                               result. Defaults to 1000
 ---
 ---@returns Map of client_id:request_result. On timeout, cancel or error,
 ---        returns `(nil, err)` where `err` is a string describing the failure
@@ -1980,9 +2051,9 @@ function lsp.buf_request_sync(bufnr, method, params, timeout_ms)
 end
 
 --- Send a notification to a server
----@param bufnr [number] (optional): The number of the buffer
----@param method [string]: Name of the request method
----@param params [string]: Arguments to send to the server
+---@param bufnr (number|nil) The number of the buffer
+---@param method (string) Name of the request method
+---@param params (any) Arguments to send to the server
 ---
 ---@returns true if any client returns true; false otherwise
 function lsp.buf_notify(bufnr, method, params)
@@ -2023,8 +2094,8 @@ end
 ---@see |complete-items|
 ---@see |CompleteDone|
 ---
----@param findstart 0 or 1, decides behavior
----@param base If findstart=0, text to match against
+---@param findstart number 0 or 1, decides behavior
+---@param base number findstart=0, text to match against
 ---
 ---@returns (number) Decided by {findstart}:
 --- - findstart=0: column where the completion starts, or -2 or -3
@@ -2153,8 +2224,8 @@ end
 --- Otherwise, uses "workspace/symbol". If no results are returned from
 --- any LSP servers, falls back to using built-in tags.
 ---
----@param pattern Pattern used to find a workspace symbol
----@param flags See |tag-function|
+---@param pattern string Pattern used to find a workspace symbol
+---@param flags string See |tag-function|
 ---
 ---@returns A list of matching tags
 function lsp.tagfunc(...)
@@ -2163,7 +2234,7 @@ end
 
 ---Checks whether a client is stopped.
 ---
----@param client_id (Number)
+---@param client_id (number)
 ---@returns true if client is stopped, false otherwise.
 function lsp.client_is_stopped(client_id)
   return active_clients[client_id] == nil
@@ -2172,7 +2243,7 @@ end
 --- Gets a map of client_id:client pairs for the given buffer, where each value
 --- is a |vim.lsp.client| object.
 ---
----@param bufnr (optional, number): Buffer handle, or 0 for current
+---@param bufnr (number|nil): Buffer handle, or 0 for current
 ---@returns (table) Table of (client_id, client) pairs
 ---@deprecated Use |vim.lsp.get_active_clients()| instead.
 function lsp.buf_get_clients(bufnr)
@@ -2201,7 +2272,7 @@ lsp.log_levels = log.levels
 ---
 ---@see |vim.lsp.log_levels|
 ---
----@param level [number|string] the case insensitive level name or number
+---@param level (number|string) the case insensitive level name or number
 function lsp.set_log_level(level)
   if type(level) == 'string' or type(level) == 'number' then
     log.set_level(level)
@@ -2222,7 +2293,7 @@ end
 ---@param fn function Function to run on each client attached to buffer
 ---                   {bufnr}. The function takes the client, client ID, and
 ---                   buffer number as arguments. Example:
----             <pre>
+---             <pre>lua
 ---               vim.lsp.for_each_buffer_client(0, function(client, client_id, bufnr)
 ---                 print(vim.inspect(client))
 ---               end)
diff --git a/runtime/lua/vim/lsp/buf.lua b/runtime/lua/vim/lsp/buf.lua
index 6a070928d9..6ac885c78f 100644
--- a/runtime/lua/vim/lsp/buf.lua
+++ b/runtime/lua/vim/lsp/buf.lua
@@ -1,4 +1,3 @@
-local vim = vim
 local api = vim.api
 local validate = vim.validate
 local util = require('vim.lsp.util')
@@ -111,7 +110,7 @@ end
 --- about the context in which a completion was triggered (how it was triggered,
 --- and by which trigger character, if applicable)
 ---
----@see |vim.lsp.protocol.constants.CompletionTriggerKind|
+---@see vim.lsp.protocol.constants.CompletionTriggerKind
 function M.completion(context)
   local params = util.make_position_params()
   params.context = context
@@ -119,35 +118,30 @@ function M.completion(context)
 end
 
 ---@private
---- If there is more than one client that supports the given method,
---- asks the user to select one.
---
----@returns The client that the user selected or nil
-local function select_client(method, on_choice)
-  validate({
-    on_choice = { on_choice, 'function', false },
-  })
-  local clients = vim.tbl_values(vim.lsp.buf_get_clients())
-  clients = vim.tbl_filter(function(client)
-    return client.supports_method(method)
-  end, clients)
-  -- better UX when choices are always in the same order (between restarts)
-  table.sort(clients, function(a, b)
-    return a.name < b.name
-  end)
-
-  if #clients > 1 then
-    vim.ui.select(clients, {
-      prompt = 'Select a language server:',
-      format_item = function(client)
-        return client.name
-      end,
-    }, on_choice)
-  elseif #clients < 1 then
-    on_choice(nil)
-  else
-    on_choice(clients[1])
-  end
+---@return table {start={row, col}, end={row, col}} using (1, 0) indexing
+local function range_from_selection()
+  -- TODO: Use `vim.region()` instead https://github.com/neovim/neovim/pull/13896
+
+  -- [bufnum, lnum, col, off]; both row and column 1-indexed
+  local start = vim.fn.getpos('v')
+  local end_ = vim.fn.getpos('.')
+  local start_row = start[2]
+  local start_col = start[3]
+  local end_row = end_[2]
+  local end_col = end_[3]
+
+  -- A user can start visual selection at the end and move backwards
+  -- Normalize the range to start < end
+  if start_row == end_row and end_col < start_col then
+    end_col, start_col = start_col, end_col
+  elseif end_row < start_row then
+    start_row, end_row = end_row, start_row
+    start_col, end_col = end_col, start_col
+  end
+  return {
+    ['start'] = { start_row, start_col - 1 },
+    ['end'] = { end_row, end_col - 1 },
+  }
 end
 
 --- Formats a buffer using the attached (and optionally filtered) language
@@ -168,11 +162,11 @@ end
 ---         Predicate used to filter clients. Receives a client as argument and must return a
 ---         boolean. Clients matching the predicate are included. Example:
 ---
----         <pre>
----         -- Never request typescript-language-server for formatting
----         vim.lsp.buf.format {
----           filter = function(client) return client.name ~= "tsserver" end
----         }
+---         <pre>lua
+---           -- Never request typescript-language-server for formatting
+---           vim.lsp.buf.format {
+---             filter = function(client) return client.name ~= "tsserver" end
+---           }
 ---         </pre>
 ---
 ---     - async boolean|nil
@@ -184,7 +178,12 @@ end
 ---         Restrict formatting to the client with ID (client.id) matching this field.
 ---     - name (string|nil):
 ---         Restrict formatting to the client with name (client.name) matching this field.
-
+---
+---     - range (table|nil) Range to format.
+---         Table must contain `start` and `end` keys with {row, col} tuples using
+---         (1,0) indexing.
+---         Defaults to current selection in visual mode
+---         Defaults to `nil` in other modes, formatting the full buffer
 function M.format(options)
   options = options or {}
   local bufnr = options.bufnr or api.nvim_get_current_buf()
@@ -198,24 +197,40 @@ function M.format(options)
     clients = vim.tbl_filter(options.filter, clients)
   end
 
+  local mode = api.nvim_get_mode().mode
+  local range = options.range
+  if not range and mode == 'v' or mode == 'V' then
+    range = range_from_selection()
+  end
+  local method = range and 'textDocument/rangeFormatting' or 'textDocument/formatting'
+
   clients = vim.tbl_filter(function(client)
-    return client.supports_method('textDocument/formatting')
+    return client.supports_method(method)
   end, clients)
 
   if #clients == 0 then
     vim.notify('[LSP] Format request failed, no matching language servers.')
   end
 
+  ---@private
+  local function set_range(client, params)
+    if range then
+      local range_params =
+        util.make_given_range_params(range.start, range['end'], bufnr, client.offset_encoding)
+      params.range = range_params.range
+    end
+    return params
+  end
+
   if options.async then
     local do_format
     do_format = function(idx, client)
       if not client then
         return
       end
-      local params = util.make_formatting_params(options.formatting_options)
-      client.request('textDocument/formatting', params, function(...)
-        local handler = client.handlers['textDocument/formatting']
-          or vim.lsp.handlers['textDocument/formatting']
+      local params = set_range(client, util.make_formatting_params(options.formatting_options))
+      client.request(method, params, function(...)
+        local handler = client.handlers[method] or vim.lsp.handlers[method]
         handler(...)
         do_format(next(clients, idx))
       end, bufnr)
@@ -224,8 +239,8 @@ function M.format(options)
   else
     local timeout_ms = options.timeout_ms or 1000
     for _, client in pairs(clients) do
-      local params = util.make_formatting_params(options.formatting_options)
-      local result, err = client.request_sync('textDocument/formatting', params, timeout_ms, bufnr)
+      local params = set_range(client, util.make_formatting_params(options.formatting_options))
+      local result, err = client.request_sync(method, params, timeout_ms, bufnr)
       if result and result.result then
         util.apply_text_edits(result.result, bufnr, client.offset_encoding)
       elseif err then
@@ -235,138 +250,6 @@ function M.format(options)
   end
 end
 
---- Formats the current buffer.
----
----@param options (table|nil) Can be used to specify FormattingOptions.
---- Some unspecified options will be automatically derived from the current
---- Neovim options.
---
----@see https://microsoft.github.io/language-server-protocol/specification#textDocument_formatting
-function M.formatting(options)
-  vim.notify_once(
-    'vim.lsp.buf.formatting is deprecated. Use vim.lsp.buf.format { async = true } instead',
-    vim.log.levels.WARN
-  )
-  local params = util.make_formatting_params(options)
-  local bufnr = api.nvim_get_current_buf()
-  select_client('textDocument/formatting', function(client)
-    if client == nil then
-      return
-    end
-
-    return client.request('textDocument/formatting', params, nil, bufnr)
-  end)
-end
-
---- Performs |vim.lsp.buf.formatting()| synchronously.
----
---- Useful for running on save, to make sure buffer is formatted prior to being
---- saved. {timeout_ms} is passed on to |vim.lsp.buf_request_sync()|. Example:
----
---- <pre>
---- autocmd BufWritePre <buffer> lua vim.lsp.buf.formatting_sync()
---- </pre>
----
----@param options table|nil with valid `FormattingOptions` entries
----@param timeout_ms (number) Request timeout
----@see |vim.lsp.buf.formatting_seq_sync|
-function M.formatting_sync(options, timeout_ms)
-  vim.notify_once(
-    'vim.lsp.buf.formatting_sync is deprecated. Use vim.lsp.buf.format instead',
-    vim.log.levels.WARN
-  )
-  local params = util.make_formatting_params(options)
-  local bufnr = api.nvim_get_current_buf()
-  select_client('textDocument/formatting', function(client)
-    if client == nil then
-      return
-    end
-
-    local result, err = client.request_sync('textDocument/formatting', params, timeout_ms, bufnr)
-    if result and result.result then
-      util.apply_text_edits(result.result, bufnr, client.offset_encoding)
-    elseif err then
-      vim.notify('vim.lsp.buf.formatting_sync: ' .. err, vim.log.levels.WARN)
-    end
-  end)
-end
-
---- Formats the current buffer by sequentially requesting formatting from attached clients.
----
---- Useful when multiple clients with formatting capability are attached.
----
---- Since it's synchronous, can be used for running on save, to make sure buffer is formatted
---- prior to being saved. {timeout_ms} is passed on to the |vim.lsp.client| `request_sync` method.
---- Example:
---- <pre>
---- vim.api.nvim_command[[autocmd BufWritePre <buffer> lua vim.lsp.buf.formatting_seq_sync()]]
---- </pre>
----
----@param options (table|nil) `FormattingOptions` entries
----@param timeout_ms (number|nil) Request timeout
----@param order (table|nil) List of client names. Formatting is requested from clients
----in the following order: first all clients that are not in the `order` list, then
----the remaining clients in the order as they occur in the `order` list.
-function M.formatting_seq_sync(options, timeout_ms, order)
-  vim.notify_once(
-    'vim.lsp.buf.formatting_seq_sync is deprecated. Use vim.lsp.buf.format instead',
-    vim.log.levels.WARN
-  )
-  local clients = vim.tbl_values(vim.lsp.buf_get_clients())
-  local bufnr = api.nvim_get_current_buf()
-
-  -- sort the clients according to `order`
-  for _, client_name in pairs(order or {}) do
-    -- if the client exists, move to the end of the list
-    for i, client in pairs(clients) do
-      if client.name == client_name then
-        table.insert(clients, table.remove(clients, i))
-        break
-      end
-    end
-  end
-
-  -- loop through the clients and make synchronous formatting requests
-  for _, client in pairs(clients) do
-    if vim.tbl_get(client.server_capabilities, 'documentFormattingProvider') then
-      local params = util.make_formatting_params(options)
-      local result, err = client.request_sync(
-        'textDocument/formatting',
-        params,
-        timeout_ms,
-        api.nvim_get_current_buf()
-      )
-      if result and result.result then
-        util.apply_text_edits(result.result, bufnr, client.offset_encoding)
-      elseif err then
-        vim.notify(
-          string.format('vim.lsp.buf.formatting_seq_sync: (%s) %s', client.name, err),
-          vim.log.levels.WARN
-        )
-      end
-    end
-  end
-end
-
---- Formats a given range.
----
----@param options Table with valid `FormattingOptions` entries.
----@param start_pos ({number, number}, optional) mark-indexed position.
----Defaults to the start of the last visual selection.
----@param end_pos ({number, number}, optional) mark-indexed position.
----Defaults to the end of the last visual selection.
-function M.range_formatting(options, start_pos, end_pos)
-  local params = util.make_given_range_params(start_pos, end_pos)
-  params.options = util.make_formatting_params(options).options
-  select_client('textDocument/rangeFormatting', function(client)
-    if client == nil then
-      return
-    end
-
-    return client.request('textDocument/rangeFormatting', params)
-  end)
-end
-
 --- Renames all references to the symbol under the cursor.
 ---
 ---@param new_name string|nil If not provided, the user will be prompted for a new
@@ -500,7 +383,7 @@ end
 
 --- Lists all the references to the symbol under the cursor in the quickfix window.
 ---
----@param context (table) Context for the request
+---@param context (table|nil) Context for the request
 ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_references
 ---@param options table|nil additional options
 ---     - on_list: (function) handler for list results. See |lsp-on-list-handler|
@@ -565,14 +448,14 @@ end
 
 --- Lists all the call sites of the symbol under the cursor in the
 --- |quickfix| window. If the symbol can resolve to multiple
---- items, the user can pick one in the |inputlist|.
+--- items, the user can pick one in the |inputlist()|.
 function M.incoming_calls()
   call_hierarchy('callHierarchy/incomingCalls')
 end
 
 --- Lists all the items that are called by the symbol under the
 --- cursor in the |quickfix| window. If the symbol can resolve to
---- multiple items, the user can pick one in the |inputlist|.
+--- multiple items, the user can pick one in the |inputlist()|.
 function M.outgoing_calls()
   call_hierarchy('callHierarchy/outgoingCalls')
 end
@@ -581,7 +464,7 @@ end
 ---
 function M.list_workspace_folders()
   local workspace_folders = {}
-  for _, client in pairs(vim.lsp.buf_get_clients()) do
+  for _, client in pairs(vim.lsp.get_active_clients({ bufnr = 0 })) do
     for _, folder in pairs(client.workspace_folders or {}) do
       table.insert(workspace_folders, folder.name)
     end
@@ -604,9 +487,9 @@ function M.add_workspace_folder(workspace_folder)
   end
   local params = util.make_workspace_params(
     { { uri = vim.uri_from_fname(workspace_folder), name = workspace_folder } },
-    { {} }
+    {}
   )
-  for _, client in pairs(vim.lsp.buf_get_clients()) do
+  for _, client in pairs(vim.lsp.get_active_clients({ bufnr = 0 })) do
     local found = false
     for _, folder in pairs(client.workspace_folders or {}) do
       if folder.name == workspace_folder then
@@ -639,7 +522,7 @@ function M.remove_workspace_folder(workspace_folder)
     { {} },
     { { uri = vim.uri_from_fname(workspace_folder), name = workspace_folder } }
   )
-  for _, client in pairs(vim.lsp.buf_get_clients()) do
+  for _, client in pairs(vim.lsp.get_active_clients({ bufnr = 0 })) do
     for idx, folder in pairs(client.workspace_folders) do
       if folder.name == workspace_folder then
         vim.lsp.buf_notify(0, 'workspace/didChangeWorkspaceFolders', params)
@@ -672,18 +555,17 @@ end
 --- Send request to the server to resolve document highlights for the current
 --- text document position. This request can be triggered by a  key mapping or
 --- by events such as `CursorHold`, e.g.:
----
---- <pre>
---- autocmd CursorHold  <buffer> lua vim.lsp.buf.document_highlight()
---- autocmd CursorHoldI <buffer> lua vim.lsp.buf.document_highlight()
---- autocmd CursorMoved <buffer> lua vim.lsp.buf.clear_references()
+--- <pre>vim
+---   autocmd CursorHold  <buffer> lua vim.lsp.buf.document_highlight()
+---   autocmd CursorHoldI <buffer> lua vim.lsp.buf.document_highlight()
+---   autocmd CursorMoved <buffer> lua vim.lsp.buf.clear_references()
 --- </pre>
 ---
 --- Note: Usage of |vim.lsp.buf.document_highlight()| requires the following highlight groups
 ---       to be defined or you won't be able to see the actual highlights.
----         |LspReferenceText|
----         |LspReferenceRead|
----         |LspReferenceWrite|
+---         |hl-LspReferenceText|
+---         |hl-LspReferenceRead|
+---         |hl-LspReferenceWrite|
 function M.document_highlight()
   local params = util.make_position_params()
   request('textDocument/documentHighlight', params)
@@ -851,6 +733,7 @@ end
 ---               List of LSP `CodeActionKind`s used to filter the code actions.
 ---               Most language servers support values like `refactor`
 ---               or `quickfix`.
+---        - triggerKind (number|nil): The reason why code actions were requested.
 ---  - filter: (function|nil)
 ---           Predicate taking an `CodeAction` and returning a boolean.
 ---  - apply: (boolean|nil)
@@ -864,6 +747,7 @@ end
 ---           using mark-like indexing. See |api-indexing|
 ---
 ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_codeAction
+---@see vim.lsp.protocol.constants.CodeActionTriggerKind
 function M.code_action(options)
   validate({ options = { options, 't', true } })
   options = options or {}
@@ -873,6 +757,9 @@ function M.code_action(options)
     options = { options = options }
   end
   local context = options.context or {}
+  if not context.triggerKind then
+    context.triggerKind = vim.lsp.protocol.CodeActionTriggerKind.Invoked
+  end
   if not context.diagnostics then
     local bufnr = api.nvim_get_current_buf()
     context.diagnostics = vim.lsp.diagnostic.get_line_diagnostics(bufnr)
@@ -885,23 +772,8 @@ function M.code_action(options)
     local end_ = assert(options.range['end'], 'range must have a `end` property')
     params = util.make_given_range_params(start, end_)
   elseif mode == 'v' or mode == 'V' then
-    -- [bufnum, lnum, col, off]; both row and column 1-indexed
-    local start = vim.fn.getpos('v')
-    local end_ = vim.fn.getpos('.')
-    local start_row = start[2]
-    local start_col = start[3]
-    local end_row = end_[2]
-    local end_col = end_[3]
-
-    -- A user can start visual selection at the end and move backwards
-    -- Normalize the range to start < end
-    if start_row == end_row and end_col < start_col then
-      end_col, start_col = start_col, end_col
-    elseif end_row < start_row then
-      start_row, end_row = end_row, start_row
-      start_col, end_col = end_col, start_col
-    end
-    params = util.make_given_range_params({ start_row, start_col - 1 }, { end_row, end_col - 1 })
+    local range = range_from_selection()
+    params = util.make_given_range_params(range.start, range['end'])
   else
     params = util.make_range_params()
   end
@@ -909,34 +781,6 @@ function M.code_action(options)
   code_action_request(params, options)
 end
 
---- Performs |vim.lsp.buf.code_action()| for a given range.
----
----
----@param context table|nil `CodeActionContext` of the LSP specification:
----               - diagnostics: (table|nil)
----                             LSP `Diagnostic[]`. Inferred from the current
----                             position if not provided.
----               - only: (table|nil)
----                      List of LSP `CodeActionKind`s used to filter the code actions.
----                      Most language servers support values like `refactor`
----                      or `quickfix`.
----@param start_pos ({number, number}, optional) mark-indexed position.
----Defaults to the start of the last visual selection.
----@param end_pos ({number, number}, optional) mark-indexed position.
----Defaults to the end of the last visual selection.
-function M.range_code_action(context, start_pos, end_pos)
-  vim.deprecate('vim.lsp.buf.range_code_action', 'vim.lsp.buf.code_action', '0.9.0')
-  validate({ context = { context, 't', true } })
-  context = context or {}
-  if not context.diagnostics then
-    local bufnr = api.nvim_get_current_buf()
-    context.diagnostics = vim.lsp.diagnostic.get_line_diagnostics(bufnr)
-  end
-  local params = util.make_given_range_params(start_pos, end_pos)
-  params.context = context
-  code_action_request(params)
-end
-
 --- Executes an LSP server command.
 ---
 ---@param command_params table A valid `ExecuteCommandParams` object
diff --git a/runtime/lua/vim/lsp/codelens.lua b/runtime/lua/vim/lsp/codelens.lua
index 4fa02c8db2..17489ed84d 100644
--- a/runtime/lua/vim/lsp/codelens.lua
+++ b/runtime/lua/vim/lsp/codelens.lua
@@ -108,13 +108,36 @@ function M.run()
   end
 end
 
+---@private
+local function resolve_bufnr(bufnr)
+  return bufnr == 0 and api.nvim_get_current_buf() or bufnr
+end
+
+--- Clear the lenses
+---
+---@param client_id number|nil filter by client_id. All clients if nil
+---@param bufnr number|nil filter by buffer. All buffers if nil
+function M.clear(client_id, bufnr)
+  local buffers = bufnr and { resolve_bufnr(bufnr) } or vim.tbl_keys(lens_cache_by_buf)
+  for _, iter_bufnr in pairs(buffers) do
+    local client_ids = client_id and { client_id } or vim.tbl_keys(namespaces)
+    for _, iter_client_id in pairs(client_ids) do
+      local ns = namespaces[iter_client_id]
+      lens_cache_by_buf[iter_bufnr][iter_client_id] = {}
+      api.nvim_buf_clear_namespace(iter_bufnr, ns, 0, -1)
+    end
+  end
+end
+
 --- Display the lenses using virtual text
 ---
 ---@param lenses table of lenses to display (`CodeLens[] | null`)
 ---@param bufnr number
 ---@param client_id number
 function M.display(lenses, bufnr, client_id)
+  local ns = namespaces[client_id]
   if not lenses or not next(lenses) then
+    api.nvim_buf_clear_namespace(bufnr, ns, 0, -1)
     return
   end
   local lenses_by_lnum = {}
@@ -126,7 +149,6 @@ function M.display(lenses, bufnr, client_id)
     end
     table.insert(line_lenses, lens)
   end
-  local ns = namespaces[client_id]
   local num_lines = api.nvim_buf_line_count(bufnr)
   for i = 0, num_lines do
     local line_lenses = lenses_by_lnum[i] or {}
@@ -241,7 +263,8 @@ end
 ---
 --- It is recommended to trigger this using an autocmd or via keymap.
 ---
---- <pre>
+--- Example:
+--- <pre>vim
 ---   autocmd BufEnter,CursorHold,InsertLeave <buffer> lua vim.lsp.codelens.refresh()
 --- </pre>
 ---
diff --git a/runtime/lua/vim/lsp/diagnostic.lua b/runtime/lua/vim/lsp/diagnostic.lua
index 1f9d084e2b..5e2bf75f1b 100644
--- a/runtime/lua/vim/lsp/diagnostic.lua
+++ b/runtime/lua/vim/lsp/diagnostic.lua
@@ -150,7 +150,7 @@ end
 ---
 --- See |vim.diagnostic.config()| for configuration options. Handler-specific
 --- configuration can be set using |vim.lsp.with()|:
---- <pre>
+--- <pre>lua
 --- vim.lsp.handlers["textDocument/publishDiagnostics"] = vim.lsp.with(
 ---   vim.lsp.diagnostic.on_publish_diagnostics, {
 ---     -- Enable underline, use default values
diff --git a/runtime/lua/vim/lsp/handlers.lua b/runtime/lua/vim/lsp/handlers.lua
index 624436bc9b..5096100a60 100644
--- a/runtime/lua/vim/lsp/handlers.lua
+++ b/runtime/lua/vim/lsp/handlers.lua
@@ -1,7 +1,6 @@
 local log = require('vim.lsp.log')
 local protocol = require('vim.lsp.protocol')
 local util = require('vim.lsp.util')
-local vim = vim
 local api = vim.api
 
 local M = {}
@@ -82,22 +81,38 @@ M['window/workDoneProgress/create'] = function(_, result, ctx)
 end
 
 --see: https://microsoft.github.io/language-server-protocol/specifications/specification-current/#window_showMessageRequest
+---@param result lsp.ShowMessageRequestParams
 M['window/showMessageRequest'] = function(_, result)
-  local actions = result.actions
-  print(result.message)
-  local option_strings = { result.message, '\nRequest Actions:' }
-  for i, action in ipairs(actions) do
-    local title = action.title:gsub('\r\n', '\\r\\n')
-    title = title:gsub('\n', '\\n')
-    table.insert(option_strings, string.format('%d. %s', i, title))
-  end
-
-  -- window/showMessageRequest can return either MessageActionItem[] or null.
-  local choice = vim.fn.inputlist(option_strings)
-  if choice < 1 or choice > #actions then
-    return vim.NIL
+  local actions = result.actions or {}
+  local co, is_main = coroutine.running()
+  if co and not is_main then
+    local opts = {
+      prompt = result.message .. ': ',
+      format_item = function(action)
+        return (action.title:gsub('\r\n', '\\r\\n')):gsub('\n', '\\n')
+      end,
+    }
+    vim.ui.select(actions, opts, function(choice)
+      -- schedule to ensure resume doesn't happen _before_ yield with
+      -- default synchronous vim.ui.select
+      vim.schedule(function()
+        coroutine.resume(co, choice or vim.NIL)
+      end)
+    end)
+    return coroutine.yield()
   else
-    return actions[choice]
+    local option_strings = { result.message, '\nRequest Actions:' }
+    for i, action in ipairs(actions) do
+      local title = action.title:gsub('\r\n', '\\r\\n')
+      title = title:gsub('\n', '\\n')
+      table.insert(option_strings, string.format('%d. %s', i, title))
+    end
+    local choice = vim.fn.inputlist(option_strings)
+    if choice < 1 or choice > #actions then
+      return vim.NIL
+    else
+      return actions[choice]
+    end
   end
 end
 
@@ -116,9 +131,10 @@ end
 
 --see: https://microsoft.github.io/language-server-protocol/specifications/specification-current/#workspace_applyEdit
 M['workspace/applyEdit'] = function(_, workspace_edit, ctx)
-  if not workspace_edit then
-    return
-  end
+  assert(
+    workspace_edit,
+    'workspace/applyEdit must be called with `ApplyWorkspaceEditParams`. Server is violating the specification'
+  )
   -- TODO(ashkan) Do something more with label?
   local client_id = ctx.client_id
   local client = vim.lsp.get_client_by_id(client_id)
@@ -298,13 +314,15 @@ M['textDocument/completion'] = function(_, result, _, _)
 end
 
 --- |lsp-handler| for the method "textDocument/hover"
---- <pre>
---- vim.lsp.handlers["textDocument/hover"] = vim.lsp.with(
----   vim.lsp.handlers.hover, {
----     -- Use a sharp border with `FloatBorder` highlights
----     border = "single"
----   }
---- )
+--- <pre>lua
+---   vim.lsp.handlers["textDocument/hover"] = vim.lsp.with(
+---     vim.lsp.handlers.hover, {
+---       -- Use a sharp border with `FloatBorder` highlights
+---       border = "single",
+---       -- add the title in hover float window
+---       title = "hover"
+---     }
+---   )
 --- </pre>
 ---@param config table Configuration table.
 ---     - border:     (default=nil)
@@ -313,8 +331,14 @@ end
 function M.hover(_, result, ctx, config)
   config = config or {}
   config.focus_id = ctx.method
+  if api.nvim_get_current_buf() ~= ctx.bufnr then
+    -- Ignore result since buffer changed. This happens for slow language servers.
+    return
+  end
   if not (result and result.contents) then
-    vim.notify('No information available')
+    if config.silent ~= true then
+      vim.notify('No information available')
+    end
     return
   end
   local markdown_lines = util.convert_input_to_markdown_lines(result.contents)
@@ -378,21 +402,25 @@ M['textDocument/implementation'] = location_handler
 
 --- |lsp-handler| for the method "textDocument/signatureHelp".
 --- The active parameter is highlighted with |hl-LspSignatureActiveParameter|.
---- <pre>
---- vim.lsp.handlers["textDocument/signatureHelp"] = vim.lsp.with(
----   vim.lsp.handlers.signature_help, {
----     -- Use a sharp border with `FloatBorder` highlights
----     border = "single"
----   }
---- )
+--- <pre>lua
+---   vim.lsp.handlers["textDocument/signatureHelp"] = vim.lsp.with(
+---     vim.lsp.handlers.signature_help, {
+---       -- Use a sharp border with `FloatBorder` highlights
+---       border = "single"
+---     }
+---   )
 --- </pre>
 ---@param config table Configuration table.
 ---     - border:     (default=nil)
 ---         - Add borders to the floating window
----         - See |vim.api.nvim_open_win()|
+---         - See |nvim_open_win()|
 function M.signature_help(_, result, ctx, config)
   config = config or {}
   config.focus_id = ctx.method
+  if api.nvim_get_current_buf() ~= ctx.bufnr then
+    -- Ignore result since buffer changed. This happens for slow language servers.
+    return
+  end
   -- When use `autocmd CompleteDone <silent><buffer> lua vim.lsp.buf.signature_help()` to call signatureHelp handler
   -- If the completion item doesn't have signatures It will make noise. Change to use `print` that can use `<silent>` to ignore
   if not (result and result.signatures and result.signatures[1]) then
@@ -512,6 +540,55 @@ M['window/showMessage'] = function(_, result, ctx, _)
   return result
 end
 
+--see: https://microsoft.github.io/language-server-protocol/specifications/specification-current/#window_showDocument
+M['window/showDocument'] = function(_, result, ctx, _)
+  local uri = result.uri
+
+  if result.external then
+    -- TODO(lvimuser): ask the user for confirmation
+    local cmd
+    if vim.fn.has('win32') == 1 then
+      cmd = { 'cmd.exe', '/c', 'start', '""', uri }
+    elseif vim.fn.has('macunix') == 1 then
+      cmd = { 'open', uri }
+    else
+      cmd = { 'xdg-open', uri }
+    end
+
+    local ret = vim.fn.system(cmd)
+    if vim.v.shell_error ~= 0 then
+      return {
+        success = false,
+        error = {
+          code = protocol.ErrorCodes.UnknownErrorCode,
+          message = ret,
+        },
+      }
+    end
+
+    return { success = true }
+  end
+
+  local client_id = ctx.client_id
+  local client = vim.lsp.get_client_by_id(client_id)
+  local client_name = client and client.name or string.format('id=%d', client_id)
+  if not client then
+    err_message({ 'LSP[', client_name, '] client has shut down after sending ', ctx.method })
+    return vim.NIL
+  end
+
+  local location = {
+    uri = uri,
+    range = result.selection,
+  }
+
+  local success = util.show_document(location, client.offset_encoding, {
+    reuse_win = true,
+    focus = result.takeFocus,
+  })
+  return { success = success or false }
+end
+
 -- Add boilerplate error validation and logging for all of these.
 for k, fn in pairs(M) do
   M[k] = function(err, result, ctx, config)
diff --git a/runtime/lua/vim/lsp/health.lua b/runtime/lua/vim/lsp/health.lua
index ba730e3d6d..987707e661 100644
--- a/runtime/lua/vim/lsp/health.lua
+++ b/runtime/lua/vim/lsp/health.lua
@@ -2,8 +2,8 @@ local M = {}
 
 --- Performs a healthcheck for LSP
 function M.check()
-  local report_info = vim.fn['health#report_info']
-  local report_warn = vim.fn['health#report_warn']
+  local report_info = vim.health.report_info
+  local report_warn = vim.health.report_warn
 
   local log = require('vim.lsp.log')
   local current_log_level = log.get_level()
@@ -27,6 +27,18 @@ function M.check()
 
   local report_fn = (log_size / 1000000 > 100 and report_warn or report_info)
   report_fn(string.format('Log size: %d KB', log_size / 1000))
+
+  local clients = vim.lsp.get_active_clients()
+  vim.health.report_start('vim.lsp: Active Clients')
+  if next(clients) then
+    for _, client in pairs(clients) do
+      report_info(
+        string.format('%s (id=%s, root_dir=%s)', client.name, client.id, client.config.root_dir)
+      )
+    end
+  else
+    report_info('No active clients')
+  end
 end
 
 return M
diff --git a/runtime/lua/vim/lsp/log.lua b/runtime/lua/vim/lsp/log.lua
index 6c6ba0f206..d1a78572aa 100644
--- a/runtime/lua/vim/lsp/log.lua
+++ b/runtime/lua/vim/lsp/log.lua
@@ -20,6 +20,17 @@ local format_func = function(arg)
 end
 
 do
+  ---@private
+  local function notify(msg, level)
+    if vim.in_fast_event() then
+      vim.schedule(function()
+        vim.notify(msg, level)
+      end)
+    else
+      vim.notify(msg, level)
+    end
+  end
+
   local path_sep = vim.loop.os_uname().version:match('Windows') and '\\' or '/'
   ---@private
   local function path_join(...)
@@ -53,7 +64,7 @@ do
     logfile, openerr = io.open(logfilename, 'a+')
     if not logfile then
       local err_msg = string.format('Failed to open LSP client log file: %s', openerr)
-      vim.notify(err_msg, vim.log.levels.ERROR)
+      notify(err_msg, vim.log.levels.ERROR)
       return false
     end
 
@@ -64,7 +75,7 @@ do
         log_info.size / (1000 * 1000),
         logfilename
       )
-      vim.notify(warn_msg)
+      notify(warn_msg)
     end
 
     -- Start message for logging
@@ -130,7 +141,7 @@ end
 vim.tbl_add_reverse_lookup(log.levels)
 
 --- Sets the current log level.
----@param level (string or number) One of `vim.lsp.log.levels`
+---@param level (string|number) One of `vim.lsp.log.levels`
 function log.set_level(level)
   if type(level) == 'string' then
     current_log_level =
diff --git a/runtime/lua/vim/lsp/protocol.lua b/runtime/lua/vim/lsp/protocol.lua
index 27da60b4ae..12345b6c8c 100644
--- a/runtime/lua/vim/lsp/protocol.lua
+++ b/runtime/lua/vim/lsp/protocol.lua
@@ -20,6 +20,14 @@ function transform_schema_to_table()
 end
 --]=]
 
+---@class lsp.ShowMessageRequestParams
+---@field type lsp.MessageType
+---@field message string
+---@field actions nil|lsp.MessageActionItem[]
+
+---@class lsp.MessageActionItem
+---@field title string
+
 local constants = {
   DiagnosticSeverity = {
     -- Reports an error.
@@ -39,6 +47,7 @@ local constants = {
     Deprecated = 2,
   },
 
+  ---@enum lsp.MessageType
   MessageType = {
     -- An error message.
     Error = 1,
@@ -142,6 +151,7 @@ local constants = {
   },
 
   -- Represents reasons why a text document is saved.
+  ---@enum lsp.TextDocumentSaveReason
   TextDocumentSaveReason = {
     -- Manually triggered, e.g. by the user pressing save, by starting debugging,
     -- or by an API call.
@@ -294,6 +304,17 @@ local constants = {
     -- Base kind for an organize imports source action
     SourceOrganizeImports = 'source.organizeImports',
   },
+  -- The reason why code actions were requested.
+  ---@enum lsp.CodeActionTriggerKind
+  CodeActionTriggerKind = {
+    -- Code actions were explicitly requested by the user or by an extension.
+    Invoked = 1,
+    -- Code actions were requested automatically.
+    --
+    -- This typically happens when current selection in a file changes, but can
+    -- also be triggered when file content changes.
+    Automatic = 2,
+  },
 }
 
 for k, v in pairs(constants) do
@@ -619,14 +640,63 @@ export interface WorkspaceClientCapabilities {
 function protocol.make_client_capabilities()
   return {
     textDocument = {
-      synchronization = {
+      semanticTokens = {
         dynamicRegistration = false,
+        tokenTypes = {
+          'namespace',
+          'type',
+          'class',
+          'enum',
+          'interface',
+          'struct',
+          'typeParameter',
+          'parameter',
+          'variable',
+          'property',
+          'enumMember',
+          'event',
+          'function',
+          'method',
+          'macro',
+          'keyword',
+          'modifier',
+          'comment',
+          'string',
+          'number',
+          'regexp',
+          'operator',
+          'decorator',
+        },
+        tokenModifiers = {
+          'declaration',
+          'definition',
+          'readonly',
+          'static',
+          'deprecated',
+          'abstract',
+          'async',
+          'modification',
+          'documentation',
+          'defaultLibrary',
+        },
+        formats = { 'relative' },
+        requests = {
+          -- TODO(jdrouhard): Add support for this
+          range = false,
+          full = { delta = true },
+        },
 
-        -- TODO(ashkan) Send textDocument/willSave before saving (BufWritePre)
-        willSave = false,
+        overlappingTokenSupport = true,
+        -- TODO(jdrouhard): Add support for this
+        multilineTokenSupport = false,
+        serverCancelSupport = false,
+        augmentsSyntaxTokens = true,
+      },
+      synchronization = {
+        dynamicRegistration = false,
 
-        -- TODO(ashkan) Implement textDocument/willSaveWaitUntil
-        willSaveWaitUntil = false,
+        willSave = true,
+        willSaveWaitUntil = true,
 
         -- Send textDocument/didSave after saving (BufWritePost)
         didSave = true,
@@ -637,7 +707,7 @@ function protocol.make_client_capabilities()
         codeActionLiteralSupport = {
           codeActionKind = {
             valueSet = (function()
-              local res = vim.tbl_values(protocol.CodeActionKind)
+              local res = vim.tbl_values(constants.CodeActionKind)
               table.sort(res)
               return res
             end)(),
@@ -742,6 +812,9 @@ function protocol.make_client_capabilities()
           end)(),
         },
       },
+      callHierarchy = {
+        dynamicRegistration = false,
+      },
     },
     workspace = {
       symbol = {
@@ -765,9 +838,9 @@ function protocol.make_client_capabilities()
       workspaceEdit = {
         resourceOperations = { 'rename', 'create', 'delete' },
       },
-    },
-    callHierarchy = {
-      dynamicRegistration = false,
+      semanticTokens = {
+        refreshSupport = true,
+      },
     },
     experimental = nil,
     window = {
@@ -778,7 +851,7 @@ function protocol.make_client_capabilities()
         },
       },
       showDocument = {
-        support = false,
+        support = true,
       },
     },
   }
@@ -861,8 +934,8 @@ function protocol._resolve_capabilities_compat(server_capabilities)
       text_document_sync_properties = {
         text_document_open_close = if_nil(textDocumentSync.openClose, false),
         text_document_did_change = if_nil(textDocumentSync.change, TextDocumentSyncKind.None),
-        text_document_will_save = if_nil(textDocumentSync.willSave, false),
-        text_document_will_save_wait_until = if_nil(textDocumentSync.willSaveWaitUntil, false),
+        text_document_will_save = if_nil(textDocumentSync.willSave, true),
+        text_document_will_save_wait_until = if_nil(textDocumentSync.willSaveWaitUntil, true),
         text_document_save = if_nil(textDocumentSync.save, false),
         text_document_save_include_text = if_nil(
           type(textDocumentSync.save) == 'table' and textDocumentSync.save.includeText,
diff --git a/runtime/lua/vim/lsp/rpc.lua b/runtime/lua/vim/lsp/rpc.lua
index 0926912066..f1492601ff 100644
--- a/runtime/lua/vim/lsp/rpc.lua
+++ b/runtime/lua/vim/lsp/rpc.lua
@@ -1,4 +1,3 @@
-local vim = vim
 local uv = vim.loop
 local log = require('vim.lsp.log')
 local protocol = require('vim.lsp.protocol')
@@ -241,247 +240,162 @@ function default_dispatchers.on_error(code, err)
   local _ = log.error() and log.error('client_error:', client_errors[code], err)
 end
 
---- Starts an LSP server process and create an LSP RPC client object to
---- interact with it. Communication with the server is currently limited to stdio.
----
----@param cmd (string) Command to start the LSP server.
----@param cmd_args (table) List of additional string arguments to pass to {cmd}.
----@param dispatchers table|nil Dispatchers for LSP message types. Valid
----dispatcher names are:
---- - `"notification"`
---- - `"server_request"`
---- - `"on_error"`
---- - `"on_exit"`
----@param extra_spawn_params table|nil Additional context for the LSP
---- server process. May contain:
---- - {cwd} (string) Working directory for the LSP server process
---- - {env} (table) Additional environment variables for LSP server process
----@returns Client RPC object.
----
----@returns Methods:
---- - `notify()` |vim.lsp.rpc.notify()|
---- - `request()` |vim.lsp.rpc.request()|
----
----@returns Members:
---- - {pid} (number) The LSP server's PID.
---- - {handle} A handle for low-level interaction with the LSP server process
----   |vim.loop|.
-local function start(cmd, cmd_args, dispatchers, extra_spawn_params)
-  local _ = log.info()
-    and log.info('Starting RPC client', { cmd = cmd, args = cmd_args, extra = extra_spawn_params })
-  validate({
-    cmd = { cmd, 's' },
-    cmd_args = { cmd_args, 't' },
-    dispatchers = { dispatchers, 't', true },
-  })
-
-  if extra_spawn_params and extra_spawn_params.cwd then
-    assert(is_dir(extra_spawn_params.cwd), 'cwd must be a directory')
-  end
-  if dispatchers then
-    local user_dispatchers = dispatchers
-    dispatchers = {}
-    for dispatch_name, default_dispatch in pairs(default_dispatchers) do
-      local user_dispatcher = user_dispatchers[dispatch_name]
-      if user_dispatcher then
-        if type(user_dispatcher) ~= 'function' then
-          error(string.format('dispatcher.%s must be a function', dispatch_name))
-        end
-        -- server_request is wrapped elsewhere.
-        if
-          not (dispatch_name == 'server_request' or dispatch_name == 'on_exit') -- TODO this blocks the loop exiting for some reason.
-        then
-          user_dispatcher = schedule_wrap(user_dispatcher)
-        end
-        dispatchers[dispatch_name] = user_dispatcher
-      else
-        dispatchers[dispatch_name] = default_dispatch
-      end
+---@private
+local function create_read_loop(handle_body, on_no_chunk, on_error)
+  local parse_chunk = coroutine.wrap(request_parser_loop)
+  parse_chunk()
+  return function(err, chunk)
+    if err then
+      on_error(err)
+      return
     end
-  else
-    dispatchers = default_dispatchers
-  end
-
-  local stdin = uv.new_pipe(false)
-  local stdout = uv.new_pipe(false)
-  local stderr = uv.new_pipe(false)
-
-  local message_index = 0
-  local message_callbacks = {}
-  local notify_reply_callbacks = {}
 
-  local handle, pid
-  do
-    ---@private
-    --- Callback for |vim.loop.spawn()| Closes all streams and runs the `on_exit` dispatcher.
-    ---@param code (number) Exit code
-    ---@param signal (number) Signal that was used to terminate (if any)
-    local function onexit(code, signal)
-      stdin:close()
-      stdout:close()
-      stderr:close()
-      handle:close()
-      -- Make sure that message_callbacks/notify_reply_callbacks can be gc'd.
-      message_callbacks = nil
-      notify_reply_callbacks = nil
-      dispatchers.on_exit(code, signal)
-    end
-    local spawn_params = {
-      args = cmd_args,
-      stdio = { stdin, stdout, stderr },
-      detached = not is_win,
-    }
-    if extra_spawn_params then
-      spawn_params.cwd = extra_spawn_params.cwd
-      spawn_params.env = env_merge(extra_spawn_params.env)
-      if extra_spawn_params.detached ~= nil then
-        spawn_params.detached = extra_spawn_params.detached
+    if not chunk then
+      if on_no_chunk then
+        on_no_chunk()
       end
+      return
     end
-    handle, pid = uv.spawn(cmd, spawn_params, onexit)
-    if handle == nil then
-      stdin:close()
-      stdout:close()
-      stderr:close()
-      local msg = string.format('Spawning language server with cmd: `%s` failed', cmd)
-      if string.match(pid, 'ENOENT') then
-        msg = msg
-          .. '. The language server is either not installed, missing from PATH, or not executable.'
+
+    while true do
+      local headers, body = parse_chunk(chunk)
+      if headers then
+        handle_body(body)
+        chunk = ''
       else
-        msg = msg .. string.format(' with error message: %s', pid)
+        break
       end
-      vim.notify(msg, vim.log.levels.WARN)
-      return
     end
   end
+end
 
-  ---@private
-  --- Encodes {payload} into a JSON-RPC message and sends it to the remote
-  --- process.
-  ---
-  ---@param payload table
-  ---@returns true if the payload could be scheduled, false if the main event-loop is in the process of closing.
-  local function encode_and_send(payload)
-    local _ = log.debug() and log.debug('rpc.send', payload)
-    if handle == nil or handle:is_closing() then
-      return false
-    end
-    local encoded = vim.json.encode(payload)
-    stdin:write(format_message_with_content_length(encoded))
-    return true
-  end
+---@class RpcClient
+---@field message_index number
+---@field message_callbacks table
+---@field notify_reply_callbacks table
+---@field transport table
+---@field dispatchers table
 
-  -- FIXME: DOC: Should be placed on the RPC client object returned by
-  -- `start()`
-  --
-  --- Sends a notification to the LSP server.
-  ---@param method (string) The invoked LSP method
-  ---@param params (table): Parameters for the invoked LSP method
-  ---@returns (bool) `true` if notification could be sent, `false` if not
-  local function notify(method, params)
-    return encode_and_send({
-      jsonrpc = '2.0',
-      method = method,
-      params = params,
-    })
-  end
+---@class RpcClient
+local Client = {}
 
-  ---@private
-  --- sends an error object to the remote LSP process.
-  local function send_response(request_id, err, result)
-    return encode_and_send({
-      id = request_id,
-      jsonrpc = '2.0',
-      error = err,
-      result = result,
-    })
+---@private
+function Client:encode_and_send(payload)
+  local _ = log.debug() and log.debug('rpc.send', payload)
+  if self.transport.is_closing() then
+    return false
   end
+  local encoded = vim.json.encode(payload)
+  self.transport.write(format_message_with_content_length(encoded))
+  return true
+end
 
-  -- FIXME: DOC: Should be placed on the RPC client object returned by
-  -- `start()`
-  --
-  --- Sends a request to the LSP server and runs {callback} upon response.
-  ---
-  ---@param method (string) The invoked LSP method
-  ---@param params (table) Parameters for the invoked LSP method
-  ---@param callback (function) Callback to invoke
-  ---@param notify_reply_callback (function|nil) Callback to invoke as soon as a request is no longer pending
-  ---@returns (bool, number) `(true, message_id)` if request could be sent, `false` if not
-  local function request(method, params, callback, notify_reply_callback)
-    validate({
-      callback = { callback, 'f' },
-      notify_reply_callback = { notify_reply_callback, 'f', true },
-    })
-    message_index = message_index + 1
-    local message_id = message_index
-    local result = encode_and_send({
-      id = message_id,
-      jsonrpc = '2.0',
-      method = method,
-      params = params,
-    })
-    if result then
-      if message_callbacks then
-        message_callbacks[message_id] = schedule_wrap(callback)
-      else
-        return false
-      end
-      if notify_reply_callback and notify_reply_callbacks then
-        notify_reply_callbacks[message_id] = schedule_wrap(notify_reply_callback)
-      end
-      return result, message_id
+---@private
+--- Sends a notification to the LSP server.
+---@param method (string) The invoked LSP method
+---@param params (any): Parameters for the invoked LSP method
+---@returns (bool) `true` if notification could be sent, `false` if not
+function Client:notify(method, params)
+  return self:encode_and_send({
+    jsonrpc = '2.0',
+    method = method,
+    params = params,
+  })
+end
+
+---@private
+--- sends an error object to the remote LSP process.
+function Client:send_response(request_id, err, result)
+  return self:encode_and_send({
+    id = request_id,
+    jsonrpc = '2.0',
+    error = err,
+    result = result,
+  })
+end
+
+---@private
+--- Sends a request to the LSP server and runs {callback} upon response.
+---
+---@param method (string) The invoked LSP method
+---@param params (table|nil) Parameters for the invoked LSP method
+---@param callback (function) Callback to invoke
+---@param notify_reply_callback (function|nil) Callback to invoke as soon as a request is no longer pending
+---@returns (bool, number) `(true, message_id)` if request could be sent, `false` if not
+function Client:request(method, params, callback, notify_reply_callback)
+  validate({
+    callback = { callback, 'f' },
+    notify_reply_callback = { notify_reply_callback, 'f', true },
+  })
+  self.message_index = self.message_index + 1
+  local message_id = self.message_index
+  local result = self:encode_and_send({
+    id = message_id,
+    jsonrpc = '2.0',
+    method = method,
+    params = params,
+  })
+  local message_callbacks = self.message_callbacks
+  local notify_reply_callbacks = self.notify_reply_callbacks
+  if result then
+    if message_callbacks then
+      message_callbacks[message_id] = schedule_wrap(callback)
     else
       return false
     end
+    if notify_reply_callback and notify_reply_callbacks then
+      notify_reply_callbacks[message_id] = schedule_wrap(notify_reply_callback)
+    end
+    return result, message_id
+  else
+    return false
   end
+end
 
-  stderr:read_start(function(_, chunk)
-    if chunk then
-      local _ = log.error() and log.error('rpc', cmd, 'stderr', chunk)
-    end
-  end)
+---@private
+function Client:on_error(errkind, ...)
+  assert(client_errors[errkind])
+  -- TODO what to do if this fails?
+  pcall(self.dispatchers.on_error, errkind, ...)
+end
 
-  ---@private
-  local function on_error(errkind, ...)
-    assert(client_errors[errkind])
-    -- TODO what to do if this fails?
-    pcall(dispatchers.on_error, errkind, ...)
-  end
-  ---@private
-  local function pcall_handler(errkind, status, head, ...)
-    if not status then
-      on_error(errkind, head, ...)
-      return status, head
-    end
-    return status, head, ...
-  end
-  ---@private
-  local function try_call(errkind, fn, ...)
-    return pcall_handler(errkind, pcall(fn, ...))
+---@private
+function Client:pcall_handler(errkind, status, head, ...)
+  if not status then
+    self:on_error(errkind, head, ...)
+    return status, head
   end
+  return status, head, ...
+end
 
-  -- TODO periodically check message_callbacks for old requests past a certain
-  -- time and log them. This would require storing the timestamp. I could call
-  -- them with an error then, perhaps.
+---@private
+function Client:try_call(errkind, fn, ...)
+  return self:pcall_handler(errkind, pcall(fn, ...))
+end
 
-  ---@private
-  local function handle_body(body)
-    local ok, decoded = pcall(vim.json.decode, body, { luanil = { object = true } })
-    if not ok then
-      on_error(client_errors.INVALID_SERVER_JSON, decoded)
-      return
-    end
-    local _ = log.debug() and log.debug('rpc.receive', decoded)
+-- TODO periodically check message_callbacks for old requests past a certain
+-- time and log them. This would require storing the timestamp. I could call
+-- them with an error then, perhaps.
 
-    if type(decoded.method) == 'string' and decoded.id then
-      local err
-      -- Schedule here so that the users functions don't trigger an error and
-      -- we can still use the result.
-      schedule(function()
+---@private
+function Client:handle_body(body)
+  local ok, decoded = pcall(vim.json.decode, body, { luanil = { object = true } })
+  if not ok then
+    self:on_error(client_errors.INVALID_SERVER_JSON, decoded)
+    return
+  end
+  local _ = log.debug() and log.debug('rpc.receive', decoded)
+
+  if type(decoded.method) == 'string' and decoded.id then
+    local err
+    -- Schedule here so that the users functions don't trigger an error and
+    -- we can still use the result.
+    schedule(function()
+      coroutine.wrap(function()
         local status, result
-        status, result, err = try_call(
+        status, result, err = self:try_call(
           client_errors.SERVER_REQUEST_HANDLER_ERROR,
-          dispatchers.server_request,
+          self.dispatchers.server_request,
           decoded.method,
           decoded.params
         )
@@ -491,8 +405,7 @@ local function start(cmd, cmd_args, dispatchers, extra_spawn_params)
             { status = status, result = result, err = err }
           )
         if status then
-          if not (result or err) then
-            -- TODO this can be a problem if `null` is sent for result. needs vim.NIL
+          if result == nil and err == nil then
             error(
               string.format(
                 'method %q: either a result or an error must be sent to the server in response',
@@ -516,120 +429,328 @@ local function start(cmd, cmd_args, dispatchers, extra_spawn_params)
           err = rpc_response_error(protocol.ErrorCodes.InternalError, result)
           result = nil
         end
-        send_response(decoded.id, err, result)
-      end)
-      -- This works because we are expecting vim.NIL here
-    elseif decoded.id and (decoded.result ~= vim.NIL or decoded.error ~= vim.NIL) then
-      -- We sent a number, so we expect a number.
-      local result_id = assert(tonumber(decoded.id), 'response id must be a number')
-
-      -- Notify the user that a response was received for the request
-      local notify_reply_callback = notify_reply_callbacks and notify_reply_callbacks[result_id]
-      if notify_reply_callback then
-        validate({
-          notify_reply_callback = { notify_reply_callback, 'f' },
-        })
-        notify_reply_callback(result_id)
-        notify_reply_callbacks[result_id] = nil
-      end
+        self:send_response(decoded.id, err, result)
+      end)()
+    end)
+    -- This works because we are expecting vim.NIL here
+  elseif decoded.id and (decoded.result ~= vim.NIL or decoded.error ~= vim.NIL) then
+    -- We sent a number, so we expect a number.
+    local result_id = assert(tonumber(decoded.id), 'response id must be a number')
+
+    -- Notify the user that a response was received for the request
+    local notify_reply_callbacks = self.notify_reply_callbacks
+    local notify_reply_callback = notify_reply_callbacks and notify_reply_callbacks[result_id]
+    if notify_reply_callback then
+      validate({
+        notify_reply_callback = { notify_reply_callback, 'f' },
+      })
+      notify_reply_callback(result_id)
+      notify_reply_callbacks[result_id] = nil
+    end
 
-      -- Do not surface RequestCancelled to users, it is RPC-internal.
-      if decoded.error then
-        local mute_error = false
-        if decoded.error.code == protocol.ErrorCodes.RequestCancelled then
-          local _ = log.debug() and log.debug('Received cancellation ack', decoded)
-          mute_error = true
-        end
+    local message_callbacks = self.message_callbacks
 
-        if mute_error then
-          -- Clear any callback since this is cancelled now.
-          -- This is safe to do assuming that these conditions hold:
-          -- - The server will not send a result callback after this cancellation.
-          -- - If the server sent this cancellation ACK after sending the result, the user of this RPC
-          -- client will ignore the result themselves.
-          if result_id and message_callbacks then
-            message_callbacks[result_id] = nil
-          end
-          return
+    -- Do not surface RequestCancelled to users, it is RPC-internal.
+    if decoded.error then
+      local mute_error = false
+      if decoded.error.code == protocol.ErrorCodes.RequestCancelled then
+        local _ = log.debug() and log.debug('Received cancellation ack', decoded)
+        mute_error = true
+      end
+
+      if mute_error then
+        -- Clear any callback since this is cancelled now.
+        -- This is safe to do assuming that these conditions hold:
+        -- - The server will not send a result callback after this cancellation.
+        -- - If the server sent this cancellation ACK after sending the result, the user of this RPC
+        -- client will ignore the result themselves.
+        if result_id and message_callbacks then
+          message_callbacks[result_id] = nil
         end
+        return
       end
+    end
 
-      local callback = message_callbacks and message_callbacks[result_id]
-      if callback then
-        message_callbacks[result_id] = nil
-        validate({
-          callback = { callback, 'f' },
+    local callback = message_callbacks and message_callbacks[result_id]
+    if callback then
+      message_callbacks[result_id] = nil
+      validate({
+        callback = { callback, 'f' },
+      })
+      if decoded.error then
+        decoded.error = setmetatable(decoded.error, {
+          __tostring = format_rpc_error,
         })
-        if decoded.error then
-          decoded.error = setmetatable(decoded.error, {
-            __tostring = format_rpc_error,
-          })
-        end
-        try_call(
-          client_errors.SERVER_RESULT_CALLBACK_ERROR,
-          callback,
-          decoded.error,
-          decoded.result
-        )
-      else
-        on_error(client_errors.NO_RESULT_CALLBACK_FOUND, decoded)
-        local _ = log.error()
-          and log.error('No callback found for server response id ' .. result_id)
       end
-    elseif type(decoded.method) == 'string' then
-      -- Notification
-      try_call(
-        client_errors.NOTIFICATION_HANDLER_ERROR,
-        dispatchers.notification,
-        decoded.method,
-        decoded.params
+      self:try_call(
+        client_errors.SERVER_RESULT_CALLBACK_ERROR,
+        callback,
+        decoded.error,
+        decoded.result
       )
     else
-      -- Invalid server message
-      on_error(client_errors.INVALID_SERVER_MESSAGE, decoded)
+      self:on_error(client_errors.NO_RESULT_CALLBACK_FOUND, decoded)
+      local _ = log.error() and log.error('No callback found for server response id ' .. result_id)
     end
+  elseif type(decoded.method) == 'string' then
+    -- Notification
+    self:try_call(
+      client_errors.NOTIFICATION_HANDLER_ERROR,
+      self.dispatchers.notification,
+      decoded.method,
+      decoded.params
+    )
+  else
+    -- Invalid server message
+    self:on_error(client_errors.INVALID_SERVER_MESSAGE, decoded)
   end
+end
 
-  local request_parser = coroutine.wrap(request_parser_loop)
-  request_parser()
-  stdout:read_start(function(err, chunk)
-    if err then
-      -- TODO better handling. Can these be intermittent errors?
-      on_error(client_errors.READ_ERROR, err)
-      return
-    end
-    -- This should signal that we are done reading from the client.
-    if not chunk then
-      return
-    end
-    -- Flush anything in the parser by looping until we don't get a result
-    -- anymore.
-    while true do
-      local headers, body = request_parser(chunk)
-      -- If we successfully parsed, then handle the response.
-      if headers then
-        handle_body(body)
-        -- Set chunk to empty so that we can call request_parser to get
-        -- anything existing in the parser to flush.
-        chunk = ''
+---@private
+---@return RpcClient
+local function new_client(dispatchers, transport)
+  local state = {
+    message_index = 0,
+    message_callbacks = {},
+    notify_reply_callbacks = {},
+    transport = transport,
+    dispatchers = dispatchers,
+  }
+  return setmetatable(state, { __index = Client })
+end
+
+---@private
+---@param client RpcClient
+local function public_client(client)
+  local result = {}
+
+  ---@private
+  function result.is_closing()
+    return client.transport.is_closing()
+  end
+
+  ---@private
+  function result.terminate()
+    client.transport.terminate()
+  end
+
+  --- Sends a request to the LSP server and runs {callback} upon response.
+  ---
+  ---@param method (string) The invoked LSP method
+  ---@param params (table|nil) Parameters for the invoked LSP method
+  ---@param callback (function) Callback to invoke
+  ---@param notify_reply_callback (function|nil) Callback to invoke as soon as a request is no longer pending
+  ---@returns (bool, number) `(true, message_id)` if request could be sent, `false` if not
+  function result.request(method, params, callback, notify_reply_callback)
+    return client:request(method, params, callback, notify_reply_callback)
+  end
+
+  --- Sends a notification to the LSP server.
+  ---@param method (string) The invoked LSP method
+  ---@param params (table|nil): Parameters for the invoked LSP method
+  ---@returns (bool) `true` if notification could be sent, `false` if not
+  function result.notify(method, params)
+    return client:notify(method, params)
+  end
+
+  return result
+end
+
+---@private
+local function merge_dispatchers(dispatchers)
+  if dispatchers then
+    local user_dispatchers = dispatchers
+    dispatchers = {}
+    for dispatch_name, default_dispatch in pairs(default_dispatchers) do
+      local user_dispatcher = user_dispatchers[dispatch_name]
+      if user_dispatcher then
+        if type(user_dispatcher) ~= 'function' then
+          error(string.format('dispatcher.%s must be a function', dispatch_name))
+        end
+        -- server_request is wrapped elsewhere.
+        if
+          not (dispatch_name == 'server_request' or dispatch_name == 'on_exit') -- TODO this blocks the loop exiting for some reason.
+        then
+          user_dispatcher = schedule_wrap(user_dispatcher)
+        end
+        dispatchers[dispatch_name] = user_dispatcher
       else
-        break
+        dispatchers[dispatch_name] = default_dispatch
       end
     end
-  end)
+  else
+    dispatchers = default_dispatchers
+  end
+  return dispatchers
+end
+
+--- Create a LSP RPC client factory that connects via TCP to the given host
+--- and port
+---
+---@param host string
+---@param port number
+---@return function
+local function connect(host, port)
+  return function(dispatchers)
+    dispatchers = merge_dispatchers(dispatchers)
+    local tcp = uv.new_tcp()
+    local closing = false
+    local transport = {
+      write = function(msg)
+        tcp:write(msg)
+      end,
+      is_closing = function()
+        return closing
+      end,
+      terminate = function()
+        if not closing then
+          closing = true
+          tcp:shutdown()
+          tcp:close()
+          dispatchers.on_exit(0, 0)
+        end
+      end,
+    }
+    local client = new_client(dispatchers, transport)
+    tcp:connect(host, port, function(err)
+      if err then
+        vim.schedule(function()
+          vim.notify(
+            string.format('Could not connect to %s:%s, reason: %s', host, port, vim.inspect(err)),
+            vim.log.levels.WARN
+          )
+        end)
+        return
+      end
+      local handle_body = function(body)
+        client:handle_body(body)
+      end
+      tcp:read_start(create_read_loop(handle_body, transport.terminate, function(read_err)
+        client:on_error(client_errors.READ_ERROR, read_err)
+      end))
+    end)
 
-  return {
-    pid = pid,
-    handle = handle,
-    request = request,
-    notify = notify,
+    return public_client(client)
+  end
+end
+
+--- Starts an LSP server process and create an LSP RPC client object to
+--- interact with it. Communication with the spawned process happens via stdio. For
+--- communication via TCP, spawn a process manually and use |vim.lsp.rpc.connect()|
+---
+---@param cmd (string) Command to start the LSP server.
+---@param cmd_args (table) List of additional string arguments to pass to {cmd}.
+---@param dispatchers table|nil Dispatchers for LSP message types. Valid
+---dispatcher names are:
+--- - `"notification"`
+--- - `"server_request"`
+--- - `"on_error"`
+--- - `"on_exit"`
+---@param extra_spawn_params table|nil Additional context for the LSP
+--- server process. May contain:
+--- - {cwd} (string) Working directory for the LSP server process
+--- - {env} (table) Additional environment variables for LSP server process
+---@returns Client RPC object.
+---
+---@returns Methods:
+--- - `notify()` |vim.lsp.rpc.notify()|
+--- - `request()` |vim.lsp.rpc.request()|
+--- - `is_closing()` returns a boolean indicating if the RPC is closing.
+--- - `terminate()` terminates the RPC client.
+local function start(cmd, cmd_args, dispatchers, extra_spawn_params)
+  local _ = log.info()
+    and log.info('Starting RPC client', { cmd = cmd, args = cmd_args, extra = extra_spawn_params })
+  validate({
+    cmd = { cmd, 's' },
+    cmd_args = { cmd_args, 't' },
+    dispatchers = { dispatchers, 't', true },
+  })
+
+  if extra_spawn_params and extra_spawn_params.cwd then
+    assert(is_dir(extra_spawn_params.cwd), 'cwd must be a directory')
+  end
+
+  dispatchers = merge_dispatchers(dispatchers)
+  local stdin = uv.new_pipe(false)
+  local stdout = uv.new_pipe(false)
+  local stderr = uv.new_pipe(false)
+  local handle, pid
+
+  local client = new_client(dispatchers, {
+    write = function(msg)
+      stdin:write(msg)
+    end,
+    is_closing = function()
+      return handle == nil or handle:is_closing()
+    end,
+    terminate = function()
+      if handle then
+        handle:kill(15)
+      end
+    end,
+  })
+
+  ---@private
+  --- Callback for |vim.loop.spawn()| Closes all streams and runs the `on_exit` dispatcher.
+  ---@param code (number) Exit code
+  ---@param signal (number) Signal that was used to terminate (if any)
+  local function onexit(code, signal)
+    stdin:close()
+    stdout:close()
+    stderr:close()
+    handle:close()
+    dispatchers.on_exit(code, signal)
+  end
+  local spawn_params = {
+    args = cmd_args,
+    stdio = { stdin, stdout, stderr },
+    detached = not is_win,
   }
+  if extra_spawn_params then
+    spawn_params.cwd = extra_spawn_params.cwd
+    spawn_params.env = env_merge(extra_spawn_params.env)
+    if extra_spawn_params.detached ~= nil then
+      spawn_params.detached = extra_spawn_params.detached
+    end
+  end
+  handle, pid = uv.spawn(cmd, spawn_params, onexit)
+  if handle == nil then
+    stdin:close()
+    stdout:close()
+    stderr:close()
+    local msg = string.format('Spawning language server with cmd: `%s` failed', cmd)
+    if string.match(pid, 'ENOENT') then
+      msg = msg
+        .. '. The language server is either not installed, missing from PATH, or not executable.'
+    else
+      msg = msg .. string.format(' with error message: %s', pid)
+    end
+    vim.notify(msg, vim.log.levels.WARN)
+    return
+  end
+
+  stderr:read_start(function(_, chunk)
+    if chunk then
+      local _ = log.error() and log.error('rpc', cmd, 'stderr', chunk)
+    end
+  end)
+
+  local handle_body = function(body)
+    client:handle_body(body)
+  end
+  stdout:read_start(create_read_loop(handle_body, nil, function(err)
+    client:on_error(client_errors.READ_ERROR, err)
+  end))
+
+  return public_client(client)
 end
 
 return {
   start = start,
+  connect = connect,
   rpc_response_error = rpc_response_error,
   format_rpc_error = format_rpc_error,
   client_errors = client_errors,
+  create_read_loop = create_read_loop,
 }
 -- vim:sw=2 ts=2 et
diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua
new file mode 100644
index 0000000000..b1bc48dac6
--- /dev/null
+++ b/runtime/lua/vim/lsp/semantic_tokens.lua
@@ -0,0 +1,702 @@
+local api = vim.api
+local handlers = require('vim.lsp.handlers')
+local util = require('vim.lsp.util')
+
+--- @class STTokenRange
+--- @field line number line number 0-based
+--- @field start_col number start column 0-based
+--- @field end_col number end column 0-based
+--- @field type string token type as string
+--- @field modifiers string[] token modifiers as strings
+--- @field extmark_added boolean whether this extmark has been added to the buffer yet
+---
+--- @class STCurrentResult
+--- @field version number document version associated with this result
+--- @field result_id string resultId from the server; used with delta requests
+--- @field highlights STTokenRange[] cache of highlight ranges for this document version
+--- @field tokens number[] raw token array as received by the server. used for calculating delta responses
+--- @field namespace_cleared boolean whether the namespace was cleared for this result yet
+---
+--- @class STActiveRequest
+--- @field request_id number the LSP request ID of the most recent request sent to the server
+--- @field version number the document version associated with the most recent request
+---
+--- @class STClientState
+--- @field namespace number
+--- @field active_request STActiveRequest
+--- @field current_result STCurrentResult
+
+---@class STHighlighter
+---@field active table<number, STHighlighter>
+---@field bufnr number
+---@field augroup number augroup for buffer events
+---@field debounce number milliseconds to debounce requests for new tokens
+---@field timer table uv_timer for debouncing requests for new tokens
+---@field client_state table<number, STClientState>
+local STHighlighter = { active = {} }
+
+---@private
+local function binary_search(tokens, line)
+  local lo = 1
+  local hi = #tokens
+  while lo < hi do
+    local mid = math.floor((lo + hi) / 2)
+    if tokens[mid].line < line then
+      lo = mid + 1
+    else
+      hi = mid
+    end
+  end
+  return lo
+end
+
+--- Extracts modifier strings from the encoded number in the token array
+---
+---@private
+---@return string[]
+local function modifiers_from_number(x, modifiers_table)
+  local modifiers = {}
+  local idx = 1
+  while x > 0 do
+    if _G.bit then
+      if _G.bit.band(x, 1) == 1 then
+        modifiers[#modifiers + 1] = modifiers_table[idx]
+      end
+      x = _G.bit.rshift(x, 1)
+    else
+      --TODO(jdrouhard): remove this branch once `bit` module is available for non-LuaJIT (#21222)
+      if x % 2 == 1 then
+        modifiers[#modifiers + 1] = modifiers_table[idx]
+      end
+      x = math.floor(x / 2)
+    end
+    idx = idx + 1
+  end
+
+  return modifiers
+end
+
+--- Converts a raw token list to a list of highlight ranges used by the on_win callback
+---
+---@private
+---@return STTokenRange[]
+local function tokens_to_ranges(data, bufnr, client)
+  local legend = client.server_capabilities.semanticTokensProvider.legend
+  local token_types = legend.tokenTypes
+  local token_modifiers = legend.tokenModifiers
+  local ranges = {}
+
+  local line
+  local start_char = 0
+  for i = 1, #data, 5 do
+    local delta_line = data[i]
+    line = line and line + delta_line or delta_line
+    local delta_start = data[i + 1]
+    start_char = delta_line == 0 and start_char + delta_start or delta_start
+
+    -- data[i+3] +1 because Lua tables are 1-indexed
+    local token_type = token_types[data[i + 3] + 1]
+    local modifiers = modifiers_from_number(data[i + 4], token_modifiers)
+
+    ---@private
+    local function _get_byte_pos(char_pos)
+      return util._get_line_byte_from_position(bufnr, {
+        line = line,
+        character = char_pos,
+      }, client.offset_encoding)
+    end
+
+    local start_col = _get_byte_pos(start_char)
+    local end_col = _get_byte_pos(start_char + data[i + 2])
+
+    if token_type then
+      ranges[#ranges + 1] = {
+        line = line,
+        start_col = start_col,
+        end_col = end_col,
+        type = token_type,
+        modifiers = modifiers,
+        extmark_added = false,
+      }
+    end
+  end
+
+  return ranges
+end
+
+--- Construct a new STHighlighter for the buffer
+---
+---@private
+---@param bufnr number
+function STHighlighter.new(bufnr)
+  local self = setmetatable({}, { __index = STHighlighter })
+
+  self.bufnr = bufnr
+  self.augroup = api.nvim_create_augroup('vim_lsp_semantic_tokens:' .. bufnr, { clear = true })
+  self.client_state = {}
+
+  STHighlighter.active[bufnr] = self
+
+  api.nvim_buf_attach(bufnr, false, {
+    on_lines = function(_, buf)
+      local highlighter = STHighlighter.active[buf]
+      if not highlighter then
+        return true
+      end
+      highlighter:on_change()
+    end,
+    on_reload = function(_, buf)
+      local highlighter = STHighlighter.active[buf]
+      if highlighter then
+        highlighter:reset()
+        highlighter:send_request()
+      end
+    end,
+    on_detach = function(_, buf)
+      local highlighter = STHighlighter.active[buf]
+      if highlighter then
+        highlighter:destroy()
+      end
+    end,
+  })
+
+  api.nvim_create_autocmd({ 'BufWinEnter', 'InsertLeave' }, {
+    buffer = self.bufnr,
+    group = self.augroup,
+    callback = function()
+      self:send_request()
+    end,
+  })
+
+  api.nvim_create_autocmd('LspDetach', {
+    buffer = self.bufnr,
+    group = self.augroup,
+    callback = function(args)
+      self:detach(args.data.client_id)
+      if vim.tbl_isempty(self.client_state) then
+        self:destroy()
+      end
+    end,
+  })
+
+  return self
+end
+
+---@private
+function STHighlighter:destroy()
+  for client_id, _ in pairs(self.client_state) do
+    self:detach(client_id)
+  end
+
+  api.nvim_del_augroup_by_id(self.augroup)
+  STHighlighter.active[self.bufnr] = nil
+end
+
+---@private
+function STHighlighter:attach(client_id)
+  local state = self.client_state[client_id]
+  if not state then
+    state = {
+      namespace = api.nvim_create_namespace('vim_lsp_semantic_tokens:' .. client_id),
+      active_request = {},
+      current_result = {},
+    }
+    self.client_state[client_id] = state
+  end
+end
+
+---@private
+function STHighlighter:detach(client_id)
+  local state = self.client_state[client_id]
+  if state then
+    --TODO: delete namespace if/when that becomes possible
+    api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1)
+    self.client_state[client_id] = nil
+  end
+end
+
+--- This is the entry point for getting all the tokens in a buffer.
+---
+--- For the given clients (or all attached, if not provided), this sends a request
+--- to ask for semantic tokens. If the server supports delta requests, that will
+--- be prioritized if we have a previous requestId and token array.
+---
+--- This function will skip servers where there is an already an active request in
+--- flight for the same version. If there is a stale request in flight, that is
+--- cancelled prior to sending a new one.
+---
+--- Finally, if the request was successful, the requestId and document version
+--- are saved to facilitate document synchronization in the response.
+---
+---@private
+function STHighlighter:send_request()
+  local version = util.buf_versions[self.bufnr]
+
+  self:reset_timer()
+
+  for client_id, state in pairs(self.client_state) do
+    local client = vim.lsp.get_client_by_id(client_id)
+
+    local current_result = state.current_result
+    local active_request = state.active_request
+
+    -- Only send a request for this client if the current result is out of date and
+    -- there isn't a current a request in flight for this version
+    if client and current_result.version ~= version and active_request.version ~= version then
+      -- cancel stale in-flight request
+      if active_request.request_id then
+        client.cancel_request(active_request.request_id)
+        active_request = {}
+        state.active_request = active_request
+      end
+
+      local spec = client.server_capabilities.semanticTokensProvider.full
+      local hasEditProvider = type(spec) == 'table' and spec.delta
+
+      local params = { textDocument = util.make_text_document_params(self.bufnr) }
+      local method = 'textDocument/semanticTokens/full'
+
+      if hasEditProvider and current_result.result_id then
+        method = method .. '/delta'
+        params.previousResultId = current_result.result_id
+      end
+      local success, request_id = client.request(method, params, function(err, response, ctx)
+        -- look client up again using ctx.client_id instead of using a captured
+        -- client object
+        local c = vim.lsp.get_client_by_id(ctx.client_id)
+        local highlighter = STHighlighter.active[ctx.bufnr]
+        if not err and c and highlighter then
+          highlighter:process_response(response, c, version)
+        end
+      end, self.bufnr)
+
+      if success then
+        active_request.request_id = request_id
+        active_request.version = version
+      end
+    end
+  end
+end
+
+--- This function will parse the semantic token responses and set up the cache
+--- (current_result). It also performs document synchronization by checking the
+--- version of the document associated with the resulting request_id and only
+--- performing work if the response is not out-of-date.
+---
+--- Delta edits are applied if necessary, and new highlight ranges are calculated
+--- and stored in the buffer state.
+---
+--- Finally, a redraw command is issued to force nvim to redraw the screen to
+--- pick up changed highlight tokens.
+---
+---@private
+function STHighlighter:process_response(response, client, version)
+  local state = self.client_state[client.id]
+  if not state then
+    return
+  end
+
+  -- ignore stale responses
+  if state.active_request.version and version ~= state.active_request.version then
+    return
+  end
+
+  -- reset active request
+  state.active_request = {}
+
+  -- skip nil responses
+  if response == nil then
+    return
+  end
+
+  -- if we have a response to a delta request, update the state of our tokens
+  -- appropriately. if it's a full response, just use that
+  local tokens
+  local token_edits = response.edits
+  if token_edits then
+    table.sort(token_edits, function(a, b)
+      return a.start < b.start
+    end)
+
+    tokens = {}
+    local old_tokens = state.current_result.tokens
+    local idx = 1
+    for _, token_edit in ipairs(token_edits) do
+      vim.list_extend(tokens, old_tokens, idx, token_edit.start)
+      if token_edit.data then
+        vim.list_extend(tokens, token_edit.data)
+      end
+      idx = token_edit.start + token_edit.deleteCount + 1
+    end
+    vim.list_extend(tokens, old_tokens, idx)
+  else
+    tokens = response.data
+  end
+
+  -- Update the state with the new results
+  local current_result = state.current_result
+  current_result.version = version
+  current_result.result_id = response.resultId
+  current_result.tokens = tokens
+  current_result.highlights = tokens_to_ranges(tokens, self.bufnr, client)
+  current_result.namespace_cleared = false
+
+  api.nvim_command('redraw!')
+end
+
+--- on_win handler for the decoration provider (see |nvim_set_decoration_provider|)
+---
+--- If there is a current result for the buffer and the version matches the
+--- current document version, then the tokens are valid and can be applied. As
+--- the buffer is drawn, this function will add extmark highlights for every
+--- token in the range of visible lines. Once a highlight has been added, it
+--- sticks around until the document changes and there's a new set of matching
+--- highlight tokens available.
+---
+--- If this is the first time a buffer is being drawn with a new set of
+--- highlights for the current document version, the namespace is cleared to
+--- remove extmarks from the last version. It's done here instead of the response
+--- handler to avoid the "blink" that occurs due to the timing between the
+--- response handler and the actual redraw.
+---
+---@private
+function STHighlighter:on_win(topline, botline)
+  for _, state in pairs(self.client_state) do
+    local current_result = state.current_result
+    if current_result.version and current_result.version == util.buf_versions[self.bufnr] then
+      if not current_result.namespace_cleared then
+        api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1)
+        current_result.namespace_cleared = true
+      end
+
+      -- We can't use ephemeral extmarks because the buffer updates are not in
+      -- sync with the list of semantic tokens. There's a delay between the
+      -- buffer changing and when the LSP server can respond with updated
+      -- tokens, and we don't want to "blink" the token highlights while
+      -- updates are in flight, and we don't want to use stale tokens because
+      -- they likely won't line up right with the actual buffer.
+      --
+      -- Instead, we have to use normal extmarks that can attach to locations
+      -- in the buffer and are persisted between redraws.
+      local highlights = current_result.highlights
+      local idx = binary_search(highlights, topline)
+
+      for i = idx, #highlights do
+        local token = highlights[i]
+
+        if token.line > botline then
+          break
+        end
+
+        if not token.extmark_added then
+          -- `strict = false` is necessary here for the 1% of cases where the
+          -- current result doesn't actually match the buffer contents. Some
+          -- LSP servers can respond with stale tokens on requests if they are
+          -- still processing changes from a didChange notification.
+          --
+          -- LSP servers that do this _should_ follow up known stale responses
+          -- with a refresh notification once they've finished processing the
+          -- didChange notification, which would re-synchronize the tokens from
+          -- our end.
+          --
+          -- The server I know of that does this is clangd when the preamble of
+          -- a file changes and the token request is processed with a stale
+          -- preamble while the new one is still being built. Once the preamble
+          -- finishes, clangd sends a refresh request which lets the client
+          -- re-synchronize the tokens.
+          api.nvim_buf_set_extmark(self.bufnr, state.namespace, token.line, token.start_col, {
+            hl_group = '@' .. token.type,
+            end_col = token.end_col,
+            priority = vim.highlight.priorities.semantic_tokens,
+            strict = false,
+          })
+
+          -- TODO(bfredl) use single extmark when hl_group supports table
+          if #token.modifiers > 0 then
+            for _, modifier in pairs(token.modifiers) do
+              api.nvim_buf_set_extmark(self.bufnr, state.namespace, token.line, token.start_col, {
+                hl_group = '@' .. modifier,
+                end_col = token.end_col,
+                priority = vim.highlight.priorities.semantic_tokens + 1,
+                strict = false,
+              })
+            end
+          end
+
+          token.extmark_added = true
+        end
+      end
+    end
+  end
+end
+
+--- Reset the buffer's highlighting state and clears the extmark highlights.
+---
+---@private
+function STHighlighter:reset()
+  for client_id, state in pairs(self.client_state) do
+    api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1)
+    state.current_result = {}
+    if state.active_request.request_id then
+      local client = vim.lsp.get_client_by_id(client_id)
+      assert(client)
+      client.cancel_request(state.active_request.request_id)
+      state.active_request = {}
+    end
+  end
+end
+
+--- Mark a client's results as dirty. This method will cancel any active
+--- requests to the server and pause new highlights from being added
+--- in the on_win callback. The rest of the current results are saved
+--- in case the server supports delta requests.
+---
+---@private
+---@param client_id number
+function STHighlighter:mark_dirty(client_id)
+  local state = self.client_state[client_id]
+  assert(state)
+
+  -- if we clear the version from current_result, it'll cause the
+  -- next request to be sent and will also pause new highlights
+  -- from being added in on_win until a new result comes from
+  -- the server
+  if state.current_result then
+    state.current_result.version = nil
+  end
+
+  if state.active_request.request_id then
+    local client = vim.lsp.get_client_by_id(client_id)
+    assert(client)
+    client.cancel_request(state.active_request.request_id)
+    state.active_request = {}
+  end
+end
+
+---@private
+function STHighlighter:on_change()
+  self:reset_timer()
+  if self.debounce > 0 then
+    self.timer = vim.defer_fn(function()
+      self:send_request()
+    end, self.debounce)
+  else
+    self:send_request()
+  end
+end
+
+---@private
+function STHighlighter:reset_timer()
+  local timer = self.timer
+  if timer then
+    self.timer = nil
+    if not timer:is_closing() then
+      timer:stop()
+      timer:close()
+    end
+  end
+end
+
+local M = {}
+
+--- Start the semantic token highlighting engine for the given buffer with the
+--- given client. The client must already be attached to the buffer.
+---
+--- NOTE: This is currently called automatically by |vim.lsp.buf_attach_client()|. To
+--- opt-out of semantic highlighting with a server that supports it, you can
+--- delete the semanticTokensProvider table from the {server_capabilities} of
+--- your client in your |LspAttach| callback or your configuration's
+--- `on_attach` callback:
+--- <pre>lua
+---   client.server_capabilities.semanticTokensProvider = nil
+--- </pre>
+---
+---@param bufnr number
+---@param client_id number
+---@param opts (nil|table) Optional keyword arguments
+---  - debounce (number, default: 200): Debounce token requests
+---        to the server by the given number in milliseconds
+function M.start(bufnr, client_id, opts)
+  vim.validate({
+    bufnr = { bufnr, 'n', false },
+    client_id = { client_id, 'n', false },
+  })
+
+  opts = opts or {}
+  assert(
+    (not opts.debounce or type(opts.debounce) == 'number'),
+    'opts.debounce must be a number with the debounce time in milliseconds'
+  )
+
+  local client = vim.lsp.get_client_by_id(client_id)
+  if not client then
+    vim.notify('[LSP] No client with id ' .. client_id, vim.log.levels.ERROR)
+    return
+  end
+
+  if not vim.lsp.buf_is_attached(bufnr, client_id) then
+    vim.notify(
+      '[LSP] Client with id ' .. client_id .. ' not attached to buffer ' .. bufnr,
+      vim.log.levels.WARN
+    )
+    return
+  end
+
+  if not vim.tbl_get(client.server_capabilities, 'semanticTokensProvider', 'full') then
+    vim.notify('[LSP] Server does not support semantic tokens', vim.log.levels.WARN)
+    return
+  end
+
+  local highlighter = STHighlighter.active[bufnr]
+
+  if not highlighter then
+    highlighter = STHighlighter.new(bufnr)
+    highlighter.debounce = opts.debounce or 200
+  else
+    highlighter.debounce = math.max(highlighter.debounce, opts.debounce or 200)
+  end
+
+  highlighter:attach(client_id)
+  highlighter:send_request()
+end
+
+--- Stop the semantic token highlighting engine for the given buffer with the
+--- given client.
+---
+--- NOTE: This is automatically called by a |LspDetach| autocmd that is set up as part
+--- of `start()`, so you should only need this function to manually disengage the semantic
+--- token engine without fully detaching the LSP client from the buffer.
+---
+---@param bufnr number
+---@param client_id number
+function M.stop(bufnr, client_id)
+  vim.validate({
+    bufnr = { bufnr, 'n', false },
+    client_id = { client_id, 'n', false },
+  })
+
+  local highlighter = STHighlighter.active[bufnr]
+  if not highlighter then
+    return
+  end
+
+  highlighter:detach(client_id)
+
+  if vim.tbl_isempty(highlighter.client_state) then
+    highlighter:destroy()
+  end
+end
+
+--- Return the semantic token(s) at the given position.
+--- If called without arguments, returns the token under the cursor.
+---
+---@param bufnr number|nil Buffer number (0 for current buffer, default)
+---@param row number|nil Position row (default cursor position)
+---@param col number|nil Position column (default cursor position)
+---
+---@return table|nil (table|nil) List of tokens at position
+function M.get_at_pos(bufnr, row, col)
+  if bufnr == nil or bufnr == 0 then
+    bufnr = api.nvim_get_current_buf()
+  end
+
+  local highlighter = STHighlighter.active[bufnr]
+  if not highlighter then
+    return
+  end
+
+  if row == nil or col == nil then
+    local cursor = api.nvim_win_get_cursor(0)
+    row, col = cursor[1] - 1, cursor[2]
+  end
+
+  local tokens = {}
+  for client_id, client in pairs(highlighter.client_state) do
+    local highlights = client.current_result.highlights
+    if highlights then
+      local idx = binary_search(highlights, row)
+      for i = idx, #highlights do
+        local token = highlights[i]
+
+        if token.line > row then
+          break
+        end
+
+        if token.start_col <= col and token.end_col > col then
+          token.client_id = client_id
+          tokens[#tokens + 1] = token
+        end
+      end
+    end
+  end
+  return tokens
+end
+
+--- Force a refresh of all semantic tokens
+---
+--- Only has an effect if the buffer is currently active for semantic token
+--- highlighting (|vim.lsp.semantic_tokens.start()| has been called for it)
+---
+---@param bufnr (nil|number) default: current buffer
+function M.force_refresh(bufnr)
+  vim.validate({
+    bufnr = { bufnr, 'n', true },
+  })
+
+  if bufnr == nil or bufnr == 0 then
+    bufnr = api.nvim_get_current_buf()
+  end
+
+  local highlighter = STHighlighter.active[bufnr]
+  if not highlighter then
+    return
+  end
+
+  highlighter:reset()
+  highlighter:send_request()
+end
+
+--- |lsp-handler| for the method `workspace/semanticTokens/refresh`
+---
+--- Refresh requests are sent by the server to indicate a project-wide change
+--- that requires all tokens to be re-requested by the client. This handler will
+--- invalidate the current results of all buffers and automatically kick off a
+--- new request for buffers that are displayed in a window. For those that aren't, a
+--- the BufWinEnter event should take care of it next time it's displayed.
+---
+---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#semanticTokens_refreshRequest
+handlers['workspace/semanticTokens/refresh'] = function(err, _, ctx)
+  if err then
+    return vim.NIL
+  end
+
+  for _, bufnr in ipairs(vim.lsp.get_buffers_by_client_id(ctx.client_id)) do
+    local highlighter = STHighlighter.active[bufnr]
+    if highlighter and highlighter.client_state[ctx.client_id] then
+      highlighter:mark_dirty(ctx.client_id)
+
+      if not vim.tbl_isempty(vim.fn.win_findbuf(bufnr)) then
+        highlighter:send_request()
+      end
+    end
+  end
+
+  return vim.NIL
+end
+
+local namespace = api.nvim_create_namespace('vim_lsp_semantic_tokens')
+api.nvim_set_decoration_provider(namespace, {
+  on_win = function(_, _, bufnr, topline, botline)
+    local highlighter = STHighlighter.active[bufnr]
+    if highlighter then
+      highlighter:on_win(topline, botline)
+    end
+  end,
+})
+
+--- for testing only! there is no guarantee of API stability with this!
+---
+---@private
+M.__STHighlighter = STHighlighter
+
+return M
diff --git a/runtime/lua/vim/lsp/sync.lua b/runtime/lua/vim/lsp/sync.lua
index 0d65e86b55..826352f036 100644
--- a/runtime/lua/vim/lsp/sync.lua
+++ b/runtime/lua/vim/lsp/sync.lua
@@ -392,7 +392,7 @@ end
 ---@param lastline number line to begin search in old_lines for last difference
 ---@param new_lastline number line to begin search in new_lines for last difference
 ---@param offset_encoding string encoding requested by language server
----@returns table TextDocumentContentChangeEvent see https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#textDocumentContentChangeEvent
+---@returns table TextDocumentContentChangeEvent see https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocumentContentChangeEvent
 function M.compute_diff(
   prev_lines,
   curr_lines,
diff --git a/runtime/lua/vim/lsp/util.lua b/runtime/lua/vim/lsp/util.lua
index 283099bbcf..38051e6410 100644
--- a/runtime/lua/vim/lsp/util.lua
+++ b/runtime/lua/vim/lsp/util.lua
@@ -1,6 +1,5 @@
 local protocol = require('vim.lsp.protocol')
 local snippet = require('vim.lsp._snippet')
-local vim = vim
 local validate = vim.validate
 local api = vim.api
 local list_extend = vim.list_extend
@@ -110,6 +109,15 @@ local function split_lines(value)
   return split(value, '\n', true)
 end
 
+---@private
+local function create_window_without_focus()
+  local prev = vim.api.nvim_get_current_win()
+  vim.cmd.new()
+  local new = vim.api.nvim_get_current_win()
+  vim.api.nvim_set_current_win(prev)
+  return new
+end
+
 --- Convert byte index to `encoding` index.
 --- Convenience wrapper around vim.str_utfindex
 ---@param line string line to be indexed
@@ -459,35 +467,52 @@ function M.apply_text_edits(text_edits, bufnr, offset_encoding)
       text = split(text_edit.newText, '\n', true),
     }
 
-    -- Some LSP servers may return +1 range of the buffer content but nvim_buf_set_text can't accept it so we should fix it here.
     local max = api.nvim_buf_line_count(bufnr)
-    if max <= e.start_row or max <= e.end_row then
-      local len = #(get_line(bufnr, max - 1) or '')
-      if max <= e.start_row then
-        e.start_row = max - 1
-        e.start_col = len
-        table.insert(e.text, 1, '')
-      end
+    -- If the whole edit is after the lines in the buffer we can simply add the new text to the end
+    -- of the buffer.
+    if max <= e.start_row then
+      api.nvim_buf_set_lines(bufnr, max, max, false, e.text)
+    else
+      local last_line_len = #(get_line(bufnr, math.min(e.end_row, max - 1)) or '')
+      -- Some LSP servers may return +1 range of the buffer content but nvim_buf_set_text can't
+      -- accept it so we should fix it here.
       if max <= e.end_row then
         e.end_row = max - 1
-        e.end_col = len
+        e.end_col = last_line_len
+        has_eol_text_edit = true
+      else
+        -- If the replacement is over the end of a line (i.e. e.end_col is out of bounds and the
+        -- replacement text ends with a newline We can likely assume that the replacement is assumed
+        -- to be meant to replace the newline with another newline and we need to make sure this
+        -- doesn't add an extra empty line. E.g. when the last line to be replaced contains a '\r'
+        -- in the file some servers (clangd on windows) will include that character in the line
+        -- while nvim_buf_set_text doesn't count it as part of the line.
+        if
+          e.end_col > last_line_len
+          and #text_edit.newText > 0
+          and string.sub(text_edit.newText, -1) == '\n'
+        then
+          table.remove(e.text, #e.text)
+        end
       end
-      has_eol_text_edit = true
-    end
-    api.nvim_buf_set_text(bufnr, e.start_row, e.start_col, e.end_row, e.end_col, e.text)
-
-    -- Fix cursor position.
-    local row_count = (e.end_row - e.start_row) + 1
-    if e.end_row < cursor.row then
-      cursor.row = cursor.row + (#e.text - row_count)
-      is_cursor_fixed = true
-    elseif e.end_row == cursor.row and e.end_col <= cursor.col then
-      cursor.row = cursor.row + (#e.text - row_count)
-      cursor.col = #e.text[#e.text] + (cursor.col - e.end_col)
-      if #e.text == 1 then
-        cursor.col = cursor.col + e.start_col
+      -- Make sure we don't go out of bounds for e.end_col
+      e.end_col = math.min(last_line_len, e.end_col)
+
+      api.nvim_buf_set_text(bufnr, e.start_row, e.start_col, e.end_row, e.end_col, e.text)
+
+      -- Fix cursor position.
+      local row_count = (e.end_row - e.start_row) + 1
+      if e.end_row < cursor.row then
+        cursor.row = cursor.row + (#e.text - row_count)
+        is_cursor_fixed = true
+      elseif e.end_row == cursor.row and e.end_col <= cursor.col then
+        cursor.row = cursor.row + (#e.text - row_count)
+        cursor.col = #e.text[#e.text] + (cursor.col - e.end_col)
+        if #e.text == 1 then
+          cursor.col = cursor.col + e.start_col
+        end
+        is_cursor_fixed = true
       end
-      is_cursor_fixed = true
     end
   end
 
@@ -755,8 +780,11 @@ local function create_file(change)
   -- from spec: Overwrite wins over `ignoreIfExists`
   local fname = vim.uri_to_fname(change.uri)
   if not opts.ignoreIfExists or opts.overwrite then
+    vim.fn.mkdir(vim.fs.dirname(fname), 'p')
     local file = io.open(fname, 'w')
-    file:close()
+    if file then
+      file:close()
+    end
   end
   vim.fn.bufadd(fname)
 end
@@ -828,7 +856,7 @@ end
 --- `textDocument/signatureHelp`, and potentially others.
 ---
 ---@param input (`MarkedString` | `MarkedString[]` | `MarkupContent`)
----@param contents (table, optional, default `{}`) List of strings to extend with converted lines
+---@param contents (table|nil) List of strings to extend with converted lines. Defaults to {}.
 ---@returns {contents}, extended with lines of converted markdown.
 ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_hover
 function M.convert_input_to_markdown_lines(input, contents)
@@ -887,8 +915,8 @@ function M.convert_signature_help_to_markdown_lines(signature_help, ft, triggers
     return
   end
   --The active signature. If omitted or the value lies outside the range of
-  --`signatures` the value defaults to zero or is ignored if `signatures.length
-  --=== 0`. Whenever possible implementors should make an active decision about
+  --`signatures` the value defaults to zero or is ignored if `signatures.length == 0`.
+  --Whenever possible implementors should make an active decision about
   --the active signature and shouldn't rely on a default value.
   local contents = {}
   local active_hl
@@ -987,6 +1015,7 @@ end
 ---        - border (string or table) override `border`
 ---        - focusable (string or table) override `focusable`
 ---        - zindex (string or table) override `zindex`, defaults to 50
+---        - relative ("mouse"|"cursor") defaults to "cursor"
 ---@returns (table) Options
 function M.make_floating_popup_options(width, height, opts)
   validate({
@@ -1001,7 +1030,8 @@ function M.make_floating_popup_options(width, height, opts)
   local anchor = ''
   local row, col
 
-  local lines_above = vim.fn.winline() - 1
+  local lines_above = opts.relative == 'mouse' and vim.fn.getmousepos().line - 1
+    or vim.fn.winline() - 1
   local lines_below = vim.fn.winheight(0) - lines_above
 
   if lines_above < lines_below then
@@ -1014,7 +1044,9 @@ function M.make_floating_popup_options(width, height, opts)
     row = 0
   end
 
-  if vim.fn.wincol() + width + (opts.offset_x or 0) <= api.nvim_get_option('columns') then
+  local wincol = opts.relative == 'mouse' and vim.fn.getmousepos().column or vim.fn.wincol()
+
+  if wincol + width + (opts.offset_x or 0) <= api.nvim_get_option('columns') then
     anchor = anchor .. 'W'
     col = 0
   else
@@ -1022,64 +1054,103 @@ function M.make_floating_popup_options(width, height, opts)
     col = 1
   end
 
+  local title = (opts.border and opts.title) and opts.title or nil
+  local title_pos
+
+  if title then
+    title_pos = opts.title_pos or 'center'
+  end
+
   return {
     anchor = anchor,
     col = col + (opts.offset_x or 0),
     height = height,
     focusable = opts.focusable,
-    relative = 'cursor',
+    relative = opts.relative == 'mouse' and 'mouse' or 'cursor',
     row = row + (opts.offset_y or 0),
     style = 'minimal',
     width = width,
     border = opts.border or default_border,
     zindex = opts.zindex or 50,
+    title = title,
+    title_pos = title_pos,
   }
 end
 
---- Jumps to a location.
+--- Shows document and optionally jumps to the location.
 ---
 ---@param location table (`Location`|`LocationLink`)
----@param offset_encoding string utf-8|utf-16|utf-32 (required)
----@param reuse_win boolean Jump to existing window if buffer is already opened.
----@returns `true` if the jump succeeded
-function M.jump_to_location(location, offset_encoding, reuse_win)
+---@param offset_encoding "utf-8" | "utf-16" | "utf-32"
+---@param opts table|nil options
+---        - reuse_win (boolean) Jump to existing window if buffer is already open.
+---        - focus (boolean) Whether to focus/jump to location if possible. Defaults to true.
+---@return boolean `true` if succeeded
+function M.show_document(location, offset_encoding, opts)
   -- location may be Location or LocationLink
   local uri = location.uri or location.targetUri
   if uri == nil then
-    return
+    return false
   end
   if offset_encoding == nil then
-    vim.notify_once(
-      'jump_to_location must be called with valid offset encoding',
-      vim.log.levels.WARN
-    )
+    vim.notify_once('show_document must be called with valid offset encoding', vim.log.levels.WARN)
   end
   local bufnr = vim.uri_to_bufnr(uri)
-  -- Save position in jumplist
-  vim.cmd("normal! m'")
 
-  -- Push a new item into tagstack
-  local from = { vim.fn.bufnr('%'), vim.fn.line('.'), vim.fn.col('.'), 0 }
-  local items = { { tagname = vim.fn.expand('<cword>'), from = from } }
-  vim.fn.settagstack(vim.fn.win_getid(), { items = items }, 't')
+  opts = opts or {}
+  local focus = vim.F.if_nil(opts.focus, true)
+  if focus then
+    -- Save position in jumplist
+    vim.cmd("normal! m'")
 
-  --- Jump to new location (adjusting for UTF-16 encoding of characters)
-  local win = reuse_win and bufwinid(bufnr)
-  if win then
+    -- Push a new item into tagstack
+    local from = { vim.fn.bufnr('%'), vim.fn.line('.'), vim.fn.col('.'), 0 }
+    local items = { { tagname = vim.fn.expand('<cword>'), from = from } }
+    vim.fn.settagstack(vim.fn.win_getid(), { items = items }, 't')
+  end
+
+  local win = opts.reuse_win and bufwinid(bufnr)
+    or focus and api.nvim_get_current_win()
+    or create_window_without_focus()
+
+  api.nvim_buf_set_option(bufnr, 'buflisted', true)
+  api.nvim_win_set_buf(win, bufnr)
+  if focus then
     api.nvim_set_current_win(win)
-  else
-    api.nvim_buf_set_option(bufnr, 'buflisted', true)
-    api.nvim_set_current_buf(bufnr)
   end
+
+  -- location may be Location or LocationLink
   local range = location.range or location.targetSelectionRange
-  local row = range.start.line
-  local col = get_line_byte_from_position(bufnr, range.start, offset_encoding)
-  api.nvim_win_set_cursor(0, { row + 1, col })
-  -- Open folds under the cursor
-  vim.cmd('normal! zv')
+  if range then
+    --- Jump to new location (adjusting for encoding of characters)
+    local row = range.start.line
+    local col = get_line_byte_from_position(bufnr, range.start, offset_encoding)
+    api.nvim_win_set_cursor(win, { row + 1, col })
+    api.nvim_win_call(win, function()
+      -- Open folds under the cursor
+      vim.cmd('normal! zv')
+    end)
+  end
+
   return true
 end
 
+--- Jumps to a location.
+---
+---@param location table (`Location`|`LocationLink`)
+---@param offset_encoding "utf-8" | "utf-16" | "utf-32"
+---@param reuse_win boolean|nil Jump to existing window if buffer is already open.
+---@return boolean `true` if the jump succeeded
+function M.jump_to_location(location, offset_encoding, reuse_win)
+  if offset_encoding == nil then
+    vim.notify_once(
+      'jump_to_location must be called with valid offset encoding',
+      vim.log.levels.WARN
+    )
+  end
+
+  return M.show_document(location, offset_encoding, { reuse_win = reuse_win, focus = true })
+end
+
 --- Previews a location in a floating window
 ---
 --- behavior depends on type of location:
@@ -1194,7 +1265,7 @@ function M.stylize_markdown(bufnr, contents, opts)
   -- when ft is nil, we get the ft from the regex match
   local matchers = {
     block = { nil, '```+([a-zA-Z0-9_]*)', '```+' },
-    pre = { '', '<pre>', '</pre>' },
+    pre = { nil, '<pre>([a-z0-9]*)', '</pre>' },
     code = { '', '<code>', '</code>' },
     text = { 'text', '<text>', '</text>' },
   }
@@ -1219,8 +1290,6 @@ function M.stylize_markdown(bufnr, contents, opts)
   -- Clean up
   contents = M._trim(contents, opts)
 
-  -- Insert blank line separator after code block?
-  local add_sep = opts.separator == nil and true or opts.separator
   local stripped = {}
   local highlights = {}
   -- keep track of lnums that contain markdown
@@ -1248,7 +1317,7 @@ function M.stylize_markdown(bufnr, contents, opts)
           finish = #stripped,
         })
         -- add a separator, but not on the last line
-        if add_sep and i < #contents then
+        if opts.separator and i < #contents then
           table.insert(stripped, '---')
           markdown_lines[#stripped] = true
         end
@@ -1484,7 +1553,7 @@ end
 ---
 ---@param contents table of lines to show in window
 ---@param syntax string of syntax to set for opened buffer
----@param opts table with optional fields (additional keys are passed on to |vim.api.nvim_open_win()|)
+---@param opts table with optional fields (additional keys are passed on to |nvim_open_win()|)
 ---             - height: (number) height of floating window
 ---             - width: (number) width of floating window
 ---             - wrap: (boolean, default true) wrap long lines
@@ -1612,7 +1681,7 @@ do --[[ References ]]
   ---@param bufnr number Buffer id
   ---@param references table List of `DocumentHighlight` objects to highlight
   ---@param offset_encoding string One of "utf-8", "utf-16", "utf-32".
-  ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#documentHighlight
+  ---@see https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocumentContentChangeEvent
   function M.buf_highlight_references(bufnr, references, offset_encoding)
     validate({
       bufnr = { bufnr, 'n', true },
@@ -1799,7 +1868,7 @@ end
 --- CAUTION: Modifies the input in-place!
 ---
 ---@param lines (table) list of lines
----@returns (string) filetype or 'markdown' if it was unchanged.
+---@returns (string) filetype or "markdown" if it was unchanged.
 function M.try_trim_markdown_code_blocks(lines)
   local language_id = lines[1]:match('^```(.*)')
   if language_id then
@@ -1843,7 +1912,7 @@ end
 --- Creates a `TextDocumentPositionParams` object for the current buffer and cursor position.
 ---
 ---@param window number|nil: window handle or 0 for current, defaults to current
----@param offset_encoding string utf-8|utf-16|utf-32|nil defaults to `offset_encoding` of first client of buffer of `window`
+---@param offset_encoding string|nil utf-8|utf-16|utf-32|nil defaults to `offset_encoding` of first client of buffer of `window`
 ---@returns `TextDocumentPositionParams` object
 ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocumentPositionParams
 function M.make_position_params(window, offset_encoding)
@@ -1866,7 +1935,7 @@ function M._get_offset_encoding(bufnr)
 
   local offset_encoding
 
-  for _, client in pairs(vim.lsp.buf_get_clients(bufnr)) do
+  for _, client in pairs(vim.lsp.get_active_clients({ bufnr = bufnr })) do
     if client.offset_encoding == nil then
       vim.notify_once(
         string.format(
@@ -1972,7 +2041,7 @@ function M.make_workspace_params(added, removed)
 end
 --- Returns indentation size.
 ---
----@see |shiftwidth|
+---@see 'shiftwidth'
 ---@param bufnr (number|nil): Buffer handle, defaults to current
 ---@returns (number) indentation size
 function M.get_effective_tabstop(bufnr)
diff --git a/runtime/lua/vim/secure.lua b/runtime/lua/vim/secure.lua
new file mode 100644
index 0000000000..443b152273
--- /dev/null
+++ b/runtime/lua/vim/secure.lua
@@ -0,0 +1,188 @@
+local M = {}
+
+---@private
+--- Reads trust database from $XDG_STATE_HOME/nvim/trust.
+---
+---@return (table) Contents of trust database, if it exists. Empty table otherwise.
+local function read_trust()
+  local trust = {}
+  local f = io.open(vim.fn.stdpath('state') .. '/trust', 'r')
+  if f then
+    local contents = f:read('*a')
+    if contents then
+      for line in vim.gsplit(contents, '\n') do
+        local hash, file = string.match(line, '^(%S+) (.+)$')
+        if hash and file then
+          trust[file] = hash
+        end
+      end
+    end
+    f:close()
+  end
+  return trust
+end
+
+---@private
+--- Writes provided {trust} table to trust database at
+--- $XDG_STATE_HOME/nvim/trust.
+---
+---@param trust (table) Trust table to write
+local function write_trust(trust)
+  vim.validate({ trust = { trust, 't' } })
+  local f = assert(io.open(vim.fn.stdpath('state') .. '/trust', 'w'))
+
+  local t = {}
+  for p, h in pairs(trust) do
+    t[#t + 1] = string.format('%s %s\n', h, p)
+  end
+  f:write(table.concat(t))
+  f:close()
+end
+
+--- Attempt to read the file at {path} prompting the user if the file should be
+--- trusted. The user's choice is persisted in a trust database at
+--- $XDG_STATE_HOME/nvim/trust.
+---
+---@see |:trust|
+---
+---@param path (string) Path to a file to read.
+---
+---@return (string|nil) The contents of the given file if it exists and is
+---        trusted, or nil otherwise.
+function M.read(path)
+  vim.validate({ path = { path, 's' } })
+  local fullpath = vim.loop.fs_realpath(vim.fs.normalize(path))
+  if not fullpath then
+    return nil
+  end
+
+  local trust = read_trust()
+
+  if trust[fullpath] == '!' then
+    -- File is denied
+    return nil
+  end
+
+  local contents
+  do
+    local f = io.open(fullpath, 'r')
+    if not f then
+      return nil
+    end
+    contents = f:read('*a')
+    f:close()
+  end
+
+  local hash = vim.fn.sha256(contents)
+  if trust[fullpath] == hash then
+    -- File already exists in trust database
+    return contents
+  end
+
+  -- File either does not exist in trust database or the hash does not match
+  local ok, result = pcall(
+    vim.fn.confirm,
+    string.format('%s is not trusted.', fullpath),
+    '&ignore\n&view\n&deny\n&allow',
+    1
+  )
+
+  if not ok and result ~= 'Keyboard interrupt' then
+    error(result)
+  elseif not ok or result == 0 or result == 1 then
+    -- Cancelled or ignored
+    return nil
+  elseif result == 2 then
+    -- View
+    vim.cmd('sview ' .. fullpath)
+    return nil
+  elseif result == 3 then
+    -- Deny
+    trust[fullpath] = '!'
+    contents = nil
+  elseif result == 4 then
+    -- Allow
+    trust[fullpath] = hash
+  end
+
+  write_trust(trust)
+
+  return contents
+end
+
+--- Manage the trust database.
+---
+--- The trust database is located at |$XDG_STATE_HOME|/nvim/trust.
+---
+---@param opts (table):
+---    - action (string): "allow" to add a file to the trust database and trust it,
+---      "deny" to add a file to the trust database and deny it,
+---      "remove" to remove file from the trust database
+---    - path (string|nil): Path to a file to update. Mutually exclusive with {bufnr}.
+---      Cannot be used when {action} is "allow".
+---    - bufnr (number|nil): Buffer number to update. Mutually exclusive with {path}.
+---@return (boolean, string) success, msg:
+---    - true and full path of target file if operation was successful
+---    - false and error message on failure
+function M.trust(opts)
+  vim.validate({
+    path = { opts.path, 's', true },
+    bufnr = { opts.bufnr, 'n', true },
+    action = {
+      opts.action,
+      function(m)
+        return m == 'allow' or m == 'deny' or m == 'remove'
+      end,
+      [["allow" or "deny" or "remove"]],
+    },
+  })
+
+  local path = opts.path
+  local bufnr = opts.bufnr
+  local action = opts.action
+
+  assert(not path or not bufnr, '"path" and "bufnr" are mutually exclusive')
+
+  if action == 'allow' then
+    assert(not path, '"path" is not valid when action is "allow"')
+  end
+
+  local fullpath
+  if path then
+    fullpath = vim.loop.fs_realpath(vim.fs.normalize(path))
+  elseif bufnr then
+    local bufname = vim.api.nvim_buf_get_name(bufnr)
+    if bufname == '' then
+      return false, 'buffer is not associated with a file'
+    end
+    fullpath = vim.loop.fs_realpath(vim.fs.normalize(bufname))
+  else
+    error('one of "path" or "bufnr" is required')
+  end
+
+  if not fullpath then
+    return false, string.format('invalid path: %s', path)
+  end
+
+  local trust = read_trust()
+
+  if action == 'allow' then
+    local newline = vim.bo[bufnr].fileformat == 'unix' and '\n' or '\r\n'
+    local contents = table.concat(vim.api.nvim_buf_get_lines(bufnr, 0, -1, false), newline)
+    if vim.bo[bufnr].endofline then
+      contents = contents .. newline
+    end
+    local hash = vim.fn.sha256(contents)
+
+    trust[fullpath] = hash
+  elseif action == 'deny' then
+    trust[fullpath] = '!'
+  elseif action == 'remove' then
+    trust[fullpath] = nil
+  end
+
+  write_trust(trust)
+  return true, fullpath
+end
+
+return M
diff --git a/runtime/lua/vim/shared.lua b/runtime/lua/vim/shared.lua
index e1b4ed4ea9..cc48e3f193 100644
--- a/runtime/lua/vim/shared.lua
+++ b/runtime/lua/vim/shared.lua
@@ -6,7 +6,7 @@
 -- or the test suite. (Eventually the test suite will be run in a worker process,
 -- so this wouldn't be a separate case to consider)
 
-local vim = vim or {}
+vim = vim or {}
 
 --- Returns a deep copy of the given object. Non-table objects are copied as
 --- in a typical Lua assignment, whereas table objects are copied recursively.
@@ -14,8 +14,9 @@ local vim = vim or {}
 --- same functions as those in the input table. Userdata and threads are not
 --- copied and will throw an error.
 ---
----@param orig table Table to copy
----@return table Table of copied keys and (nested) values.
+---@generic T: table
+---@param orig T Table to copy
+---@return T Table of copied keys and (nested) values.
 function vim.deepcopy(orig) end -- luacheck: no unused
 vim.deepcopy = (function()
   local function _id(v)
@@ -48,6 +49,9 @@ vim.deepcopy = (function()
     if f then
       return f(orig, cache or {})
     else
+      if type(orig) == 'userdata' and orig == vim.NIL then
+        return vim.NIL
+      end
       error('Cannot deepcopy object of type ' .. type(orig))
     end
   end
@@ -56,13 +60,14 @@ end)()
 --- Splits a string at each instance of a separator.
 ---
 ---@see |vim.split()|
+---@see |luaref-patterns|
 ---@see https://www.lua.org/pil/20.2.html
 ---@see http://lua-users.org/wiki/StringLibraryTutorial
 ---
 ---@param s string String to split
 ---@param sep string Separator or pattern
----@param plain boolean If `true` use `sep` literally (passed to string.find)
----@return function Iterator over the split components
+---@param plain (boolean|nil) If `true` use `sep` literally (passed to string.find)
+---@return fun():string (function) Iterator over the split components
 function vim.gsplit(s, sep, plain)
   vim.validate({ s = { s, 's' }, sep = { sep, 's' }, plain = { plain, 'b', true } })
 
@@ -98,10 +103,10 @@ end
 --- Splits a string at each instance of a separator.
 ---
 --- Examples:
---- <pre>
----  split(":aa::b:", ":")     --> {'','aa','','b',''}
----  split("axaby", "ab?")     --> {'','x','y'}
----  split("x*yz*o", "*", {plain=true})  --> {'x','yz','o'}
+--- <pre>lua
+---  split(":aa::b:", ":")                   --> {'','aa','','b',''}
+---  split("axaby", "ab?")                   --> {'','x','y'}
+---  split("x*yz*o", "*", {plain=true})      --> {'x','yz','o'}
 ---  split("|x|y|z|", "|", {trimempty=true}) --> {'x', 'y', 'z'}
 --- </pre>
 ---
@@ -109,11 +114,11 @@ end
 ---
 ---@param s string String to split
 ---@param sep string Separator or pattern
----@param kwargs table Keyword arguments:
+---@param kwargs (table|nil) Keyword arguments:
 ---       - plain: (boolean) If `true` use `sep` literally (passed to string.find)
 ---       - trimempty: (boolean) If `true` remove empty items from the front
 ---         and back of the list
----@return table List of split components
+---@return string[] List of split components
 function vim.split(s, sep, kwargs)
   local plain
   local trimempty = false
@@ -156,8 +161,9 @@ end
 ---
 ---@see From https://github.com/premake/premake-core/blob/master/src/base/table.lua
 ---
----@param t table Table
----@return table List of keys
+---@generic T: table
+---@param t table<T, any> (table) Table
+---@return T[] (list) List of keys
 function vim.tbl_keys(t)
   assert(type(t) == 'table', string.format('Expected table, got %s', type(t)))
 
@@ -171,8 +177,9 @@ end
 --- Return a list of all values used in a table.
 --- However, the order of the return table of values is not guaranteed.
 ---
----@param t table Table
----@return table List of values
+---@generic T
+---@param t table<any, T> (table) Table
+---@return T[] (list) List of values
 function vim.tbl_values(t)
   assert(type(t) == 'table', string.format('Expected table, got %s', type(t)))
 
@@ -185,8 +192,9 @@ end
 
 --- Apply a function to all values of a table.
 ---
----@param func function|table Function or callable table
----@param t table Table
+---@generic T
+---@param func fun(value: T): any (function) Function
+---@param t table<any, T> (table) Table
 ---@return table Table of transformed values
 function vim.tbl_map(func, t)
   vim.validate({ func = { func, 'c' }, t = { t, 't' } })
@@ -200,9 +208,10 @@ end
 
 --- Filter a table using a predicate function
 ---
----@param func function|table Function or callable table
----@param t table Table
----@return table Table of filtered values
+---@generic T
+---@param func fun(value: T): boolean (function) Function
+---@param t table<any, T> (table) Table
+---@return T[] (table) Table of filtered values
 function vim.tbl_filter(func, t)
   vim.validate({ func = { func, 'c' }, t = { t, 't' } })
 
@@ -302,14 +311,16 @@ end
 
 --- Merges recursively two or more map-like tables.
 ---
----@see |tbl_extend()|
+---@see |vim.tbl_extend()|
 ---
----@param behavior string Decides what to do if a key is found in more than one map:
+---@generic T1: table
+---@generic T2: table
+---@param behavior "error"|"keep"|"force" (string) Decides what to do if a key is found in more than one map:
 ---      - "error": raise an error
 ---      - "keep":  use value from the leftmost map
 ---      - "force": use value from the rightmost map
----@param ... table Two or more map-like tables
----@return table Merged table
+---@param ... T2 Two or more map-like tables
+---@return T1|T2 (table) Merged table
 function vim.tbl_deep_extend(behavior, ...)
   return tbl_extend(behavior, true, ...)
 end
@@ -373,7 +384,7 @@ end
 --- Return `nil` if the key does not exist.
 ---
 --- Examples:
---- <pre>
+--- <pre>lua
 ---  vim.tbl_get({ key = { nested_key = true }}, 'key', 'nested_key') == true
 ---  vim.tbl_get({ key = {}}, 'key', 'nested_key') == nil
 --- </pre>
@@ -385,15 +396,14 @@ end
 function vim.tbl_get(o, ...)
   local keys = { ... }
   if #keys == 0 then
-    return
+    return nil
   end
   for i, k in ipairs(keys) do
-    if type(o[k]) ~= 'table' and next(keys, i) then
-      return nil
-    end
     o = o[k]
     if o == nil then
-      return
+      return nil
+    elseif type(o) ~= 'table' and next(keys, i) then
+      return nil
     end
   end
   return o
@@ -405,11 +415,12 @@ end
 ---
 ---@see |vim.tbl_extend()|
 ---
----@param dst table List which will be modified and appended to
+---@generic T: table
+---@param dst T List which will be modified and appended to
 ---@param src table List from which values will be inserted
----@param start number Start index on src. Defaults to 1
----@param finish number Final index on src. Defaults to `#src`
----@return table dst
+---@param start (number|nil) Start index on src. Defaults to 1
+---@param finish (number|nil) Final index on src. Defaults to `#src`
+---@return T dst
 function vim.list_extend(dst, src, start, finish)
   vim.validate({
     dst = { dst, 't' },
@@ -447,6 +458,33 @@ function vim.tbl_flatten(t)
   return result
 end
 
+--- Enumerate a table sorted by its keys.
+---
+---@see Based on https://github.com/premake/premake-core/blob/master/src/base/table.lua
+---
+---@param t table List-like table
+---@return iterator over sorted keys and their values
+function vim.spairs(t)
+  assert(type(t) == 'table', string.format('Expected table, got %s', type(t)))
+
+  -- collect the keys
+  local keys = {}
+  for k in pairs(t) do
+    table.insert(keys, k)
+  end
+  table.sort(keys)
+
+  -- Return the iterator function.
+  -- TODO(justinmk): Return "iterator function, table {t}, and nil", like pairs()?
+  local i = 0
+  return function()
+    i = i + 1
+    if keys[i] then
+      return keys[i], t[keys[i]]
+    end
+  end
+end
+
 --- Tests if a Lua table can be treated as an array.
 ---
 --- Empty table `{}` is assumed to be an array, unless it was created by
@@ -476,7 +514,7 @@ function vim.tbl_islist(t)
     -- TODO(bfredl): in the future, we will always be inside nvim
     -- then this check can be deleted.
     if vim._empty_dict_mt == nil then
-      return nil
+      return false
     end
     return getmetatable(t) ~= vim._empty_dict_mt
   end
@@ -484,9 +522,9 @@ end
 
 --- Counts the number of non-nil values in table `t`.
 ---
---- <pre>
---- vim.tbl_count({ a=1, b=2 }) => 2
---- vim.tbl_count({ 1, 2 }) => 2
+--- <pre>lua
+--- vim.tbl_count({ a=1, b=2 })  --> 2
+--- vim.tbl_count({ 1, 2 })      --> 2
 --- </pre>
 ---
 ---@see https://github.com/Tieske/Penlight/blob/master/lua/pl/tablex.lua
@@ -504,10 +542,11 @@ end
 
 --- Creates a copy of a table containing only elements from start to end (inclusive)
 ---
----@param list table Table
----@param start number Start range of slice
----@param finish number End range of slice
----@return table Copy of table sliced from start to finish (inclusive)
+---@generic T
+---@param list T[] (list) Table
+---@param start number|nil Start range of slice
+---@param finish number|nil End range of slice
+---@return T[] (list) Copy of table sliced from start to finish (inclusive)
 function vim.list_slice(list, start, finish)
   local new_list = {}
   for i = start or 1, finish or #list do
@@ -518,6 +557,7 @@ end
 
 --- Trim whitespace (Lua pattern "%s") from both sides of a string.
 ---
+---@see |luaref-patterns|
 ---@see https://www.lua.org/pil/20.2.html
 ---@param s string String to trim
 ---@return string String with whitespace removed from its beginning and end
@@ -533,7 +573,7 @@ end
 ---@return string %-escaped pattern string
 function vim.pesc(s)
   vim.validate({ s = { s, 's' } })
-  return s:gsub('[%(%)%.%%%+%-%*%?%[%]%^%$]', '%%%1')
+  return (s:gsub('[%(%)%.%%%+%-%*%?%[%]%^%$]', '%%%1'))
 end
 
 --- Tests if `s` starts with `prefix`.
@@ -559,7 +599,7 @@ end
 --- Validates a parameter specification (types and values).
 ---
 --- Usage example:
---- <pre>
+--- <pre>lua
 ---  function user.new(name, age, hobbies)
 ---    vim.validate{
 ---      name={name, 'string'},
@@ -571,24 +611,24 @@ end
 --- </pre>
 ---
 --- Examples with explicit argument values (can be run directly):
---- <pre>
+--- <pre>lua
 ---  vim.validate{arg1={{'foo'}, 'table'}, arg2={'foo', 'string'}}
----     => NOP (success)
+---     --> NOP (success)
 ---
 ---  vim.validate{arg1={1, 'table'}}
----     => error('arg1: expected table, got number')
+---     --> error('arg1: expected table, got number')
 ---
 ---  vim.validate{arg1={3, function(a) return (a % 2) == 0 end, 'even number'}}
----     => error('arg1: expected even number, got 3')
+---     --> error('arg1: expected even number, got 3')
 --- </pre>
 ---
 --- If multiple types are valid they can be given as a list.
---- <pre>
+--- <pre>lua
 ---  vim.validate{arg1={{'foo'}, {'table', 'string'}}, arg2={'foo', {'table', 'string'}}}
----     => NOP (success)
+---     --> NOP (success)
 ---
 ---  vim.validate{arg1={1, {'string', table'}}}
----     => error('arg1: expected string|table, got number')
+---     --> error('arg1: expected string|table, got number')
 ---
 --- </pre>
 ---
@@ -715,5 +755,30 @@ function vim.is_callable(f)
   return type(m.__call) == 'function'
 end
 
+--- Creates a table whose members are automatically created when accessed, if they don't already
+--- exist.
+---
+--- They mimic defaultdict in python.
+---
+--- If {create} is `nil`, this will create a defaulttable whose constructor function is
+--- this function, effectively allowing to create nested tables on the fly:
+---
+--- <pre>lua
+--- local a = vim.defaulttable()
+--- a.b.c = 1
+--- </pre>
+---
+---@param create function|nil The function called to create a missing value.
+---@return table Empty table with metamethod
+function vim.defaulttable(create)
+  create = create or vim.defaulttable
+  return setmetatable({}, {
+    __index = function(tbl, key)
+      rawset(tbl, key, create())
+      return rawget(tbl, key)
+    end,
+  })
+end
+
 return vim
 -- vim:sw=2 ts=2 et
diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua
index 70f2c425ed..582922ecb6 100644
--- a/runtime/lua/vim/treesitter.lua
+++ b/runtime/lua/vim/treesitter.lua
@@ -3,10 +3,7 @@ local query = require('vim.treesitter.query')
 local language = require('vim.treesitter.language')
 local LanguageTree = require('vim.treesitter.languagetree')
 
--- TODO(bfredl): currently we retain parsers for the lifetime of the buffer.
--- Consider use weak references to release parser if all plugins are done with
--- it.
-local parsers = {}
+local parsers = setmetatable({}, { __mode = 'v' })
 
 local M = vim.tbl_extend('error', query, language)
 
@@ -28,13 +25,15 @@ setmetatable(M, {
   end,
 })
 
---- Creates a new parser.
+--- Creates a new parser
 ---
---- It is not recommended to use this, use vim.treesitter.get_parser() instead.
+--- It is not recommended to use this; use |get_parser()| instead.
 ---
----@param bufnr The buffer the parser will be tied to
----@param lang The language of the parser
----@param opts Options to pass to the created language tree
+---@param bufnr string Buffer the parser will be tied to (0 for current buffer)
+---@param lang string Language of the parser
+---@param opts (table|nil) Options to pass to the created language tree
+---
+---@return LanguageTree |LanguageTree| object to use for parsing
 function M._create_parser(bufnr, lang, opts)
   language.require_language(lang)
   if bufnr == 0 then
@@ -74,16 +73,15 @@ function M._create_parser(bufnr, lang, opts)
   return self
 end
 
---- Gets the parser for this bufnr / ft combination.
+--- Returns the parser for a specific buffer and filetype and attaches it to the buffer
 ---
---- If needed this will create the parser.
---- Unconditionally attach the provided callback
+--- If needed, this will create the parser.
 ---
----@param bufnr The buffer the parser should be tied to
----@param lang The filetype of this parser
----@param opts Options object to pass to the created language tree
+---@param bufnr (number|nil) Buffer the parser should be tied to (default: current buffer)
+---@param lang (string|nil) Filetype of this parser (default: buffer filetype)
+---@param opts (table|nil) Options to pass to the created language tree
 ---
----@returns The parser
+---@return LanguageTree |LanguageTree| object to use for parsing
 function M.get_parser(bufnr, lang, opts)
   opts = opts or {}
 
@@ -103,11 +101,13 @@ function M.get_parser(bufnr, lang, opts)
   return parsers[bufnr]
 end
 
---- Gets a string parser
+--- Returns a string parser
+---
+---@param str string Text to parse
+---@param lang string Language of this string
+---@param opts (table|nil) Options to pass to the created language tree
 ---
----@param str The string to parse
----@param lang The language of this string
----@param opts Options to pass to the created language tree
+---@return LanguageTree |LanguageTree| object to use for parsing
 function M.get_string_parser(str, lang, opts)
   vim.validate({
     str = { str, 'string' },
@@ -118,4 +118,425 @@ function M.get_string_parser(str, lang, opts)
   return LanguageTree.new(str, lang, opts)
 end
 
+--- Determines whether a node is the ancestor of another
+---
+---@param dest userdata Possible ancestor |tsnode|
+---@param source userdata Possible descendant |tsnode|
+---
+---@return boolean True if {dest} is an ancestor of {source}
+function M.is_ancestor(dest, source)
+  if not (dest and source) then
+    return false
+  end
+
+  local current = source
+  while current ~= nil do
+    if current == dest then
+      return true
+    end
+
+    current = current:parent()
+  end
+
+  return false
+end
+
+--- Returns the node's range or an unpacked range table
+---
+---@param node_or_range (userdata|table) |tsnode| or table of positions
+---
+---@return table `{ start_row, start_col, end_row, end_col }`
+function M.get_node_range(node_or_range)
+  if type(node_or_range) == 'table' then
+    return unpack(node_or_range)
+  else
+    return node_or_range:range()
+  end
+end
+
+--- Determines whether (line, col) position is in node range
+---
+---@param node userdata |tsnode| defining the range
+---@param line number Line (0-based)
+---@param col number Column (0-based)
+---
+---@return boolean True if the position is in node range
+function M.is_in_node_range(node, line, col)
+  local start_line, start_col, end_line, end_col = M.get_node_range(node)
+  if line >= start_line and line <= end_line then
+    if line == start_line and line == end_line then
+      return col >= start_col and col < end_col
+    elseif line == start_line then
+      return col >= start_col
+    elseif line == end_line then
+      return col < end_col
+    else
+      return true
+    end
+  else
+    return false
+  end
+end
+
+--- Determines if a node contains a range
+---
+---@param node userdata |tsnode|
+---@param range table
+---
+---@return boolean True if the {node} contains the {range}
+function M.node_contains(node, range)
+  local start_row, start_col, end_row, end_col = node:range()
+  local start_fits = start_row < range[1] or (start_row == range[1] and start_col <= range[2])
+  local end_fits = end_row > range[3] or (end_row == range[3] and end_col >= range[4])
+
+  return start_fits and end_fits
+end
+
+--- Returns a list of highlight captures at the given position
+---
+--- Each capture is represented by a table containing the capture name as a string as
+--- well as a table of metadata (`priority`, `conceal`, ...; empty if none are defined).
+---
+---@param bufnr number Buffer number (0 for current buffer)
+---@param row number Position row
+---@param col number Position column
+---
+---@return table[] List of captures `{ capture = "capture name", metadata = { ... } }`
+function M.get_captures_at_pos(bufnr, row, col)
+  if bufnr == 0 then
+    bufnr = a.nvim_get_current_buf()
+  end
+  local buf_highlighter = M.highlighter.active[bufnr]
+
+  if not buf_highlighter then
+    return {}
+  end
+
+  local matches = {}
+
+  buf_highlighter.tree:for_each_tree(function(tstree, tree)
+    if not tstree then
+      return
+    end
+
+    local root = tstree:root()
+    local root_start_row, _, root_end_row, _ = root:range()
+
+    -- Only worry about trees within the line range
+    if root_start_row > row or root_end_row < row then
+      return
+    end
+
+    local q = buf_highlighter:get_query(tree:lang())
+
+    -- Some injected languages may not have highlight queries.
+    if not q:query() then
+      return
+    end
+
+    local iter = q:query():iter_captures(root, buf_highlighter.bufnr, row, row + 1)
+
+    for capture, node, metadata in iter do
+      if M.is_in_node_range(node, row, col) then
+        local c = q._query.captures[capture] -- name of the capture in the query
+        if c ~= nil then
+          table.insert(matches, { capture = c, metadata = metadata, lang = tree:lang() })
+        end
+      end
+    end
+  end, true)
+  return matches
+end
+
+--- Returns a list of highlight capture names under the cursor
+---
+---@param winnr (number|nil) Window handle or 0 for current window (default)
+---
+---@return string[] List of capture names
+function M.get_captures_at_cursor(winnr)
+  winnr = winnr or 0
+  local bufnr = a.nvim_win_get_buf(winnr)
+  local cursor = a.nvim_win_get_cursor(winnr)
+
+  local data = M.get_captures_at_pos(bufnr, cursor[1] - 1, cursor[2])
+
+  local captures = {}
+
+  for _, capture in ipairs(data) do
+    table.insert(captures, capture.capture)
+  end
+
+  return captures
+end
+
+--- Returns the smallest named node at the given position
+---
+---@param bufnr number Buffer number (0 for current buffer)
+---@param row number Position row
+---@param col number Position column
+---@param opts table Optional keyword arguments:
+---             - lang string|nil Parser language
+---             - ignore_injections boolean Ignore injected languages (default true)
+---
+---@return userdata|nil |tsnode| under the cursor
+function M.get_node_at_pos(bufnr, row, col, opts)
+  if bufnr == 0 then
+    bufnr = a.nvim_get_current_buf()
+  end
+  local ts_range = { row, col, row, col }
+
+  local root_lang_tree = M.get_parser(bufnr, opts.lang)
+  if not root_lang_tree then
+    return
+  end
+
+  return root_lang_tree:named_node_for_range(ts_range, opts)
+end
+
+--- Returns the smallest named node under the cursor
+---
+---@param winnr (number|nil) Window handle or 0 for current window (default)
+---
+---@return string Name of node under the cursor
+function M.get_node_at_cursor(winnr)
+  winnr = winnr or 0
+  local bufnr = a.nvim_win_get_buf(winnr)
+  local cursor = a.nvim_win_get_cursor(winnr)
+
+  return M.get_node_at_pos(bufnr, cursor[1] - 1, cursor[2], { ignore_injections = false }):type()
+end
+
+--- Starts treesitter highlighting for a buffer
+---
+--- Can be used in an ftplugin or FileType autocommand.
+---
+--- Note: By default, disables regex syntax highlighting, which may be required for some plugins.
+--- In this case, add ``vim.bo.syntax = 'on'`` after the call to `start`.
+---
+--- Example:
+--- <pre>lua
+--- vim.api.nvim_create_autocmd( 'FileType', { pattern = 'tex',
+---     callback = function(args)
+---         vim.treesitter.start(args.buf, 'latex')
+---         vim.bo[args.buf].syntax = 'on'  -- only if additional legacy syntax is needed
+---     end
+--- })
+--- </pre>
+---
+---@param bufnr (number|nil) Buffer to be highlighted (default: current buffer)
+---@param lang (string|nil) Language of the parser (default: buffer filetype)
+function M.start(bufnr, lang)
+  bufnr = bufnr or a.nvim_get_current_buf()
+  local parser = M.get_parser(bufnr, lang)
+  M.highlighter.new(parser)
+end
+
+--- Stops treesitter highlighting for a buffer
+---
+---@param bufnr (number|nil) Buffer to stop highlighting (default: current buffer)
+function M.stop(bufnr)
+  bufnr = bufnr or a.nvim_get_current_buf()
+
+  if M.highlighter.active[bufnr] then
+    M.highlighter.active[bufnr]:destroy()
+  end
+end
+
+--- Open a window that displays a textual representation of the nodes in the language tree.
+---
+--- While in the window, press "a" to toggle display of anonymous nodes, "I" to toggle the
+--- display of the source language of each node, and press <Enter> to jump to the node under the
+--- cursor in the source buffer.
+---
+---@param opts table|nil Optional options table with the following possible keys:
+---                      - lang (string|nil): The language of the source buffer. If omitted, the
+---                        filetype of the source buffer is used.
+---                      - bufnr (number|nil): Buffer to draw the tree into. If omitted, a new
+---                        buffer is created.
+---                      - winid (number|nil): Window id to display the tree buffer in. If omitted,
+---                        a new window is created with {command}.
+---                      - command (string|nil): Vimscript command to create the window. Default
+---                        value is "topleft 60vnew". Only used when {winid} is nil.
+---                      - title (string|fun(bufnr:number):string|nil): Title of the window. If a
+---                        function, it accepts the buffer number of the source buffer as its only
+---                        argument and should return a string.
+function M.show_tree(opts)
+  vim.validate({
+    opts = { opts, 't', true },
+  })
+
+  opts = opts or {}
+
+  local Playground = require('vim.treesitter.playground')
+  local buf = a.nvim_get_current_buf()
+  local win = a.nvim_get_current_win()
+  local pg = assert(Playground:new(buf, opts.lang))
+
+  -- Close any existing playground window
+  if vim.b[buf].playground then
+    local w = vim.b[buf].playground
+    if a.nvim_win_is_valid(w) then
+      a.nvim_win_close(w, true)
+    end
+  end
+
+  local w = opts.winid
+  if not w then
+    vim.cmd(opts.command or 'topleft 60vnew')
+    w = a.nvim_get_current_win()
+  end
+
+  local b = opts.bufnr
+  if b then
+    a.nvim_win_set_buf(w, b)
+  else
+    b = a.nvim_win_get_buf(w)
+  end
+
+  vim.b[buf].playground = w
+
+  vim.wo[w].scrolloff = 5
+  vim.wo[w].wrap = false
+  vim.bo[b].buflisted = false
+  vim.bo[b].buftype = 'nofile'
+  vim.bo[b].bufhidden = 'wipe'
+
+  local title = opts.title
+  if not title then
+    local bufname = a.nvim_buf_get_name(buf)
+    title = string.format('Syntax tree for %s', vim.fn.fnamemodify(bufname, ':.'))
+  elseif type(title) == 'function' then
+    title = title(buf)
+  end
+
+  assert(type(title) == 'string', 'Window title must be a string')
+  a.nvim_buf_set_name(b, title)
+
+  pg:draw(b)
+
+  vim.fn.matchadd('Comment', '\\[[0-9:-]\\+\\]')
+  vim.fn.matchadd('String', '".*"')
+
+  a.nvim_buf_clear_namespace(buf, pg.ns, 0, -1)
+  a.nvim_buf_set_keymap(b, 'n', '<CR>', '', {
+    desc = 'Jump to the node under the cursor in the source buffer',
+    callback = function()
+      local row = a.nvim_win_get_cursor(w)[1]
+      local pos = pg:get(row)
+      a.nvim_set_current_win(win)
+      a.nvim_win_set_cursor(win, { pos.lnum + 1, pos.col })
+    end,
+  })
+  a.nvim_buf_set_keymap(b, 'n', 'a', '', {
+    desc = 'Toggle anonymous nodes',
+    callback = function()
+      pg.opts.anon = not pg.opts.anon
+      pg:draw(b)
+    end,
+  })
+  a.nvim_buf_set_keymap(b, 'n', 'I', '', {
+    desc = 'Toggle language display',
+    callback = function()
+      pg.opts.lang = not pg.opts.lang
+      pg:draw(b)
+    end,
+  })
+
+  local group = a.nvim_create_augroup('treesitter/playground', {})
+
+  a.nvim_create_autocmd('CursorMoved', {
+    group = group,
+    buffer = b,
+    callback = function()
+      a.nvim_buf_clear_namespace(buf, pg.ns, 0, -1)
+      local row = a.nvim_win_get_cursor(w)[1]
+      local pos = pg:get(row)
+      a.nvim_buf_set_extmark(buf, pg.ns, pos.lnum, pos.col, {
+        end_row = pos.end_lnum,
+        end_col = math.max(0, pos.end_col),
+        hl_group = 'Visual',
+      })
+    end,
+  })
+
+  a.nvim_create_autocmd('CursorMoved', {
+    group = group,
+    buffer = buf,
+    callback = function()
+      if not a.nvim_buf_is_loaded(b) then
+        return true
+      end
+
+      a.nvim_buf_clear_namespace(b, pg.ns, 0, -1)
+
+      local cursor = a.nvim_win_get_cursor(win)
+      local cursor_node = M.get_node_at_pos(buf, cursor[1] - 1, cursor[2], {
+        lang = opts.lang,
+        ignore_injections = false,
+      })
+      if not cursor_node then
+        return
+      end
+
+      local cursor_node_id = cursor_node:id()
+      for i, v in pg:iter() do
+        if v.id == cursor_node_id then
+          local start = v.depth
+          local end_col = start + #v.text
+          a.nvim_buf_set_extmark(b, pg.ns, i - 1, start, {
+            end_col = end_col,
+            hl_group = 'Visual',
+          })
+          a.nvim_win_set_cursor(w, { i, 0 })
+          break
+        end
+      end
+    end,
+  })
+
+  a.nvim_create_autocmd({ 'TextChanged', 'InsertLeave' }, {
+    group = group,
+    buffer = buf,
+    callback = function()
+      if not a.nvim_buf_is_loaded(b) then
+        return true
+      end
+
+      pg = assert(Playground:new(buf, opts.lang))
+      pg:draw(b)
+    end,
+  })
+
+  a.nvim_create_autocmd('BufLeave', {
+    group = group,
+    buffer = b,
+    callback = function()
+      a.nvim_buf_clear_namespace(buf, pg.ns, 0, -1)
+    end,
+  })
+
+  a.nvim_create_autocmd('BufLeave', {
+    group = group,
+    buffer = buf,
+    callback = function()
+      if not a.nvim_buf_is_loaded(b) then
+        return true
+      end
+
+      a.nvim_buf_clear_namespace(b, pg.ns, 0, -1)
+    end,
+  })
+
+  a.nvim_create_autocmd('BufHidden', {
+    group = group,
+    buffer = buf,
+    once = true,
+    callback = function()
+      if a.nvim_win_is_valid(w) then
+        a.nvim_win_close(w, true)
+      end
+    end,
+  })
+end
+
 return M
diff --git a/runtime/lua/vim/treesitter/health.lua b/runtime/lua/vim/treesitter/health.lua
index 3bd59ca282..c0a1eca0ce 100644
--- a/runtime/lua/vim/treesitter/health.lua
+++ b/runtime/lua/vim/treesitter/health.lua
@@ -1,36 +1,33 @@
 local M = {}
 local ts = vim.treesitter
+local health = require('vim.health')
 
 --- Lists the parsers currently installed
 ---
----@return A list of parsers
+---@return string[] list of parser files
 function M.list_parsers()
   return vim.api.nvim_get_runtime_file('parser/*', true)
 end
 
 --- Performs a healthcheck for treesitter integration
 function M.check()
-  local report_info = vim.fn['health#report_info']
-  local report_ok = vim.fn['health#report_ok']
-  local report_error = vim.fn['health#report_error']
   local parsers = M.list_parsers()
 
-  report_info(string.format('Runtime ABI version : %d', ts.language_version))
+  health.report_info(string.format('Nvim runtime ABI version: %d', ts.language_version))
 
   for _, parser in pairs(parsers) do
     local parsername = vim.fn.fnamemodify(parser, ':t:r')
-
     local is_loadable, ret = pcall(ts.language.require_language, parsername)
 
-    if not is_loadable then
-      report_error(string.format('Impossible to load parser for %s: %s', parsername, ret))
+    if not is_loadable or not ret then
+      health.report_error(
+        string.format('Parser "%s" failed to load (path: %s): %s', parsername, parser, ret or '?')
+      )
     elseif ret then
       local lang = ts.language.inspect_language(parsername)
-      report_ok(
-        string.format('Loaded parser for %s: ABI version %d', parsername, lang._abi_version)
+      health.report_ok(
+        string.format('Parser: %-10s ABI: %d, path: %s', parsername, lang._abi_version, parser)
       )
-    else
-      report_error(string.format('Unable to load parser for %s', parsername))
     end
   end
 end
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index e27a5fa9c3..d77a0d0d03 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -2,6 +2,7 @@ local a = vim.api
 local query = require('vim.treesitter.query')
 
 -- support reload for quick experimentation
+---@class TSHighlighter
 local TSHighlighter = rawget(vim.treesitter, 'TSHighlighter') or {}
 TSHighlighter.__index = TSHighlighter
 
@@ -12,105 +13,18 @@ TSHighlighterQuery.__index = TSHighlighterQuery
 
 local ns = a.nvim_create_namespace('treesitter/highlighter')
 
-local _default_highlights = {}
-local _link_default_highlight_once = function(from, to)
-  if not _default_highlights[from] then
-    _default_highlights[from] = true
-    a.nvim_set_hl(0, from, { link = to, default = true })
-  end
-
-  return from
-end
-
--- If @definition.special does not exist use @definition instead
-local subcapture_fallback = {
-  __index = function(self, capture)
-    local rtn
-    local shortened = capture
-    while not rtn and shortened do
-      shortened = shortened:match('(.*)%.')
-      rtn = shortened and rawget(self, shortened)
-    end
-    rawset(self, capture, rtn or '__notfound')
-    return rtn
-  end,
-}
-
-TSHighlighter.hl_map = setmetatable({
-  ['error'] = 'Error',
-  ['text.underline'] = 'Underlined',
-  ['todo'] = 'Todo',
-  ['debug'] = 'Debug',
-
-  -- Miscs
-  ['comment'] = 'Comment',
-  ['punctuation.delimiter'] = 'Delimiter',
-  ['punctuation.bracket'] = 'Delimiter',
-  ['punctuation.special'] = 'Delimiter',
-
-  -- Constants
-  ['constant'] = 'Constant',
-  ['constant.builtin'] = 'Special',
-  ['constant.macro'] = 'Define',
-  ['define'] = 'Define',
-  ['macro'] = 'Macro',
-  ['string'] = 'String',
-  ['string.regex'] = 'String',
-  ['string.escape'] = 'SpecialChar',
-  ['character'] = 'Character',
-  ['character.special'] = 'SpecialChar',
-  ['number'] = 'Number',
-  ['boolean'] = 'Boolean',
-  ['float'] = 'Float',
-
-  -- Functions
-  ['function'] = 'Function',
-  ['function.special'] = 'Function',
-  ['function.builtin'] = 'Special',
-  ['function.macro'] = 'Macro',
-  ['parameter'] = 'Identifier',
-  ['method'] = 'Function',
-  ['field'] = 'Identifier',
-  ['property'] = 'Identifier',
-  ['constructor'] = 'Special',
-
-  -- Keywords
-  ['conditional'] = 'Conditional',
-  ['repeat'] = 'Repeat',
-  ['label'] = 'Label',
-  ['operator'] = 'Operator',
-  ['keyword'] = 'Keyword',
-  ['exception'] = 'Exception',
-
-  ['type'] = 'Type',
-  ['type.builtin'] = 'Type',
-  ['type.qualifier'] = 'Type',
-  ['type.definition'] = 'Typedef',
-  ['storageclass'] = 'StorageClass',
-  ['structure'] = 'Structure',
-  ['include'] = 'Include',
-  ['preproc'] = 'PreProc',
-}, subcapture_fallback)
-
----@private
-local function is_highlight_name(capture_name)
-  local firstc = string.sub(capture_name, 1, 1)
-  return firstc ~= string.lower(firstc)
-end
-
 ---@private
 function TSHighlighterQuery.new(lang, query_string)
   local self = setmetatable({}, { __index = TSHighlighterQuery })
 
   self.hl_cache = setmetatable({}, {
     __index = function(table, capture)
-      local hl, is_vim_highlight = self:_get_hl_from_capture(capture)
-      if not is_vim_highlight then
-        hl = _link_default_highlight_once(lang .. hl, hl)
+      local name = self._query.captures[capture]
+      local id = 0
+      if not vim.startswith(name, '_') then
+        id = a.nvim_get_hl_id_by_name('@' .. name .. '.' .. lang)
       end
 
-      local id = a.nvim_get_hl_id_by_name(hl)
-
       rawset(table, capture, id)
       return id
     end,
@@ -130,25 +44,12 @@ function TSHighlighterQuery:query()
   return self._query
 end
 
----@private
---- Get the hl from capture.
---- Returns a tuple { highlight_name: string, is_builtin: bool }
-function TSHighlighterQuery:_get_hl_from_capture(capture)
-  local name = self._query.captures[capture]
-
-  if is_highlight_name(name) then
-    -- From "Normal.left" only keep "Normal"
-    return vim.split(name, '.', true)[1], true
-  else
-    return TSHighlighter.hl_map[name] or 0, false
-  end
-end
-
 --- Creates a new highlighter using @param tree
 ---
----@param tree The language tree to use for highlighting
----@param opts Table used to configure the highlighter
----           - queries: Table to overwrite queries used by the highlighter
+---@param tree LanguageTree |LanguageTree| parser object to use for highlighting
+---@param opts (table|nil) Configuration of the highlighter:
+---           - queries table overwrite queries used by the highlighter
+---@return TSHighlighter Created highlighter object
 function TSHighlighter.new(tree, opts)
   local self = setmetatable({}, TSHighlighter)
 
@@ -187,7 +88,10 @@ function TSHighlighter.new(tree, opts)
     end
   end
 
-  a.nvim_buf_set_option(self.bufnr, 'syntax', '')
+  self.orig_spelloptions = vim.bo[self.bufnr].spelloptions
+
+  vim.bo[self.bufnr].syntax = ''
+  vim.b[self.bufnr].ts_highlight = true
 
   TSHighlighter.active[self.bufnr] = self
 
@@ -196,9 +100,13 @@ function TSHighlighter.new(tree, opts)
   -- syntax FileType autocmds. Later on we should integrate with the
   -- `:syntax` and `set syntax=...` machinery properly.
   if vim.g.syntax_on ~= 1 then
-    vim.api.nvim_command('runtime! syntax/synload.vim')
+    vim.cmd.runtime({ 'syntax/synload.vim', bang = true })
   end
 
+  a.nvim_buf_call(self.bufnr, function()
+    vim.opt_local.spelloptions:append('noplainbuffer')
+  end)
+
   self.tree:parse()
 
   return self
@@ -209,6 +117,14 @@ function TSHighlighter:destroy()
   if TSHighlighter.active[self.bufnr] then
     TSHighlighter.active[self.bufnr] = nil
   end
+
+  if vim.api.nvim_buf_is_loaded(self.bufnr) then
+    vim.bo[self.bufnr].spelloptions = self.orig_spelloptions
+    vim.b[self.bufnr].ts_highlight = nil
+    if vim.g.syntax_on == 1 then
+      a.nvim_exec_autocmds('FileType', { group = 'syntaxset', buffer = self.bufnr })
+    end
+  end
 end
 
 ---@private
@@ -246,8 +162,10 @@ function TSHighlighter:on_changedtree(changes)
 end
 
 --- Gets the query used for @param lang
----
----@param lang A language used by the highlighter.
+--
+---@private
+---@param lang string Language used by the highlighter.
+---@return Query
 function TSHighlighter:get_query(lang)
   if not self._queries[lang] then
     self._queries[lang] = TSHighlighterQuery.new(lang)
@@ -257,7 +175,7 @@ function TSHighlighter:get_query(lang)
 end
 
 ---@private
-local function on_line_impl(self, buf, line)
+local function on_line_impl(self, buf, line, is_spell_nav)
   self.tree:for_each_tree(function(tstree, tree)
     if not tstree then
       return
@@ -294,14 +212,26 @@ local function on_line_impl(self, buf, line)
       local start_row, start_col, end_row, end_col = node:range()
       local hl = highlighter_query.hl_cache[capture]
 
-      if hl and end_row >= line then
+      local capture_name = highlighter_query:query().captures[capture]
+      local spell = nil
+      if capture_name == 'spell' then
+        spell = true
+      elseif capture_name == 'nospell' then
+        spell = false
+      end
+
+      -- Give nospell a higher priority so it always overrides spell captures.
+      local spell_pri_offset = capture_name == 'nospell' and 1 or 0
+
+      if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
         a.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
           end_line = end_row,
           end_col = end_col,
           hl_group = hl,
           ephemeral = true,
-          priority = tonumber(metadata.priority) or 100, -- Low but leaves room below
+          priority = (tonumber(metadata.priority) or 100) + spell_pri_offset, -- Low but leaves room below
           conceal = metadata.conceal,
+          spell = spell,
         })
       end
       if start_row > line then
@@ -318,7 +248,21 @@ function TSHighlighter._on_line(_, _win, buf, line, _)
     return
   end
 
-  on_line_impl(self, buf, line)
+  on_line_impl(self, buf, line, false)
+end
+
+---@private
+function TSHighlighter._on_spell_nav(_, _, buf, srow, _, erow, _)
+  local self = TSHighlighter.active[buf]
+  if not self then
+    return
+  end
+
+  self:reset_highlight_state()
+
+  for row = srow, erow do
+    on_line_impl(self, buf, row, true)
+  end
 end
 
 ---@private
@@ -345,6 +289,7 @@ a.nvim_set_decoration_provider(ns, {
   on_buf = TSHighlighter._on_buf,
   on_win = TSHighlighter._on_win,
   on_line = TSHighlighter._on_line,
+  _on_spell_nav = TSHighlighter._on_spell_nav,
 })
 
 return TSHighlighter
diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua
index dfb6f5be84..c92d63b8c4 100644
--- a/runtime/lua/vim/treesitter/language.lua
+++ b/runtime/lua/vim/treesitter/language.lua
@@ -2,14 +2,16 @@ local a = vim.api
 
 local M = {}
 
---- Asserts that the provided language is installed, and optionally provide a path for the parser
+--- Asserts that a parser for the language {lang} is installed.
 ---
---- Parsers are searched in the `parser` runtime directory.
+--- Parsers are searched in the `parser` runtime directory, or the provided {path}
 ---
----@param lang The language the parser should parse
----@param path Optional path the parser is located at
----@param silent Don't throw an error if language not found
-function M.require_language(lang, path, silent)
+---@param lang string Language the parser should parse
+---@param path (string|nil) Optional path the parser is located at
+---@param silent (boolean|nil) Don't throw an error if language not found
+---@param symbol_name (string|nil) Internal symbol name for the language to load
+---@return boolean If the specified language is installed
+function M.require_language(lang, path, silent, symbol_name)
   if vim._ts_has_language(lang) then
     return true
   end
@@ -21,7 +23,6 @@ function M.require_language(lang, path, silent)
         return false
       end
 
-      -- TODO(bfredl): help tag?
       error("no parser for '" .. lang .. "' language, see :help treesitter-parsers")
     end
     path = paths[1]
@@ -29,10 +30,10 @@ function M.require_language(lang, path, silent)
 
   if silent then
     return pcall(function()
-      vim._ts_add_language(path, lang)
+      vim._ts_add_language(path, lang, symbol_name)
     end)
   else
-    vim._ts_add_language(path, lang)
+    vim._ts_add_language(path, lang, symbol_name)
   end
 
   return true
@@ -42,7 +43,8 @@ end
 ---
 --- Inspecting provides some useful information on the language like node names, ...
 ---
----@param lang The language.
+---@param lang string Language
+---@return table
 function M.inspect_language(lang)
   M.require_language(lang)
   return vim._ts_inspect_language(lang)
diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index 4d3b0631a2..a1e96f8ef2 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -2,19 +2,35 @@ local a = vim.api
 local query = require('vim.treesitter.query')
 local language = require('vim.treesitter.language')
 
+---@class LanguageTree
+---@field _callbacks function[] Callback handlers
+---@field _children LanguageTree[] Injected languages
+---@field _injection_query table Queries defining injected languages
+---@field _opts table Options
+---@field _parser userdata Parser for language
+---@field _regions table List of regions this tree should manage and parse
+---@field _lang string Language name
+---@field _regions table
+---@field _source (number|string) Buffer or string to parse
+---@field _trees userdata[] Reference to parsed |tstree| (one for each language)
+---@field _valid boolean If the parsed tree is valid
+
 local LanguageTree = {}
 LanguageTree.__index = LanguageTree
 
---- Represents a single treesitter parser for a language.
---- The language can contain child languages with in its range,
---- hence the tree.
+--- A |LanguageTree| holds the treesitter parser for a given language {lang} used
+--- to parse a buffer. As the buffer may contain injected languages, the LanguageTree
+--- needs to store parsers for these child languages as well (which in turn may contain
+--- child languages themselves, hence the name).
 ---
----@param source Can be a bufnr or a string of text to parse
----@param lang The language this tree represents
----@param opts Options table
----@param opts.injections A table of language to injection query strings.
----                      This is useful for overriding the built-in runtime file
----                      searching for the injection language query per language.
+---@param source (number|string) Buffer or a string of text to parse
+---@param lang string Root language this tree represents
+---@param opts (table|nil) Optional keyword arguments:
+---             - injections table Mapping language to injection query strings.
+---                                This is useful for overriding the built-in
+---                                runtime file searching for the injection language
+---                                query per language.
+---@return LanguageTree |LanguageTree| parser object
 function LanguageTree.new(source, lang, opts)
   language.require_language(lang)
   opts = opts or {}
@@ -94,6 +110,9 @@ end
 --- for the language this tree represents.
 --- This will run the injection query for this language to
 --- determine if any child languages should be created.
+---
+---@return userdata[] Table of parsed |tstree|
+---@return table Change list
 function LanguageTree:parse()
   if self._valid then
     return self._trees
@@ -167,10 +186,10 @@ function LanguageTree:parse()
   return self._trees, changes
 end
 
---- Invokes the callback for each LanguageTree and it's children recursively
+--- Invokes the callback for each |LanguageTree| and its children recursively
 ---
----@param fn The function to invoke. This is invoked with arguments (tree: LanguageTree, lang: string)
----@param include_self Whether to include the invoking tree in the results.
+---@param fn function(tree: LanguageTree, lang: string)
+---@param include_self boolean Whether to include the invoking tree in the results
 function LanguageTree:for_each_child(fn, include_self)
   if include_self then
     fn(self, self._lang)
@@ -181,12 +200,11 @@ function LanguageTree:for_each_child(fn, include_self)
   end
 end
 
---- Invokes the callback for each treesitter trees recursively.
+--- Invokes the callback for each |LanguageTree| recursively.
 ---
---- Note, this includes the invoking language tree's trees as well.
+--- Note: This includes the invoking tree's child trees as well.
 ---
----@param fn The callback to invoke. The callback is invoked with arguments
----         (tree: TSTree, languageTree: LanguageTree)
+---@param fn function(tree: TSTree, languageTree: LanguageTree)
 function LanguageTree:for_each_tree(fn)
   for _, tree in ipairs(self._trees) do
     fn(tree, self)
@@ -197,11 +215,13 @@ function LanguageTree:for_each_tree(fn)
   end
 end
 
---- Adds a child language to this tree.
+--- Adds a child language to this |LanguageTree|.
 ---
 --- If the language already exists as a child, it will first be removed.
 ---
----@param lang The language to add.
+---@private
+---@param lang string Language to add.
+---@return LanguageTree Injected |LanguageTree|
 function LanguageTree:add_child(lang)
   if self._children[lang] then
     self:remove_child(lang)
@@ -215,9 +235,10 @@ function LanguageTree:add_child(lang)
   return self._children[lang]
 end
 
---- Removes a child language from this tree.
+--- Removes a child language from this |LanguageTree|.
 ---
----@param lang The language to remove.
+---@private
+---@param lang string Language to remove.
 function LanguageTree:remove_child(lang)
   local child = self._children[lang]
 
@@ -229,12 +250,11 @@ function LanguageTree:remove_child(lang)
   end
 end
 
---- Destroys this language tree and all its children.
+--- Destroys this |LanguageTree| and all its children.
 ---
 --- Any cleanup logic should be performed here.
 ---
---- Note:
---- This DOES NOT remove this tree from a parent. Instead,
+--- Note: This DOES NOT remove this tree from a parent. Instead,
 --- `remove_child` must be called on the parent to remove it.
 function LanguageTree:destroy()
   -- Cleanup here
@@ -243,23 +263,24 @@ function LanguageTree:destroy()
   end
 end
 
---- Sets the included regions that should be parsed by this parser.
+--- Sets the included regions that should be parsed by this |LanguageTree|.
 --- A region is a set of nodes and/or ranges that will be parsed in the same context.
 ---
---- For example, `{ { node1 }, { node2} }` is two separate regions.
---- This will be parsed by the parser in two different contexts... thus resulting
+--- For example, `{ { node1 }, { node2} }` contains two separate regions.
+--- They will be parsed by the parser in two different contexts, thus resulting
 --- in two separate trees.
 ---
---- `{ { node1, node2 } }` is a single region consisting of two nodes.
---- This will be parsed by the parser in a single context... thus resulting
+--- On the other hand, `{ { node1, node2 } }` is a single region consisting of
+--- two nodes. This will be parsed by the parser in a single context, thus resulting
 --- in a single tree.
 ---
 --- This allows for embedded languages to be parsed together across different
 --- nodes, which is useful for templating languages like ERB and EJS.
 ---
---- Note, this call invalidates the tree and requires it to be parsed again.
+--- Note: This call invalidates the tree and requires it to be parsed again.
 ---
----@param regions (table) list of regions this tree should manage and parse.
+---@private
+---@param regions table List of regions this tree should manage and parse.
 function LanguageTree:set_included_regions(regions)
   -- Transform the tables from 4 element long to 6 element long (with byte offset)
   for _, region in ipairs(regions) do
@@ -288,7 +309,7 @@ function LanguageTree:set_included_regions(regions)
   -- Trees are no longer valid now that we have changed regions.
   -- TODO(vigoux,steelsojka): Look into doing this smarter so we can use some of the
   --                          old trees for incremental parsing. Currently, this only
-  --                          effects injected languages.
+  --                          affects injected languages.
   self._trees = {}
   self:invalidate()
 end
@@ -299,7 +320,7 @@ function LanguageTree:included_regions()
 end
 
 ---@private
-local function get_node_range(node, id, metadata)
+local function get_range_from_metadata(node, id, metadata)
   if metadata[id] and metadata[id].range then
     return metadata[id].range
   end
@@ -362,7 +383,7 @@ function LanguageTree:_get_injections()
         elseif name == 'combined' then
           combined = true
         elseif name == 'content' and #ranges == 0 then
-          table.insert(ranges, get_node_range(node, id, metadata))
+          table.insert(ranges, get_range_from_metadata(node, id, metadata))
           -- Ignore any tags that start with "_"
           -- Allows for other tags to be used in matches
         elseif string.sub(name, 1, 1) ~= '_' then
@@ -371,7 +392,7 @@ function LanguageTree:_get_injections()
           end
 
           if #ranges == 0 then
-            table.insert(ranges, get_node_range(node, id, metadata))
+            table.insert(ranges, get_range_from_metadata(node, id, metadata))
           end
         end
       end
@@ -493,8 +514,8 @@ function LanguageTree:_on_detach(...)
   self:_do_callback('detach', ...)
 end
 
---- Registers callbacks for the parser.
----@param cbs table An |nvim_buf_attach()|-like table argument with the following keys :
+--- Registers callbacks for the |LanguageTree|.
+---@param cbs table An |nvim_buf_attach()|-like table argument with the following handlers:
 ---           - `on_bytes` : see |nvim_buf_attach()|, but this will be called _after_ the parsers callback.
 ---           - `on_changedtree` : a callback that will be called every time the tree has syntactical changes.
 ---              It will only be passed one argument, which is a table of the ranges (as node ranges) that
@@ -536,9 +557,10 @@ local function tree_contains(tree, range)
   return start_fits and end_fits
 end
 
---- Determines whether {range} is contained in this language tree
+--- Determines whether {range} is contained in the |LanguageTree|.
 ---
----@param range A range, that is a `{ start_line, start_col, end_line, end_col }` table.
+---@param range table `{ start_line, start_col, end_line, end_col }`
+---@return boolean
 function LanguageTree:contains(range)
   for _, tree in pairs(self._trees) do
     if tree_contains(tree, range) then
@@ -549,9 +571,52 @@ function LanguageTree:contains(range)
   return false
 end
 
---- Gets the appropriate language that contains {range}
+--- Gets the tree that contains {range}.
+---
+---@param range table `{ start_line, start_col, end_line, end_col }`
+---@param opts table|nil Optional keyword arguments:
+---             - ignore_injections boolean Ignore injected languages (default true)
+---@return userdata|nil Contained |tstree|
+function LanguageTree:tree_for_range(range, opts)
+  opts = opts or {}
+  local ignore = vim.F.if_nil(opts.ignore_injections, true)
+
+  if not ignore then
+    for _, child in pairs(self._children) do
+      for _, tree in pairs(child:trees()) do
+        if tree_contains(tree, range) then
+          return tree
+        end
+      end
+    end
+  end
+
+  for _, tree in pairs(self._trees) do
+    if tree_contains(tree, range) then
+      return tree
+    end
+  end
+
+  return nil
+end
+
+--- Gets the smallest named node that contains {range}.
+---
+---@param range table `{ start_line, start_col, end_line, end_col }`
+---@param opts table|nil Optional keyword arguments:
+---             - ignore_injections boolean Ignore injected languages (default true)
+---@return userdata|nil Found |tsnode|
+function LanguageTree:named_node_for_range(range, opts)
+  local tree = self:tree_for_range(range, opts)
+  if tree then
+    return tree:root():named_descendant_for_range(unpack(range))
+  end
+end
+
+--- Gets the appropriate language that contains {range}.
 ---
----@param range A text range, see |LanguageTree:contains|
+---@param range table `{ start_line, start_col, end_line, end_col }`
+---@return LanguageTree Managing {range}
 function LanguageTree:language_for_range(range)
   for _, child in pairs(self._children) do
     if child:contains(range) then
diff --git a/runtime/lua/vim/treesitter/playground.lua b/runtime/lua/vim/treesitter/playground.lua
new file mode 100644
index 0000000000..bb073290c6
--- /dev/null
+++ b/runtime/lua/vim/treesitter/playground.lua
@@ -0,0 +1,186 @@
+local api = vim.api
+
+local M = {}
+
+---@class Playground
+---@field ns number API namespace
+---@field opts table Options table with the following keys:
+---                  - anon (boolean): If true, display anonymous nodes
+---                  - lang (boolean): If true, display the language alongside each node
+---
+---@class Node
+---@field id number Node id
+---@field text string Node text
+---@field named boolean True if this is a named (non-anonymous) node
+---@field depth number Depth of the node within the tree
+---@field lnum number Beginning line number of this node in the source buffer
+---@field col number Beginning column number of this node in the source buffer
+---@field end_lnum number Final line number of this node in the source buffer
+---@field end_col number Final column number of this node in the source buffer
+---@field lang string Source language of this node
+
+--- Traverse all child nodes starting at {node}.
+---
+--- This is a recursive function. The {depth} parameter indicates the current recursion level.
+--- {lang} is a string indicating the language of the tree currently being traversed. Each traversed
+--- node is added to {tree}. When recursion completes, {tree} is an array of all nodes in the order
+--- they were visited.
+---
+--- {injections} is a table mapping node ids from the primary tree to language tree injections. Each
+--- injected language has a series of trees nested within the primary language's tree, and the root
+--- node of each of these trees is contained within a node in the primary tree. The {injections}
+--- table maps nodes in the primary tree to root nodes of injected trees.
+---
+---@param node userdata Starting node to begin traversal |tsnode|
+---@param depth number Current recursion depth
+---@param lang string Language of the tree currently being traversed
+---@param injections table Mapping of node ids to root nodes of injected language trees (see
+---                        explanation above)
+---@param tree Node[] Output table containing a list of tables each representing a node in the tree
+---@private
+local function traverse(node, depth, lang, injections, tree)
+  local injection = injections[node:id()]
+  if injection then
+    traverse(injection.root, depth, injection.lang, injections, tree)
+  end
+
+  for child, field in node:iter_children() do
+    local type = child:type()
+    local lnum, col, end_lnum, end_col = child:range()
+    local named = child:named()
+    local text
+    if named then
+      if field then
+        text = string.format('%s: (%s)', field, type)
+      else
+        text = string.format('(%s)', type)
+      end
+    else
+      text = string.format('"%s"', type:gsub('\n', '\\n'))
+    end
+
+    table.insert(tree, {
+      id = child:id(),
+      text = text,
+      named = named,
+      depth = depth,
+      lnum = lnum,
+      col = col,
+      end_lnum = end_lnum,
+      end_col = end_col,
+      lang = lang,
+    })
+
+    traverse(child, depth + 1, lang, injections, tree)
+  end
+
+  return tree
+end
+
+--- Create a new Playground object.
+---
+---@param bufnr number Source buffer number
+---@param lang string|nil Language of source buffer
+---
+---@return Playground|nil
+---@return string|nil Error message, if any
+---
+---@private
+function M.new(self, bufnr, lang)
+  local ok, parser = pcall(vim.treesitter.get_parser, bufnr or 0, lang)
+  if not ok then
+    return nil, 'No parser available for the given buffer'
+  end
+
+  -- For each child tree (injected language), find the root of the tree and locate the node within
+  -- the primary tree that contains that root. Add a mapping from the node in the primary tree to
+  -- the root in the child tree to the {injections} table.
+  local root = parser:parse()[1]:root()
+  local injections = {}
+  parser:for_each_child(function(child, lang_)
+    child:for_each_tree(function(tree)
+      local r = tree:root()
+      local node = root:named_descendant_for_range(r:range())
+      if node then
+        injections[node:id()] = {
+          lang = lang_,
+          root = r,
+        }
+      end
+    end)
+  end)
+
+  local nodes = traverse(root, 0, parser:lang(), injections, {})
+
+  local named = {}
+  for _, v in ipairs(nodes) do
+    if v.named then
+      named[#named + 1] = v
+    end
+  end
+
+  local t = {
+    ns = api.nvim_create_namespace(''),
+    nodes = nodes,
+    named = named,
+    opts = {
+      anon = false,
+      lang = false,
+    },
+  }
+
+  setmetatable(t, self)
+  self.__index = self
+  return t
+end
+
+--- Write the contents of this Playground into {bufnr}.
+---
+---@param bufnr number Buffer number to write into.
+---@private
+function M.draw(self, bufnr)
+  vim.bo[bufnr].modifiable = true
+  local lines = {}
+  for _, item in self:iter() do
+    lines[#lines + 1] = table.concat({
+      string.rep(' ', item.depth),
+      item.text,
+      item.lnum == item.end_lnum
+          and string.format(' [%d:%d-%d]', item.lnum + 1, item.col + 1, item.end_col)
+        or string.format(
+          ' [%d:%d-%d:%d]',
+          item.lnum + 1,
+          item.col + 1,
+          item.end_lnum + 1,
+          item.end_col
+        ),
+      self.opts.lang and string.format(' %s', item.lang) or '',
+    })
+  end
+  api.nvim_buf_set_lines(bufnr, 0, -1, false, lines)
+  vim.bo[bufnr].modifiable = false
+end
+
+--- Get node {i} from this Playground object.
+---
+--- The node number is dependent on whether or not anonymous nodes are displayed.
+---
+---@param i number Node number to get
+---@return Node
+---@private
+function M.get(self, i)
+  local t = self.opts.anon and self.nodes or self.named
+  return t[i]
+end
+
+--- Iterate over all of the nodes in this Playground object.
+---
+---@return function Iterator over all nodes in this Playground
+---@return table
+---@return number
+---@private
+function M.iter(self)
+  return ipairs(self.opts.anon and self.nodes or self.named)
+end
+
+return M
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 103e85abfd..dbf134573d 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -3,6 +3,11 @@ local language = require('vim.treesitter.language')
 
 -- query: pattern matching on trees
 -- predicate matching is implemented in lua
+--
+---@class Query
+---@field captures string[] List of captures used in query
+---@field info table Contains used queries, predicates, directives
+---@field query userdata Parsed query
 local Query = {}
 Query.__index = Query
 
@@ -34,11 +39,24 @@ local function safe_read(filename, read_quantifier)
   return content
 end
 
+---@private
+--- Adds {ilang} to {base_langs}, only if {ilang} is different than {lang}
+---
+---@return boolean true If lang == ilang
+local function add_included_lang(base_langs, lang, ilang)
+  if lang == ilang then
+    return true
+  end
+  table.insert(base_langs, ilang)
+  return false
+end
+
 --- Gets the list of files used to make up a query
 ---
----@param lang The language
----@param query_name The name of the query to load
----@param is_included Internal parameter, most of the time left as `nil`
+---@param lang string Language to get query for
+---@param query_name string Name of the query to load (e.g., "highlights")
+---@param is_included (boolean|nil) Internal parameter, most of the time left as `nil`
+---@return string[] query_files List of files to load for given query and language
 function M.get_query_files(lang, query_name, is_included)
   local query_path = string.format('queries/%s/%s.scm', lang, query_name)
   local lang_files = dedupe_files(a.nvim_get_runtime_file(query_path, true))
@@ -47,6 +65,9 @@ function M.get_query_files(lang, query_name, is_included)
     return {}
   end
 
+  local base_query = nil
+  local extensions = {}
+
   local base_langs = {}
 
   -- Now get the base languages by looking at the first line of every file
@@ -55,27 +76,53 @@ function M.get_query_files(lang, query_name, is_included)
   --
   -- {language} ::= {lang} | ({lang})
   local MODELINE_FORMAT = '^;+%s*inherits%s*:?%s*([a-z_,()]+)%s*$'
+  local EXTENDS_FORMAT = '^;+%s*extends%s*$'
 
-  for _, file in ipairs(lang_files) do
-    local modeline = safe_read(file, '*l')
+  for _, filename in ipairs(lang_files) do
+    local file, err = io.open(filename, 'r')
+    if not file then
+      error(err)
+    end
 
-    if modeline then
-      local langlist = modeline:match(MODELINE_FORMAT)
+    local extension = false
 
+    for modeline in
+      function()
+        return file:read('*l')
+      end
+    do
+      if not vim.startswith(modeline, ';') then
+        break
+      end
+
+      local langlist = modeline:match(MODELINE_FORMAT)
       if langlist then
         for _, incllang in ipairs(vim.split(langlist, ',', true)) do
           local is_optional = incllang:match('%(.*%)')
 
           if is_optional then
             if not is_included then
-              table.insert(base_langs, incllang:sub(2, #incllang - 1))
+              if add_included_lang(base_langs, lang, incllang:sub(2, #incllang - 1)) then
+                extension = true
+              end
             end
           else
-            table.insert(base_langs, incllang)
+            if add_included_lang(base_langs, lang, incllang) then
+              extension = true
+            end
           end
         end
+      elseif modeline:match(EXTENDS_FORMAT) then
+        extension = true
       end
     end
+
+    if extension then
+      table.insert(extensions, filename)
+    elseif base_query == nil then
+      base_query = filename
+    end
+    io.close(file)
   end
 
   local query_files = {}
@@ -83,7 +130,8 @@ function M.get_query_files(lang, query_name, is_included)
     local base_files = M.get_query_files(base_lang, query_name, true)
     vim.list_extend(query_files, base_files)
   end
-  vim.list_extend(query_files, lang_files)
+  vim.list_extend(query_files, { base_query })
+  vim.list_extend(query_files, extensions)
 
   return query_files
 end
@@ -109,24 +157,24 @@ local explicit_queries = setmetatable({}, {
   end,
 })
 
---- Sets the runtime query {query_name} for {lang}
+--- Sets the runtime query named {query_name} for {lang}
 ---
 --- This allows users to override any runtime files and/or configuration
 --- set by plugins.
 ---
----@param lang string: The language to use for the query
----@param query_name string: The name of the query (i.e. "highlights")
----@param text string: The query text (unparsed).
+---@param lang string Language to use for the query
+---@param query_name string Name of the query (e.g., "highlights")
+---@param text string Query text (unparsed).
 function M.set_query(lang, query_name, text)
   explicit_queries[lang][query_name] = M.parse_query(lang, text)
 end
 
 --- Returns the runtime query {query_name} for {lang}.
 ---
----@param lang The language to use for the query
----@param query_name The name of the query (i.e. "highlights")
+---@param lang string Language to use for the query
+---@param query_name string Name of the query (e.g. "highlights")
 ---
----@return The corresponding query, parsed.
+---@return Query Parsed query
 function M.get_query(lang, query_name)
   if explicit_queries[lang][query_name] then
     return explicit_queries[lang][query_name]
@@ -140,12 +188,9 @@ function M.get_query(lang, query_name)
   end
 end
 
-local query_cache = setmetatable({}, {
-  __index = function(tbl, key)
-    rawset(tbl, key, {})
-    return rawget(tbl, key)
-  end,
-})
+local query_cache = vim.defaulttable(function()
+  return setmetatable({}, { __mode = 'v' })
+end)
 
 --- Parse {query} as a string. (If the query is in a file, the caller
 --- should read the contents into a string before calling).
@@ -160,10 +205,10 @@ local query_cache = setmetatable({}, {
 ---   -` info.captures` also points to `captures`.
 ---   - `info.patterns` contains information about predicates.
 ---
----@param lang string The language
----@param query string A string containing the query (s-expr syntax)
+---@param lang string Language to use for the query
+---@param query string Query in s-expr syntax
 ---
----@returns The query
+---@return Query Parsed query
 function M.parse_query(lang, query)
   language.require_language(lang)
   local cached = query_cache[lang][query]
@@ -181,9 +226,15 @@ end
 
 --- Gets the text corresponding to a given node
 ---
----@param node the node
----@param source The buffer or string from which the node is extracted
-function M.get_node_text(node, source)
+---@param node userdata |tsnode|
+---@param source (number|string) Buffer or string from which the {node} is extracted
+---@param opts (table|nil) Optional parameters.
+---          - concat: (boolean) Concatenate result in a string (default true)
+---@return (string[]|string)
+function M.get_node_text(node, source, opts)
+  opts = opts or {}
+  local concat = vim.F.if_nil(opts.concat, true)
+
   local start_row, start_col, start_byte = node:start()
   local end_row, end_col, end_byte = node:end_()
 
@@ -210,7 +261,7 @@ function M.get_node_text(node, source)
       end
     end
 
-    return table.concat(lines, '\n')
+    return concat and table.concat(lines, '\n') or lines
   elseif type(source) == 'string' then
     return source:sub(start_byte + 1, end_byte)
   end
@@ -367,9 +418,9 @@ local directive_handlers = {
 
 --- Adds a new predicate to be used in queries
 ---
----@param name the name of the predicate, without leading #
----@param handler the handler function to be used
----      signature will be (match, pattern, bufnr, predicate)
+---@param name string Name of the predicate, without leading #
+---@param handler function(match:table, pattern:string, bufnr:number, predicate:string[])
+---   - see |vim.treesitter.query.add_directive()| for argument meanings
 function M.add_predicate(name, handler, force)
   if predicate_handlers[name] and not force then
     error(string.format('Overriding %s', name))
@@ -385,9 +436,13 @@ end
 --- can set node level data by using the capture id on the
 --- metadata table `metadata[capture_id].key = value`
 ---
----@param name the name of the directive, without leading #
----@param handler the handler function to be used
----      signature will be (match, pattern, bufnr, predicate, metadata)
+---@param name string Name of the directive, without leading #
+---@param handler function(match:table, pattern:string, bufnr:number, predicate:string[], metadata:table)
+---   - match: see |treesitter-query|
+---      - node-level data are accessible via `match[capture_id]`
+---   - pattern: see |treesitter-query|
+---   - predicate: list of strings containing the full directive being called, e.g.
+---     `(node (#set! conceal "-"))` would get the predicate `{ "#set!", "conceal", "-" }`
 function M.add_directive(name, handler, force)
   if directive_handlers[name] and not force then
     error(string.format('Overriding %s', name))
@@ -397,12 +452,13 @@ function M.add_directive(name, handler, force)
 end
 
 --- Lists the currently available directives to use in queries.
----@return The list of supported directives.
+---@return string[] List of supported directives.
 function M.list_directives()
   return vim.tbl_keys(directive_handlers)
 end
 
----@return The list of supported predicates.
+--- Lists the currently available predicates to use in queries.
+---@return string[] List of supported predicates.
 function M.list_predicates()
   return vim.tbl_keys(predicate_handlers)
 end
@@ -489,18 +545,17 @@ end
 
 --- Iterate over all captures from all matches inside {node}
 ---
---- {source} is needed if the query contains predicates, then the caller
+--- {source} is needed if the query contains predicates; then the caller
 --- must ensure to use a freshly parsed tree consistent with the current
 --- text of the buffer (if relevant). {start_row} and {end_row} can be used to limit
 --- matches inside a row range (this is typically used with root node
---- as the node, i e to get syntax highlight matches in the current
---- viewport). When omitted the start and end row values are used from the given node.
+--- as the {node}, i.e., to get syntax highlight matches in the current
+--- viewport). When omitted, the {start} and {end} row values are used from the given node.
 ---
---- The iterator returns three values, a numeric id identifying the capture,
+--- The iterator returns three values: a numeric id identifying the capture,
 --- the captured node, and metadata from any directives processing the match.
 --- The following example shows how to get captures by name:
----
---- <pre>
+--- <pre>lua
 --- for id, node, metadata in query:iter_captures(tree:root(), bufnr, first, last) do
 ---   local name = query.captures[id] -- name of the capture in the query
 ---   -- typically useful info about the node:
@@ -510,13 +565,14 @@ end
 --- end
 --- </pre>
 ---
----@param node The node under which the search will occur
----@param source The source buffer or string to extract text from
----@param start The starting line of the search
----@param stop The stopping line of the search (end-exclusive)
+---@param node userdata |tsnode| under which the search will occur
+---@param source (number|string) Source buffer or string to extract text from
+---@param start number Starting line for the search
+---@param stop number Stopping line for the search (end-exclusive)
 ---
----@returns The matching capture id
----@returns The captured node
+---@return number capture Matching capture id
+---@return table capture_node Capture for {node}
+---@return table metadata for the {capture}
 function Query:iter_captures(node, source, start, stop)
   if type(source) == 'number' and source == 0 then
     source = vim.api.nvim_get_current_buf()
@@ -546,15 +602,14 @@ end
 
 --- Iterates the matches of self on a given range.
 ---
---- Iterate over all matches within a node. The arguments are the same as
---- for |query:iter_captures()| but the iterated values are different:
+--- Iterate over all matches within a {node}. The arguments are the same as
+--- for |Query:iter_captures()| but the iterated values are different:
 --- an (1-based) index of the pattern in the query, a table mapping
 --- capture indices to nodes, and metadata from any directives processing the match.
---- If the query has more than one pattern the capture table might be sparse,
+--- If the query has more than one pattern, the capture table might be sparse
 --- and e.g. `pairs()` method should be used over `ipairs`.
---- Here an example iterating over all captures in every match:
----
---- <pre>
+--- Here is an example iterating over all captures in every match:
+--- <pre>lua
 --- for pattern, match, metadata in cquery:iter_matches(tree:root(), bufnr, first, last) do
 ---   for id, node in pairs(match) do
 ---     local name = query.captures[id]
@@ -567,13 +622,14 @@ end
 --- end
 --- </pre>
 ---
----@param node The node under which the search will occur
----@param source The source buffer or string to search
----@param start The starting line of the search
----@param stop The stopping line of the search (end-exclusive)
+---@param node userdata |tsnode| under which the search will occur
+---@param source (number|string) Source buffer or string to search
+---@param start number Starting line for the search
+---@param stop number Stopping line for the search (end-exclusive)
 ---
----@returns The matching pattern id
----@returns The matching match
+---@return number pattern id
+---@return table match
+---@return table metadata
 function Query:iter_matches(node, source, start, stop)
   if type(source) == 'number' and source == 0 then
     source = vim.api.nvim_get_current_buf()
diff --git a/runtime/lua/vim/ui.lua b/runtime/lua/vim/ui.lua
index 6f1ce3089d..8f5be15221 100644
--- a/runtime/lua/vim/ui.lua
+++ b/runtime/lua/vim/ui.lua
@@ -21,7 +21,7 @@ local M = {}
 ---
 ---
 --- Example:
---- <pre>
+--- <pre>lua
 --- vim.ui.select({ 'tabs', 'spaces' }, {
 ---     prompt = 'Select tabs or spaces:',
 ---     format_item = function(item)
@@ -73,11 +73,12 @@ end
 ---               user inputs.
 ---@param on_confirm function ((input|nil) -> ())
 ---               Called once the user confirms or abort the input.
----               `input` is what the user typed.
+---               `input` is what the user typed (it might be
+---               an empty string if nothing was entered), or
 ---               `nil` if the user aborted the dialog.
 ---
 --- Example:
---- <pre>
+--- <pre>lua
 --- vim.ui.input({ prompt = 'Enter value for shiftwidth: ' }, function(input)
 ---     vim.o.shiftwidth = tonumber(input)
 --- end)
@@ -88,11 +89,17 @@ function M.input(opts, on_confirm)
   })
 
   opts = (opts and not vim.tbl_isempty(opts)) and opts or vim.empty_dict()
-  local input = vim.fn.input(opts)
-  if #input > 0 then
-    on_confirm(input)
-  else
+
+  -- Note that vim.fn.input({}) returns an empty string when cancelled.
+  -- vim.ui.input() should distinguish aborting from entering an empty string.
+  local _canceled = vim.NIL
+  opts = vim.tbl_extend('keep', opts, { cancelreturn = _canceled })
+
+  local ok, input = pcall(vim.fn.input, opts)
+  if not ok or input == _canceled then
     on_confirm(nil)
+  else
+    on_confirm(input)
   end
 end