Merge remote-tracking branch 'upstream/master' into usermarks

15 files changed, 1087 insertions, 670 deletions

diff --git a/runtime/lua/vim/diagnostic.lua b/runtime/lua/vim/diagnostic.lua
index 3f71d4f70d..db92085423 100644
--- a/runtime/lua/vim/diagnostic.lua
+++ b/runtime/lua/vim/diagnostic.lua
@@ -45,18 +45,24 @@ local bufnr_and_namespace_cacher_mt = {
   end,
 }
 
-local diagnostic_cache = setmetatable({}, {
-  __index = function(t, bufnr)
-    assert(bufnr > 0, 'Invalid buffer number')
-    vim.api.nvim_buf_attach(bufnr, false, {
-      on_detach = function()
-        rawset(t, bufnr, nil) -- clear cache
-      end,
-    })
-    t[bufnr] = {}
-    return t[bufnr]
-  end,
-})
+local diagnostic_cache
+do
+  local group = vim.api.nvim_create_augroup('DiagnosticBufDelete', {})
+  diagnostic_cache = setmetatable({}, {
+    __index = function(t, bufnr)
+      assert(bufnr > 0, 'Invalid buffer number')
+      vim.api.nvim_create_autocmd('BufDelete', {
+        group = group,
+        buffer = bufnr,
+        callback = function()
+          rawset(t, bufnr, nil)
+        end,
+      })
+      t[bufnr] = {}
+      return t[bufnr]
+    end,
+  })
+end
 
 local diagnostic_cache_extmarks = setmetatable({}, bufnr_and_namespace_cacher_mt)
 local diagnostic_attached_buffers = {}
diff --git a/runtime/lua/vim/filetype.lua b/runtime/lua/vim/filetype.lua
index 1b209e6a9d..fcd697a7c1 100644
--- a/runtime/lua/vim/filetype.lua
+++ b/runtime/lua/vim/filetype.lua
@@ -142,6 +142,7 @@ local extension = {
   asp = function(path, bufnr)
     return require('vim.filetype.detect').asp(bufnr)
   end,
+  astro = 'astro',
   atl = 'atlas',
   as = 'atlas',
   ahk = 'autohotkey',
@@ -421,9 +422,11 @@ local extension = {
   gdb = 'gdb',
   gdmo = 'gdmo',
   mo = 'gdmo',
-  tres = 'gdresource',
   tscn = 'gdresource',
+  tres = 'gdresource',
   gd = 'gdscript',
+  gdshader = 'gdshader',
+  shader = 'gdshader',
   ged = 'gedcom',
   gmi = 'gemtext',
   gemini = 'gemtext',
@@ -668,6 +671,21 @@ local extension = {
   moo = 'moo',
   moon = 'moonscript',
   mp = 'mp',
+  mpiv = function(path, bufnr)
+    return 'mp', function(b)
+      vim.b[b].mp_metafun = 1
+    end
+  end,
+  mpvi = function(path, bufnr)
+    return 'mp', function(b)
+      vim.b[b].mp_metafun = 1
+    end
+  end,
+  mpxl = function(path, bufnr)
+    return 'mp', function(b)
+      vim.b[b].mp_metafun = 1
+    end
+  end,
   mof = 'msidl',
   odl = 'msidl',
   msql = 'msql',
@@ -776,6 +794,7 @@ local extension = {
   ptl = 'python',
   ql = 'ql',
   qll = 'ql',
+  qmd = 'quarto',
   R = function(path, bufnr)
     return require('vim.filetype.detect').r(bufnr)
   end,
@@ -994,6 +1013,11 @@ local extension = {
   dsm = 'vb',
   ctl = 'vb',
   vbs = 'vb',
+  vdmpp = 'vdmpp',
+  vpp = 'vdmpp',
+  vdmrt = 'vdmrt',
+  vdmsl = 'vdmsl',
+  vdm = 'vdmsl',
   vr = 'vera',
   vri = 'vera',
   vrh = 'vera',
@@ -2472,7 +2496,15 @@ function M.match(args)
 
   -- Finally, check file contents
   if contents or bufnr then
-    contents = contents or M.getlines(bufnr)
+    if contents == nil then
+      if api.nvim_buf_line_count(bufnr) > 101 then
+        -- only need first 100 and last line for current checks
+        contents = M.getlines(bufnr, 1, 100)
+        contents[#contents + 1] = M.getlines(bufnr, -1)
+      else
+        contents = M.getlines(bufnr)
+      end
+    end
     -- If name is nil, catch any errors from the contents filetype detection function.
     -- If the function tries to use the filename that is nil then it will fail,
     -- but this enables checks which do not need a filename to still work.
diff --git a/runtime/lua/vim/filetype/detect.lua b/runtime/lua/vim/filetype/detect.lua
index 14f076717f..2be9dcff88 100644
--- a/runtime/lua/vim/filetype/detect.lua
+++ b/runtime/lua/vim/filetype/detect.lua
@@ -930,7 +930,7 @@ function M.progress_pascal(bufnr)
   return 'progress'
 end
 
--- Distinguish between "default" and Cproto prototype file.
+-- Distinguish between "default", Prolog and Cproto prototype file.
 function M.proto(bufnr, default)
   -- Cproto files have a comment in the first line and a function prototype in
   -- the second line, it always ends in ";".  Indent files may also have
@@ -940,7 +940,18 @@ function M.proto(bufnr, default)
   if getlines(bufnr, 2):find('.;$') then
     return 'cpp'
   else
-    return default
+    -- Recognize Prolog by specific text in the first non-empty line;
+    -- require a blank after the '%' because Perl uses "%list" and "%translate"
+    local line = nextnonblank(bufnr, 1)
+    if
+      line and line:find(':%-')
+      or matchregex(line, [[\c\<prolog\>]])
+      or findany(line, { '^%s*%%+%s', '^%s*%%+$', '^%s*/%*' })
+    then
+      return 'prolog'
+    else
+      return default
+    end
   end
 end
 
diff --git a/runtime/lua/vim/inspect.lua b/runtime/lua/vim/inspect.lua
index 0a53fb203b..c232f69590 100644
--- a/runtime/lua/vim/inspect.lua
+++ b/runtime/lua/vim/inspect.lua
@@ -89,8 +89,38 @@ local function escape(str)
   )
 end
 
+-- List of lua keywords
+local luaKeywords = {
+  ['and'] = true,
+  ['break'] = true,
+  ['do'] = true,
+  ['else'] = true,
+  ['elseif'] = true,
+  ['end'] = true,
+  ['false'] = true,
+  ['for'] = true,
+  ['function'] = true,
+  ['goto'] = true,
+  ['if'] = true,
+  ['in'] = true,
+  ['local'] = true,
+  ['nil'] = true,
+  ['not'] = true,
+  ['or'] = true,
+  ['repeat'] = true,
+  ['return'] = true,
+  ['then'] = true,
+  ['true'] = true,
+  ['until'] = true,
+  ['while'] = true,
+}
+
 local function isIdentifier(str)
-  return type(str) == 'string' and not not str:match('^[_%a][_%a%d]*$')
+  return type(str) == 'string'
+    -- identifier must start with a letter and underscore, and be followed by letters, numbers, and underscores
+    and not not str:match('^[_%a][_%a%d]*$')
+    -- lua keywords are not valid identifiers
+    and not luaKeywords[str]
 end
 
 local flr = math.floor
diff --git a/runtime/lua/vim/keymap.lua b/runtime/lua/vim/keymap.lua
index 7265beb56b..219de16b5c 100644
--- a/runtime/lua/vim/keymap.lua
+++ b/runtime/lua/vim/keymap.lua
@@ -36,14 +36,17 @@ local keymap = {}
 ---@param lhs string           Left-hand side |{lhs}| of the mapping.
 ---@param rhs string|function  Right-hand side |{rhs}| of the mapping. Can also be a Lua function.
 --
----@param opts table A table of |:map-arguments| such as "silent". In addition to the options
----                  listed in |nvim_set_keymap()|, this table also accepts the following keys:
----                  - buffer: (number or boolean) Add a mapping to the given buffer. When "true"
----                    or 0, use the current buffer.
----                  - remap: (boolean) Make the mapping recursive. This is the
----                  inverse of the "noremap" option from |nvim_set_keymap()|.
----                  Default `false`.
----                  - replace_keycodes: (boolean) defaults to true if "expr" is true.
+---@param opts table A table of |:map-arguments|.
+---                  + Accepts options accepted by the {opts} parameter in |nvim_set_keymap()|,
+---                    with the following notable differences:
+---                    - replace_keycodes: Defaults to `true` if "expr" is `true`.
+---                    - noremap: Always overridden with the inverse of "remap" (see below).
+---                  + In addition to those options, the table accepts the following keys:
+---                    - buffer: (number or boolean) Add a mapping to the given buffer.
+---                    When `0` or `true`, use the current buffer.
+---                    - remap: (boolean) Make the mapping recursive.
+---                    This is the inverse of the "noremap" option from |nvim_set_keymap()|.
+---                    Defaults to `false`.
 ---@see |nvim_set_keymap()|
 function keymap.set(mode, lhs, rhs, opts)
   vim.validate({
diff --git a/runtime/lua/vim/lsp.lua b/runtime/lua/vim/lsp.lua
index bf2201d9c8..1dc1a045fd 100644
--- a/runtime/lua/vim/lsp.lua
+++ b/runtime/lua/vim/lsp.lua
@@ -289,7 +289,12 @@ local function validate_client_config(config)
     'flags.debounce_text_changes must be a number with the debounce time in milliseconds'
   )
 
-  local cmd, cmd_args = lsp._cmd_parts(config.cmd)
+  local cmd, cmd_args
+  if type(config.cmd) == 'function' then
+    cmd = config.cmd
+  else
+    cmd, cmd_args = lsp._cmd_parts(config.cmd)
+  end
   local offset_encoding = valid_encodings.UTF16
   if config.offset_encoding then
     offset_encoding = validate_encoding(config.offset_encoding)
@@ -338,54 +343,166 @@ end
 
 local changetracking = {}
 do
-  --@private
-  --- client_id → state
+  ---@private
+  ---
+  --- LSP has 3 different sync modes:
+  ---   - None (Servers will read the files themselves when needed)
+  ---   - Full (Client sends the full buffer content on updates)
+  ---   - Incremental (Client sends only the changed parts)
   ---
-  ---   state
-  ---     use_incremental_sync: bool
-  ---     buffers: bufnr -> buffer_state
+  --- Changes are tracked per buffer.
+  --- A buffer can have multiple clients attached and each client needs to send the changes
+  --- To minimize the amount of changesets to compute, computation is grouped:
   ---
-  ---   buffer_state
-  ---     pending_change?: function that the timer starts to trigger didChange
-  ---     pending_changes: table (uri -> list of pending changeset tables));
-  ---                      Only set if incremental_sync is used
+  ---   None: One group for all clients
+  ---   Full: One group for all clients
+  ---   Incremental: One group per `offset_encoding`
   ---
-  ---     timer?: uv_timer
-  ---     lines: table
-  local state_by_client = {}
+  --- Sending changes can be debounced per buffer. To simplify the implementation the
+  --- smallest debounce interval is used and we don't group clients by different intervals.
+  ---
+  --- @class CTGroup
+  --- @field sync_kind number TextDocumentSyncKind, considers config.flags.allow_incremental_sync
+  --- @field offset_encoding "utf-8"|"utf-16"|"utf-32"
+  ---
+  --- @class CTBufferState
+  --- @field name string name of the buffer
+  --- @field lines string[] snapshot of buffer lines from last didChange
+  --- @field lines_tmp string[]
+  --- @field pending_changes table[] List of debounced changes in incremental sync mode
+  --- @field timer nil|userdata uv_timer
+  --- @field last_flush nil|number uv.hrtime of the last flush/didChange-notification
+  --- @field needs_flush boolean true if buffer updates haven't been sent to clients/servers yet
+  --- @field refs number how many clients are using this group
+  ---
+  --- @class CTGroupState
+  --- @field buffers table<number, CTBufferState>
+  --- @field debounce number debounce duration in ms
+  --- @field clients table<number, table> clients using this state. {client_id, client}
 
   ---@private
-  function changetracking.init(client, bufnr)
-    local use_incremental_sync = (
-      if_nil(client.config.flags.allow_incremental_sync, true)
-      and vim.tbl_get(client.server_capabilities, 'textDocumentSync', 'change')
-        == protocol.TextDocumentSyncKind.Incremental
+  ---@param group CTGroup
+  ---@return string
+  local function group_key(group)
+    if group.sync_kind == protocol.TextDocumentSyncKind.Incremental then
+      return tostring(group.sync_kind) .. '\0' .. group.offset_encoding
+    end
+    return tostring(group.sync_kind)
+  end
+
+  ---@private
+  ---@type table<CTGroup, CTGroupState>
+  local state_by_group = setmetatable({}, {
+    __index = function(tbl, k)
+      return rawget(tbl, group_key(k))
+    end,
+    __newindex = function(tbl, k, v)
+      rawset(tbl, group_key(k), v)
+    end,
+  })
+
+  ---@private
+  ---@return CTGroup
+  local function get_group(client)
+    local allow_inc_sync = if_nil(client.config.flags.allow_incremental_sync, true)
+    local change_capability =
+      vim.tbl_get(client.server_capabilities or {}, 'textDocumentSync', 'change')
+    local sync_kind = change_capability or protocol.TextDocumentSyncKind.None
+    if not allow_inc_sync and change_capability == protocol.TextDocumentSyncKind.Incremental then
+      sync_kind = protocol.TextDocumentSyncKind.Full
+    end
+    return {
+      sync_kind = sync_kind,
+      offset_encoding = client.offset_encoding,
+    }
+  end
+
+  ---@private
+  ---@param state CTBufferState
+  local function incremental_changes(state, encoding, bufnr, firstline, lastline, new_lastline)
+    local prev_lines = state.lines
+    local curr_lines = state.lines_tmp
+
+    local changed_lines = nvim_buf_get_lines(bufnr, firstline, new_lastline, true)
+    for i = 1, firstline do
+      curr_lines[i] = prev_lines[i]
+    end
+    for i = firstline + 1, new_lastline do
+      curr_lines[i] = changed_lines[i - firstline]
+    end
+    for i = lastline + 1, #prev_lines do
+      curr_lines[i - lastline + new_lastline] = prev_lines[i]
+    end
+    if tbl_isempty(curr_lines) then
+      -- Can happen when deleting the entire contents of a buffer, see https://github.com/neovim/neovim/issues/16259.
+      curr_lines[1] = ''
+    end
+
+    local line_ending = buf_get_line_ending(bufnr)
+    local incremental_change = sync.compute_diff(
+      state.lines,
+      curr_lines,
+      firstline,
+      lastline,
+      new_lastline,
+      encoding,
+      line_ending
     )
-    local state = state_by_client[client.id]
-    if not state then
+
+    -- Double-buffering of lines tables is used to reduce the load on the garbage collector.
+    -- At this point the prev_lines table is useless, but its internal storage has already been allocated,
+    -- so let's keep it around for the next didChange event, in which it will become the next
+    -- curr_lines table. Note that setting elements to nil doesn't actually deallocate slots in the
+    -- internal storage - it merely marks them as free, for the GC to deallocate them.
+    for i in ipairs(prev_lines) do
+      prev_lines[i] = nil
+    end
+    state.lines = curr_lines
+    state.lines_tmp = prev_lines
+
+    return incremental_change
+  end
+
+  ---@private
+  function changetracking.init(client, bufnr)
+    assert(client.offset_encoding, 'lsp client must have an offset_encoding')
+    local group = get_group(client)
+    local state = state_by_group[group]
+    if state then
+      state.debounce = math.min(state.debounce, client.config.flags.debounce_text_changes or 150)
+      state.clients[client.id] = client
+    else
       state = {
         buffers = {},
         debounce = client.config.flags.debounce_text_changes or 150,
-        use_incremental_sync = use_incremental_sync,
+        clients = {
+          [client.id] = client,
+        },
       }
-      state_by_client[client.id] = state
+      state_by_group[group] = state
     end
-    if not state.buffers[bufnr] then
-      local buf_state = {
+    local buf_state = state.buffers[bufnr]
+    if buf_state then
+      buf_state.refs = buf_state.refs + 1
+    else
+      buf_state = {
         name = api.nvim_buf_get_name(bufnr),
+        lines = {},
+        lines_tmp = {},
+        pending_changes = {},
+        needs_flush = false,
+        refs = 1,
       }
       state.buffers[bufnr] = buf_state
-      if use_incremental_sync then
+      if group.sync_kind == protocol.TextDocumentSyncKind.Incremental then
         buf_state.lines = nvim_buf_get_lines(bufnr, 0, -1, true)
-        buf_state.lines_tmp = {}
-        buf_state.pending_changes = {}
       end
     end
   end
 
   ---@private
   function changetracking._get_and_set_name(client, bufnr, name)
-    local state = state_by_client[client.id] or {}
+    local state = state_by_group[get_group(client)] or {}
     local buf_state = (state.buffers or {})[bufnr]
     local old_name = buf_state.name
     buf_state.name = name
@@ -395,32 +512,33 @@ do
   ---@private
   function changetracking.reset_buf(client, bufnr)
     changetracking.flush(client, bufnr)
-    local state = state_by_client[client.id]
-    if state and state.buffers then
-      local buf_state = state.buffers[bufnr]
+    local state = state_by_group[get_group(client)]
+    if not state then
+      return
+    end
+    assert(state.buffers, 'CTGroupState must have buffers')
+    local buf_state = state.buffers[bufnr]
+    buf_state.refs = buf_state.refs - 1
+    assert(buf_state.refs >= 0, 'refcount on buffer state must not get negative')
+    if buf_state.refs == 0 then
       state.buffers[bufnr] = nil
-      if buf_state and buf_state.timer then
-        buf_state.timer:stop()
-        buf_state.timer:close()
-        buf_state.timer = nil
-      end
+      changetracking._reset_timer(buf_state)
     end
   end
 
   ---@private
-  function changetracking.reset(client_id)
-    local state = state_by_client[client_id]
+  function changetracking.reset(client)
+    local state = state_by_group[get_group(client)]
     if not state then
       return
     end
-    for _, buf_state in pairs(state.buffers) do
-      if buf_state.timer then
-        buf_state.timer:stop()
-        buf_state.timer:close()
-        buf_state.timer = nil
+    state.clients[client.id] = nil
+    if vim.tbl_count(state.clients) == 0 then
+      for _, buf_state in pairs(state.buffers) do
+        changetracking._reset_timer(buf_state)
       end
+      state.buffers = {}
     end
-    state.buffers = {}
   end
 
   ---@private
@@ -430,6 +548,10 @@ do
   -- debounce can be skipped and otherwise maybe reduced.
   --
   -- This turns the debounce into a kind of client rate limiting
+  --
+  ---@param debounce number
+  ---@param buf_state CTBufferState
+  ---@return number
   local function next_debounce(debounce, buf_state)
     if debounce == 0 then
       return 0
@@ -444,83 +566,36 @@ do
   end
 
   ---@private
-  function changetracking.prepare(bufnr, firstline, lastline, new_lastline)
-    local incremental_changes = function(client, buf_state)
-      local prev_lines = buf_state.lines
-      local curr_lines = buf_state.lines_tmp
-
-      local changed_lines = nvim_buf_get_lines(bufnr, firstline, new_lastline, true)
-      for i = 1, firstline do
-        curr_lines[i] = prev_lines[i]
-      end
-      for i = firstline + 1, new_lastline do
-        curr_lines[i] = changed_lines[i - firstline]
-      end
-      for i = lastline + 1, #prev_lines do
-        curr_lines[i - lastline + new_lastline] = prev_lines[i]
-      end
-      if tbl_isempty(curr_lines) then
-        -- Can happen when deleting the entire contents of a buffer, see https://github.com/neovim/neovim/issues/16259.
-        curr_lines[1] = ''
-      end
-
-      local line_ending = buf_get_line_ending(bufnr)
-      local incremental_change = sync.compute_diff(
-        buf_state.lines,
-        curr_lines,
-        firstline,
-        lastline,
-        new_lastline,
-        client.offset_encoding or 'utf-16',
-        line_ending
-      )
-
-      -- Double-buffering of lines tables is used to reduce the load on the garbage collector.
-      -- At this point the prev_lines table is useless, but its internal storage has already been allocated,
-      -- so let's keep it around for the next didChange event, in which it will become the next
-      -- curr_lines table. Note that setting elements to nil doesn't actually deallocate slots in the
-      -- internal storage - it merely marks them as free, for the GC to deallocate them.
-      for i in ipairs(prev_lines) do
-        prev_lines[i] = nil
-      end
-      buf_state.lines = curr_lines
-      buf_state.lines_tmp = prev_lines
+  ---@param bufnr number
+  ---@param sync_kind number protocol.TextDocumentSyncKind
+  ---@param state CTGroupState
+  ---@param buf_state CTBufferState
+  local function send_changes(bufnr, sync_kind, state, buf_state)
+    if not buf_state.needs_flush then
+      return
+    end
+    buf_state.last_flush = uv.hrtime()
+    buf_state.needs_flush = false
 
-      return incremental_change
+    if not api.nvim_buf_is_valid(bufnr) then
+      buf_state.pending_changes = {}
+      return
     end
-    local full_changes = once(function()
-      return {
-        text = buf_get_full_text(bufnr),
+
+    local changes
+    if sync_kind == protocol.TextDocumentSyncKind.None then
+      return
+    elseif sync_kind == protocol.TextDocumentSyncKind.Incremental then
+      changes = buf_state.pending_changes
+      buf_state.pending_changes = {}
+    else
+      changes = {
+        { text = buf_get_full_text(bufnr) },
       }
-    end)
+    end
     local uri = vim.uri_from_bufnr(bufnr)
-    return function(client)
-      if
-        vim.tbl_get(client.server_capabilities, 'textDocumentSync', 'change')
-        == protocol.TextDocumentSyncKind.None
-      then
-        return
-      end
-      local state = state_by_client[client.id]
-      local buf_state = state.buffers[bufnr]
-      changetracking._reset_timer(buf_state)
-      local debounce = next_debounce(state.debounce, buf_state)
-      if state.use_incremental_sync then
-        -- This must be done immediately and cannot be delayed
-        -- The contents would further change and startline/endline may no longer fit
-        table.insert(buf_state.pending_changes, incremental_changes(client, buf_state))
-      end
-      buf_state.pending_change = function()
-        if buf_state.pending_change == nil then
-          return
-        end
-        buf_state.pending_change = nil
-        buf_state.last_flush = uv.hrtime()
-        if client.is_stopped() or not api.nvim_buf_is_valid(bufnr) then
-          return
-        end
-        local changes = state.use_incremental_sync and buf_state.pending_changes
-          or { full_changes() }
+    for _, client in pairs(state.clients) do
+      if not client.is_stopped() and lsp.buf_is_attached(bufnr, client.id) then
         client.notify('textDocument/didChange', {
           textDocument = {
             uri = uri,
@@ -528,46 +603,90 @@ do
           },
           contentChanges = changes,
         })
-        buf_state.pending_changes = {}
+      end
+    end
+  end
+
+  ---@private
+  function changetracking.send_changes(bufnr, firstline, lastline, new_lastline)
+    local groups = {}
+    for _, client in pairs(lsp.get_active_clients({ bufnr = bufnr })) do
+      local group = get_group(client)
+      groups[group_key(group)] = group
+    end
+    for _, group in pairs(groups) do
+      local state = state_by_group[group]
+      if not state then
+        error(
+          string.format(
+            'changetracking.init must have been called for all LSP clients. group=%s states=%s',
+            vim.inspect(group),
+            vim.inspect(vim.tbl_keys(state_by_group))
+          )
+        )
+      end
+      local buf_state = state.buffers[bufnr]
+      buf_state.needs_flush = true
+      changetracking._reset_timer(buf_state)
+      local debounce = next_debounce(state.debounce, buf_state)
+      if group.sync_kind == protocol.TextDocumentSyncKind.Incremental then
+        -- This must be done immediately and cannot be delayed
+        -- The contents would further change and startline/endline may no longer fit
+        local changes = incremental_changes(
+          buf_state,
+          group.offset_encoding,
+          bufnr,
+          firstline,
+          lastline,
+          new_lastline
+        )
+        table.insert(buf_state.pending_changes, changes)
       end
       if debounce == 0 then
-        buf_state.pending_change()
+        send_changes(bufnr, group.sync_kind, state, buf_state)
       else
         local timer = uv.new_timer()
         buf_state.timer = timer
-        -- Must use schedule_wrap because `full_changes()` calls nvim_buf_get_lines
-        timer:start(debounce, 0, vim.schedule_wrap(buf_state.pending_change))
+        timer:start(
+          debounce,
+          0,
+          vim.schedule_wrap(function()
+            changetracking._reset_timer(buf_state)
+            send_changes(bufnr, group.sync_kind, state, buf_state)
+          end)
+        )
       end
     end
   end
 
+  ---@private
   function changetracking._reset_timer(buf_state)
-    if buf_state.timer then
-      buf_state.timer:stop()
-      buf_state.timer:close()
+    local timer = buf_state.timer
+    if timer then
       buf_state.timer = nil
+      if not timer:is_closing() then
+        timer:stop()
+        timer:close()
+      end
     end
   end
 
   --- Flushes any outstanding change notification.
   ---@private
   function changetracking.flush(client, bufnr)
-    local state = state_by_client[client.id]
+    local group = get_group(client)
+    local state = state_by_group[group]
     if not state then
       return
     end
     if bufnr then
       local buf_state = state.buffers[bufnr] or {}
       changetracking._reset_timer(buf_state)
-      if buf_state.pending_change then
-        buf_state.pending_change()
-      end
+      send_changes(bufnr, group.sync_kind, state, buf_state)
     else
-      for _, buf_state in pairs(state.buffers) do
+      for buf, buf_state in pairs(state.buffers) do
         changetracking._reset_timer(buf_state)
-        if buf_state.pending_change then
-          buf_state.pending_change()
-        end
+        send_changes(buf, group.sync_kind, state, buf_state)
       end
     end
   end
@@ -577,7 +696,7 @@ end
 --- Default handler for the 'textDocument/didOpen' LSP notification.
 ---
 ---@param bufnr number Number of the buffer, or 0 for current
----@param client Client object
+---@param client table Client object
 local function text_document_did_open_handler(bufnr, client)
   changetracking.init(client, bufnr)
   if not vim.tbl_get(client.server_capabilities, 'textDocumentSync', 'openClose') then
@@ -741,14 +860,17 @@ end
 ---                            Used on all running clients.
 ---                            The default implementation re-uses a client if name
 ---                            and root_dir matches.
----@return number client_id
+---@return number|nil client_id
 function lsp.start(config, opts)
   opts = opts or {}
   local reuse_client = opts.reuse_client
     or function(client, conf)
       return client.config.root_dir == conf.root_dir and client.name == conf.name
     end
-  config.name = config.name or (config.cmd[1] and vim.fs.basename(config.cmd[1])) or nil
+  config.name = config.name
+  if not config.name and type(config.cmd) == 'table' then
+    config.name = config.cmd[1] and vim.fs.basename(config.cmd[1]) or nil
+  end
   local bufnr = api.nvim_get_current_buf()
   for _, clients in ipairs({ uninitialized_clients, lsp.get_active_clients() }) do
     for _, client in pairs(clients) do
@@ -779,8 +901,13 @@ end
 --- The following parameters describe fields in the {config} table.
 ---
 ---
----@param cmd: (required, string or list treated like |jobstart()|) Base command
---- that initiates the LSP client.
+---@param cmd: (table|string|fun(dispatchers: table):table) command string or
+--- list treated like |jobstart|. The command must launch the language server
+--- process. `cmd` can also be a function that creates an RPC client.
+--- The function receives a dispatchers table and must return a table with the
+--- functions `request`, `notify`, `is_closing` and `terminate`
+--- See |vim.lsp.rpc.request| and |vim.lsp.rpc.notify|
+--- For TCP there is a built-in rpc client factory: |vim.lsp.rpc.connect|
 ---
 ---@param cmd_cwd: (string, default=|getcwd()|) Directory to launch
 --- the `cmd` process. Not related to `root_dir`.
@@ -871,7 +998,7 @@ end
 --- - debounce_text_changes (number, default 150): Debounce didChange
 ---       notifications to the server by the given number in milliseconds. No debounce
 ---       occurs if nil
---- - exit_timeout (number, default 500): Milliseconds to wait for server to
+--- - exit_timeout (number|boolean, default false): Milliseconds to wait for server to
 ---       exit cleanly after sending the 'shutdown' request before sending kill -15.
 ---       If set to false, nvim exits immediately after sending the 'shutdown' request to the server.
 ---
@@ -968,12 +1095,20 @@ function lsp.start_client(config)
 
   ---@private
   local function set_defaults(client, bufnr)
-    if client.server_capabilities.definitionProvider and vim.bo[bufnr].tagfunc == '' then
+    local capabilities = client.server_capabilities
+    if capabilities.definitionProvider and vim.bo[bufnr].tagfunc == '' then
       vim.bo[bufnr].tagfunc = 'v:lua.vim.lsp.tagfunc'
     end
-    if client.server_capabilities.completionProvider and vim.bo[bufnr].omnifunc == '' then
+    if capabilities.completionProvider and vim.bo[bufnr].omnifunc == '' then
       vim.bo[bufnr].omnifunc = 'v:lua.vim.lsp.omnifunc'
     end
+    if
+      capabilities.documentRangeFormattingProvider
+      and vim.bo[bufnr].formatprg == ''
+      and vim.bo[bufnr].formatexpr == ''
+    then
+      vim.bo[bufnr].formatexpr = 'v:lua.vim.lsp.formatexpr()'
+    end
   end
 
   ---@private
@@ -986,6 +1121,9 @@ function lsp.start_client(config)
     if vim.bo[bufnr].omnifunc == 'v:lua.vim.lsp.omnifunc' then
       vim.bo[bufnr].omnifunc = nil
     end
+    if vim.bo[bufnr].formatexpr == 'v:lua.vim.lsp.formatexpr()' then
+      vim.bo[bufnr].formatexpr = nil
+    end
   end
 
   ---@private
@@ -1019,11 +1157,16 @@ function lsp.start_client(config)
         end)
       end
     end
-
+    local client = active_clients[client_id] and active_clients[client_id]
+      or uninitialized_clients[client_id]
     active_clients[client_id] = nil
     uninitialized_clients[client_id] = nil
 
-    changetracking.reset(client_id)
+    -- Client can be absent if executable starts, but initialize fails
+    -- init/attach won't have happened
+    if client then
+      changetracking.reset(client)
+    end
     if code ~= 0 or (signal ~= 0 and signal ~= 15) then
       local msg =
         string.format('Client %s quit with exit code %s and signal %s', client_id, code, signal)
@@ -1034,11 +1177,16 @@ function lsp.start_client(config)
   end
 
   -- Start the RPC client.
-  local rpc = lsp_rpc.start(cmd, cmd_args, dispatch, {
-    cwd = config.cmd_cwd,
-    env = config.cmd_env,
-    detached = config.detached,
-  })
+  local rpc
+  if type(cmd) == 'function' then
+    rpc = cmd(dispatch)
+  else
+    rpc = lsp_rpc.start(cmd, cmd_args, dispatch, {
+      cwd = config.cmd_cwd,
+      env = config.cmd_env,
+      detached = config.detached,
+    })
+  end
 
   -- Return nil if client fails to start
   if not rpc then
@@ -1334,14 +1482,13 @@ function lsp.start_client(config)
   --- you request to stop a client which has previously been requested to
   --- shutdown, it will automatically escalate and force shutdown.
   ---
-  ---@param force (bool, optional)
+  ---@param force boolean|nil
   function client.stop(force)
-    local handle = rpc.handle
-    if handle:is_closing() then
+    if rpc.is_closing() then
       return
     end
     if force or not client.initialized or graceful_shutdown_failed then
-      handle:kill(15)
+      rpc.terminate()
       return
     end
     -- Sending a signal after a process has exited is acceptable.
@@ -1350,7 +1497,7 @@ function lsp.start_client(config)
         rpc.notify('exit')
       else
         -- If there was an error in the shutdown request, then term to be safe.
-        handle:kill(15)
+        rpc.terminate()
         graceful_shutdown_failed = true
       end
     end)
@@ -1362,7 +1509,7 @@ function lsp.start_client(config)
   ---@returns (bool) true if client is stopped or in the process of being
   ---stopped; false otherwise
   function client.is_stopped()
-    return rpc.handle:is_closing()
+    return rpc.is_closing()
   end
 
   ---@private
@@ -1403,9 +1550,7 @@ do
         return true
       end
       util.buf_versions[bufnr] = changedtick
-      local compute_change_and_notify =
-        changetracking.prepare(bufnr, firstline, lastline, new_lastline)
-      for_each_buffer_client(bufnr, compute_change_and_notify)
+      changetracking.send_changes(bufnr, firstline, lastline, new_lastline)
     end
 end
 
@@ -1681,7 +1826,7 @@ api.nvim_create_autocmd('VimLeavePre', {
     local send_kill = false
 
     for client_id, client in pairs(active_clients) do
-      local timeout = if_nil(client.config.flags.exit_timeout, 500)
+      local timeout = if_nil(client.config.flags.exit_timeout, false)
       if timeout then
         send_kill = true
         timeouts[client_id] = timeout
@@ -1717,13 +1862,14 @@ api.nvim_create_autocmd('VimLeavePre', {
   end,
 })
 
+---@private
 --- Sends an async request for all active clients attached to the
 --- buffer.
 ---
 ---@param bufnr (number) Buffer handle, or 0 for current.
 ---@param method (string) LSP method name
----@param params (optional, table) Parameters to send to the server
----@param handler (optional, function) See |lsp-handler|
+---@param params table|nil Parameters to send to the server
+---@param handler function|nil See |lsp-handler|
 ---       If nil, follows resolution strategy defined in |lsp-handler-configuration|
 ---
 ---@returns 2-tuple:
@@ -1982,29 +2128,32 @@ function lsp.formatexpr(opts)
     return 1
   end
 
-  local start_line = vim.v.lnum
-  local end_line = start_line + vim.v.count - 1
-
-  if start_line > 0 and end_line > 0 then
-    local params = {
-      textDocument = util.make_text_document_params(),
-      range = {
-        start = { line = start_line - 1, character = 0 },
-        ['end'] = { line = end_line - 1, character = 0 },
-      },
-    }
-    params.options = util.make_formatting_params().options
-    local client_results =
-      vim.lsp.buf_request_sync(0, 'textDocument/rangeFormatting', params, timeout_ms)
+  local start_lnum = vim.v.lnum
+  local end_lnum = start_lnum + vim.v.count - 1
 
-    -- Apply the text edits from one and only one of the clients.
-    for client_id, response in pairs(client_results) do
+  if start_lnum <= 0 or end_lnum <= 0 then
+    return 0
+  end
+  local bufnr = api.nvim_get_current_buf()
+  for _, client in pairs(lsp.get_active_clients({ bufnr = bufnr })) do
+    if client.supports_method('textDocument/rangeFormatting') then
+      local params = util.make_formatting_params()
+      local end_line = vim.fn.getline(end_lnum)
+      local end_col = util._str_utfindex_enc(end_line, nil, client.offset_encoding)
+      params.range = {
+        start = {
+          line = start_lnum - 1,
+          character = 0,
+        },
+        ['end'] = {
+          line = end_lnum - 1,
+          character = end_col,
+        },
+      }
+      local response =
+        client.request_sync('textDocument/rangeFormatting', params, timeout_ms, bufnr)
       if response.result then
-        vim.lsp.util.apply_text_edits(
-          response.result,
-          0,
-          vim.lsp.get_client_by_id(client_id).offset_encoding
-        )
+        vim.lsp.util.apply_text_edits(response.result, 0, client.offset_encoding)
         return 0
       end
     end
diff --git a/runtime/lua/vim/lsp/buf.lua b/runtime/lua/vim/lsp/buf.lua
index 63f4688d94..6a070928d9 100644
--- a/runtime/lua/vim/lsp/buf.lua
+++ b/runtime/lua/vim/lsp/buf.lua
@@ -61,7 +61,7 @@ end
 ---
 ---@param options table|nil additional options
 ---     - reuse_win: (boolean) Jump to existing window if buffer is already open.
----     - on_list: (function) handler for list results. See |on-list-handler|
+---     - on_list: (function) handler for list results. See |lsp-on-list-handler|
 function M.declaration(options)
   local params = util.make_position_params()
   request_with_options('textDocument/declaration', params, options)
@@ -71,7 +71,7 @@ end
 ---
 ---@param options table|nil additional options
 ---     - reuse_win: (boolean) Jump to existing window if buffer is already open.
----     - on_list: (function) handler for list results. See |on-list-handler|
+---     - on_list: (function) handler for list results. See |lsp-on-list-handler|
 function M.definition(options)
   local params = util.make_position_params()
   request_with_options('textDocument/definition', params, options)
@@ -81,7 +81,7 @@ end
 ---
 ---@param options table|nil additional options
 ---     - reuse_win: (boolean) Jump to existing window if buffer is already open.
----     - on_list: (function) handler for list results. See |on-list-handler|
+---     - on_list: (function) handler for list results. See |lsp-on-list-handler|
 function M.type_definition(options)
   local params = util.make_position_params()
   request_with_options('textDocument/typeDefinition', params, options)
@@ -91,7 +91,7 @@ end
 --- quickfix window.
 ---
 ---@param options table|nil additional options
----     - on_list: (function) handler for list results. See |on-list-handler|
+---     - on_list: (function) handler for list results. See |lsp-on-list-handler|
 function M.implementation(options)
   local params = util.make_position_params()
   request_with_options('textDocument/implementation', params, options)
@@ -503,7 +503,7 @@ end
 ---@param context (table) Context for the request
 ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_references
 ---@param options table|nil additional options
----     - on_list: (function) handler for list results. See |on-list-handler|
+---     - on_list: (function) handler for list results. See |lsp-on-list-handler|
 function M.references(context, options)
   validate({ context = { context, 't', true } })
   local params = util.make_position_params()
@@ -516,7 +516,7 @@ end
 --- Lists all symbols in the current buffer in the quickfix window.
 ---
 ---@param options table|nil additional options
----     - on_list: (function) handler for list results. See |on-list-handler|
+---     - on_list: (function) handler for list results. See |lsp-on-list-handler|
 function M.document_symbol(options)
   local params = { textDocument = util.make_text_document_params() }
   request_with_options('textDocument/documentSymbol', params, options)
@@ -659,7 +659,7 @@ end
 ---
 ---@param query (string, optional)
 ---@param options table|nil additional options
----     - on_list: (function) handler for list results. See |on-list-handler|
+---     - on_list: (function) handler for list results. See |lsp-on-list-handler|
 function M.workspace_symbol(query, options)
   query = query or npcall(vim.fn.input, 'Query: ')
   if query == nil then
diff --git a/runtime/lua/vim/lsp/rpc.lua b/runtime/lua/vim/lsp/rpc.lua
index 913eee19a2..70f838f34d 100644
--- a/runtime/lua/vim/lsp/rpc.lua
+++ b/runtime/lua/vim/lsp/rpc.lua
@@ -58,12 +58,10 @@ end
 ---@private
 --- Parses an LSP Message's header
 ---
----@param header: The header to parse.
----@returns Parsed headers
+---@param header string: The header to parse.
+---@return table parsed headers
 local function parse_headers(header)
-  if type(header) ~= 'string' then
-    return nil
-  end
+  assert(type(header) == 'string', 'header must be a string')
   local headers = {}
   for line in vim.gsplit(header, '\r\n', true) do
     if line == '' then
@@ -189,9 +187,9 @@ end
 
 --- Creates an RPC response object/table.
 ---
----@param code RPC error code defined in `vim.lsp.protocol.ErrorCodes`
----@param message (optional) arbitrary message to send to server
----@param data (optional) arbitrary data to send to server
+---@param code number RPC error code defined in `vim.lsp.protocol.ErrorCodes`
+---@param message string|nil arbitrary message to send to server
+---@param data any|nil arbitrary data to send to server
 local function rpc_response_error(code, message, data)
   -- TODO should this error or just pick a sane error (like InternalError)?
   local code_name = assert(protocol.ErrorCodes[code], 'Invalid RPC error code')
@@ -243,392 +241,515 @@ function default_dispatchers.on_error(code, err)
   local _ = log.error() and log.error('client_error:', client_errors[code], err)
 end
 
---- Starts an LSP server process and create an LSP RPC client object to
---- interact with it. Communication with the server is currently limited to stdio.
----
----@param cmd (string) Command to start the LSP server.
----@param cmd_args (table) List of additional string arguments to pass to {cmd}.
----@param dispatchers (table, optional) Dispatchers for LSP message types. Valid
----dispatcher names are:
---- - `"notification"`
---- - `"server_request"`
---- - `"on_error"`
---- - `"on_exit"`
----@param extra_spawn_params (table, optional) Additional context for the LSP
---- server process. May contain:
---- - {cwd} (string) Working directory for the LSP server process
---- - {env} (table) Additional environment variables for LSP server process
----@returns Client RPC object.
----
----@returns Methods:
---- - `notify()` |vim.lsp.rpc.notify()|
---- - `request()` |vim.lsp.rpc.request()|
+---@private
+local function create_read_loop(handle_body, on_no_chunk, on_error)
+  local parse_chunk = coroutine.wrap(request_parser_loop)
+  parse_chunk()
+  return function(err, chunk)
+    if err then
+      on_error(err)
+      return
+    end
+
+    if not chunk then
+      if on_no_chunk then
+        on_no_chunk()
+      end
+      return
+    end
+
+    while true do
+      local headers, body = parse_chunk(chunk)
+      if headers then
+        handle_body(body)
+        chunk = ''
+      else
+        break
+      end
+    end
+  end
+end
+
+---@class RpcClient
+---@field message_index number
+---@field message_callbacks table
+---@field notify_reply_callbacks table
+---@field transport table
+---@field dispatchers table
+
+---@class RpcClient
+local Client = {}
+
+---@private
+function Client:encode_and_send(payload)
+  local _ = log.debug() and log.debug('rpc.send', payload)
+  if self.transport.is_closing() then
+    return false
+  end
+  local encoded = vim.json.encode(payload)
+  self.transport.write(format_message_with_content_length(encoded))
+  return true
+end
+
+---@private
+--- Sends a notification to the LSP server.
+---@param method (string) The invoked LSP method
+---@param params (table|nil): Parameters for the invoked LSP method
+---@returns (bool) `true` if notification could be sent, `false` if not
+function Client:notify(method, params)
+  return self:encode_and_send({
+    jsonrpc = '2.0',
+    method = method,
+    params = params,
+  })
+end
+
+---@private
+--- sends an error object to the remote LSP process.
+function Client:send_response(request_id, err, result)
+  return self:encode_and_send({
+    id = request_id,
+    jsonrpc = '2.0',
+    error = err,
+    result = result,
+  })
+end
+
+---@private
+--- Sends a request to the LSP server and runs {callback} upon response.
 ---
----@returns Members:
---- - {pid} (number) The LSP server's PID.
---- - {handle} A handle for low-level interaction with the LSP server process
----   |vim.loop|.
-local function start(cmd, cmd_args, dispatchers, extra_spawn_params)
-  local _ = log.info()
-    and log.info('Starting RPC client', { cmd = cmd, args = cmd_args, extra = extra_spawn_params })
+---@param method (string) The invoked LSP method
+---@param params (table|nil) Parameters for the invoked LSP method
+---@param callback (function) Callback to invoke
+---@param notify_reply_callback (function|nil) Callback to invoke as soon as a request is no longer pending
+---@returns (bool, number) `(true, message_id)` if request could be sent, `false` if not
+function Client:request(method, params, callback, notify_reply_callback)
   validate({
-    cmd = { cmd, 's' },
-    cmd_args = { cmd_args, 't' },
-    dispatchers = { dispatchers, 't', true },
+    callback = { callback, 'f' },
+    notify_reply_callback = { notify_reply_callback, 'f', true },
+  })
+  self.message_index = self.message_index + 1
+  local message_id = self.message_index
+  local result = self:encode_and_send({
+    id = message_id,
+    jsonrpc = '2.0',
+    method = method,
+    params = params,
   })
+  local message_callbacks = self.message_callbacks
+  local notify_reply_callbacks = self.notify_reply_callbacks
+  if result then
+    if message_callbacks then
+      message_callbacks[message_id] = schedule_wrap(callback)
+    else
+      return false
+    end
+    if notify_reply_callback and notify_reply_callbacks then
+      notify_reply_callbacks[message_id] = schedule_wrap(notify_reply_callback)
+    end
+    return result, message_id
+  else
+    return false
+  end
+end
 
-  if extra_spawn_params and extra_spawn_params.cwd then
-    assert(is_dir(extra_spawn_params.cwd), 'cwd must be a directory')
+---@private
+function Client:on_error(errkind, ...)
+  assert(client_errors[errkind])
+  -- TODO what to do if this fails?
+  pcall(self.dispatchers.on_error, errkind, ...)
+end
+
+---@private
+function Client:pcall_handler(errkind, status, head, ...)
+  if not status then
+    self:on_error(errkind, head, ...)
+    return status, head
   end
-  if dispatchers then
-    local user_dispatchers = dispatchers
-    dispatchers = {}
-    for dispatch_name, default_dispatch in pairs(default_dispatchers) do
-      local user_dispatcher = user_dispatchers[dispatch_name]
-      if user_dispatcher then
-        if type(user_dispatcher) ~= 'function' then
-          error(string.format('dispatcher.%s must be a function', dispatch_name))
+  return status, head, ...
+end
+
+---@private
+function Client:try_call(errkind, fn, ...)
+  return self:pcall_handler(errkind, pcall(fn, ...))
+end
+
+-- TODO periodically check message_callbacks for old requests past a certain
+-- time and log them. This would require storing the timestamp. I could call
+-- them with an error then, perhaps.
+
+---@private
+function Client:handle_body(body)
+  local ok, decoded = pcall(vim.json.decode, body, { luanil = { object = true } })
+  if not ok then
+    self:on_error(client_errors.INVALID_SERVER_JSON, decoded)
+    return
+  end
+  local _ = log.debug() and log.debug('rpc.receive', decoded)
+
+  if type(decoded.method) == 'string' and decoded.id then
+    local err
+    -- Schedule here so that the users functions don't trigger an error and
+    -- we can still use the result.
+    schedule(function()
+      local status, result
+      status, result, err = self:try_call(
+        client_errors.SERVER_REQUEST_HANDLER_ERROR,
+        self.dispatchers.server_request,
+        decoded.method,
+        decoded.params
+      )
+      local _ = log.debug()
+        and log.debug(
+          'server_request: callback result',
+          { status = status, result = result, err = err }
+        )
+      if status then
+        if not (result or err) then
+          -- TODO this can be a problem if `null` is sent for result. needs vim.NIL
+          error(
+            string.format(
+              'method %q: either a result or an error must be sent to the server in response',
+              decoded.method
+            )
+          )
         end
-        -- server_request is wrapped elsewhere.
-        if
-          not (dispatch_name == 'server_request' or dispatch_name == 'on_exit') -- TODO this blocks the loop exiting for some reason.
-        then
-          user_dispatcher = schedule_wrap(user_dispatcher)
+        if err then
+          assert(
+            type(err) == 'table',
+            'err must be a table. Use rpc_response_error to help format errors.'
+          )
+          local code_name = assert(
+            protocol.ErrorCodes[err.code],
+            'Errors must use protocol.ErrorCodes. Use rpc_response_error to help format errors.'
+          )
+          err.message = err.message or code_name
         end
-        dispatchers[dispatch_name] = user_dispatcher
       else
-        dispatchers[dispatch_name] = default_dispatch
+        -- On an exception, result will contain the error message.
+        err = rpc_response_error(protocol.ErrorCodes.InternalError, result)
+        result = nil
       end
+      self:send_response(decoded.id, err, result)
+    end)
+    -- This works because we are expecting vim.NIL here
+  elseif decoded.id and (decoded.result ~= vim.NIL or decoded.error ~= vim.NIL) then
+    -- We sent a number, so we expect a number.
+    local result_id = assert(tonumber(decoded.id), 'response id must be a number')
+
+    -- Notify the user that a response was received for the request
+    local notify_reply_callbacks = self.notify_reply_callbacks
+    local notify_reply_callback = notify_reply_callbacks and notify_reply_callbacks[result_id]
+    if notify_reply_callback then
+      validate({
+        notify_reply_callback = { notify_reply_callback, 'f' },
+      })
+      notify_reply_callback(result_id)
+      notify_reply_callbacks[result_id] = nil
     end
-  else
-    dispatchers = default_dispatchers
-  end
 
-  local stdin = uv.new_pipe(false)
-  local stdout = uv.new_pipe(false)
-  local stderr = uv.new_pipe(false)
+    local message_callbacks = self.message_callbacks
 
-  local message_index = 0
-  local message_callbacks = {}
-  local notify_reply_callbacks = {}
+    -- Do not surface RequestCancelled to users, it is RPC-internal.
+    if decoded.error then
+      local mute_error = false
+      if decoded.error.code == protocol.ErrorCodes.RequestCancelled then
+        local _ = log.debug() and log.debug('Received cancellation ack', decoded)
+        mute_error = true
+      end
 
-  local handle, pid
-  do
-    ---@private
-    --- Callback for |vim.loop.spawn()| Closes all streams and runs the `on_exit` dispatcher.
-    ---@param code (number) Exit code
-    ---@param signal (number) Signal that was used to terminate (if any)
-    local function onexit(code, signal)
-      stdin:close()
-      stdout:close()
-      stderr:close()
-      handle:close()
-      -- Make sure that message_callbacks/notify_reply_callbacks can be gc'd.
-      message_callbacks = nil
-      notify_reply_callbacks = nil
-      dispatchers.on_exit(code, signal)
-    end
-    local spawn_params = {
-      args = cmd_args,
-      stdio = { stdin, stdout, stderr },
-      detached = not is_win,
-    }
-    if extra_spawn_params then
-      spawn_params.cwd = extra_spawn_params.cwd
-      spawn_params.env = env_merge(extra_spawn_params.env)
-      if extra_spawn_params.detached ~= nil then
-        spawn_params.detached = extra_spawn_params.detached
+      if mute_error then
+        -- Clear any callback since this is cancelled now.
+        -- This is safe to do assuming that these conditions hold:
+        -- - The server will not send a result callback after this cancellation.
+        -- - If the server sent this cancellation ACK after sending the result, the user of this RPC
+        -- client will ignore the result themselves.
+        if result_id and message_callbacks then
+          message_callbacks[result_id] = nil
+        end
+        return
       end
     end
-    handle, pid = uv.spawn(cmd, spawn_params, onexit)
-    if handle == nil then
-      local msg = string.format('Spawning language server with cmd: `%s` failed', cmd)
-      if string.match(pid, 'ENOENT') then
-        msg = msg
-          .. '. The language server is either not installed, missing from PATH, or not executable.'
-      else
-        msg = msg .. string.format(' with error message: %s', pid)
+
+    local callback = message_callbacks and message_callbacks[result_id]
+    if callback then
+      message_callbacks[result_id] = nil
+      validate({
+        callback = { callback, 'f' },
+      })
+      if decoded.error then
+        decoded.error = setmetatable(decoded.error, {
+          __tostring = format_rpc_error,
+        })
       end
-      vim.notify(msg, vim.log.levels.WARN)
-      return
+      self:try_call(
+        client_errors.SERVER_RESULT_CALLBACK_ERROR,
+        callback,
+        decoded.error,
+        decoded.result
+      )
+    else
+      self:on_error(client_errors.NO_RESULT_CALLBACK_FOUND, decoded)
+      local _ = log.error() and log.error('No callback found for server response id ' .. result_id)
     end
+  elseif type(decoded.method) == 'string' then
+    -- Notification
+    self:try_call(
+      client_errors.NOTIFICATION_HANDLER_ERROR,
+      self.dispatchers.notification,
+      decoded.method,
+      decoded.params
+    )
+  else
+    -- Invalid server message
+    self:on_error(client_errors.INVALID_SERVER_MESSAGE, decoded)
   end
+end
 
-  ---@private
-  --- Encodes {payload} into a JSON-RPC message and sends it to the remote
-  --- process.
-  ---
-  ---@param payload table
-  ---@returns true if the payload could be scheduled, false if the main event-loop is in the process of closing.
-  local function encode_and_send(payload)
-    local _ = log.debug() and log.debug('rpc.send', payload)
-    if handle == nil or handle:is_closing() then
-      return false
-    end
-    local encoded = vim.json.encode(payload)
-    stdin:write(format_message_with_content_length(encoded))
-    return true
-  end
+---@private
+---@return RpcClient
+local function new_client(dispatchers, transport)
+  local state = {
+    message_index = 0,
+    message_callbacks = {},
+    notify_reply_callbacks = {},
+    transport = transport,
+    dispatchers = dispatchers,
+  }
+  return setmetatable(state, { __index = Client })
+end
 
-  -- FIXME: DOC: Should be placed on the RPC client object returned by
-  -- `start()`
-  --
-  --- Sends a notification to the LSP server.
-  ---@param method (string) The invoked LSP method
-  ---@param params (table): Parameters for the invoked LSP method
-  ---@returns (bool) `true` if notification could be sent, `false` if not
-  local function notify(method, params)
-    return encode_and_send({
-      jsonrpc = '2.0',
-      method = method,
-      params = params,
-    })
+---@private
+---@param client RpcClient
+local function public_client(client)
+  local result = {}
+
+  ---@private
+  function result.is_closing()
+    return client.transport.is_closing()
   end
 
   ---@private
-  --- sends an error object to the remote LSP process.
-  local function send_response(request_id, err, result)
-    return encode_and_send({
-      id = request_id,
-      jsonrpc = '2.0',
-      error = err,
-      result = result,
-    })
+  function result.terminate()
+    client.transport.terminate()
   end
 
-  -- FIXME: DOC: Should be placed on the RPC client object returned by
-  -- `start()`
-  --
   --- Sends a request to the LSP server and runs {callback} upon response.
   ---
   ---@param method (string) The invoked LSP method
-  ---@param params (table) Parameters for the invoked LSP method
+  ---@param params (table|nil) Parameters for the invoked LSP method
   ---@param callback (function) Callback to invoke
   ---@param notify_reply_callback (function|nil) Callback to invoke as soon as a request is no longer pending
   ---@returns (bool, number) `(true, message_id)` if request could be sent, `false` if not
-  local function request(method, params, callback, notify_reply_callback)
-    validate({
-      callback = { callback, 'f' },
-      notify_reply_callback = { notify_reply_callback, 'f', true },
-    })
-    message_index = message_index + 1
-    local message_id = message_index
-    local result = encode_and_send({
-      id = message_id,
-      jsonrpc = '2.0',
-      method = method,
-      params = params,
-    })
-    if result then
-      if message_callbacks then
-        message_callbacks[message_id] = schedule_wrap(callback)
+  function result.request(method, params, callback, notify_reply_callback)
+    return client:request(method, params, callback, notify_reply_callback)
+  end
+
+  --- Sends a notification to the LSP server.
+  ---@param method (string) The invoked LSP method
+  ---@param params (table|nil): Parameters for the invoked LSP method
+  ---@returns (bool) `true` if notification could be sent, `false` if not
+  function result.notify(method, params)
+    return client:notify(method, params)
+  end
+
+  return result
+end
+
+---@private
+local function merge_dispatchers(dispatchers)
+  if dispatchers then
+    local user_dispatchers = dispatchers
+    dispatchers = {}
+    for dispatch_name, default_dispatch in pairs(default_dispatchers) do
+      local user_dispatcher = user_dispatchers[dispatch_name]
+      if user_dispatcher then
+        if type(user_dispatcher) ~= 'function' then
+          error(string.format('dispatcher.%s must be a function', dispatch_name))
+        end
+        -- server_request is wrapped elsewhere.
+        if
+          not (dispatch_name == 'server_request' or dispatch_name == 'on_exit') -- TODO this blocks the loop exiting for some reason.
+        then
+          user_dispatcher = schedule_wrap(user_dispatcher)
+        end
+        dispatchers[dispatch_name] = user_dispatcher
       else
-        return false
-      end
-      if notify_reply_callback and notify_reply_callbacks then
-        notify_reply_callbacks[message_id] = schedule_wrap(notify_reply_callback)
+        dispatchers[dispatch_name] = default_dispatch
       end
-      return result, message_id
-    else
-      return false
     end
+  else
+    dispatchers = default_dispatchers
   end
+  return dispatchers
+end
 
-  stderr:read_start(function(_err, chunk)
-    if chunk then
-      local _ = log.error() and log.error('rpc', cmd, 'stderr', chunk)
-    end
-  end)
+--- Create a LSP RPC client factory that connects via TCP to the given host
+--- and port
+---
+---@param host string
+---@param port number
+---@return function
+local function connect(host, port)
+  return function(dispatchers)
+    dispatchers = merge_dispatchers(dispatchers)
+    local tcp = uv.new_tcp()
+    local closing = false
+    local transport = {
+      write = function(msg)
+        tcp:write(msg)
+      end,
+      is_closing = function()
+        return closing
+      end,
+      terminate = function()
+        if not closing then
+          closing = true
+          tcp:shutdown()
+          tcp:close()
+          dispatchers.on_exit(0, 0)
+        end
+      end,
+    }
+    local client = new_client(dispatchers, transport)
+    tcp:connect(host, port, function(err)
+      if err then
+        vim.schedule(function()
+          vim.notify(
+            string.format('Could not connect to %s:%s, reason: %s', host, port, vim.inspect(err)),
+            vim.log.levels.WARN
+          )
+        end)
+        return
+      end
+      local handle_body = function(body)
+        client:handle_body(body)
+      end
+      tcp:read_start(create_read_loop(handle_body, transport.terminate, function(read_err)
+        client:on_error(client_errors.READ_ERROR, read_err)
+      end))
+    end)
 
-  ---@private
-  local function on_error(errkind, ...)
-    assert(client_errors[errkind])
-    -- TODO what to do if this fails?
-    pcall(dispatchers.on_error, errkind, ...)
-  end
-  ---@private
-  local function pcall_handler(errkind, status, head, ...)
-    if not status then
-      on_error(errkind, head, ...)
-      return status, head
-    end
-    return status, head, ...
-  end
-  ---@private
-  local function try_call(errkind, fn, ...)
-    return pcall_handler(errkind, pcall(fn, ...))
+    return public_client(client)
   end
+end
 
-  -- TODO periodically check message_callbacks for old requests past a certain
-  -- time and log them. This would require storing the timestamp. I could call
-  -- them with an error then, perhaps.
+--- Starts an LSP server process and create an LSP RPC client object to
+--- interact with it. Communication with the server is currently limited to stdio.
+---
+---@param cmd (string) Command to start the LSP server.
+---@param cmd_args (table) List of additional string arguments to pass to {cmd}.
+---@param dispatchers table|nil Dispatchers for LSP message types. Valid
+---dispatcher names are:
+--- - `"notification"`
+--- - `"server_request"`
+--- - `"on_error"`
+--- - `"on_exit"`
+---@param extra_spawn_params table|nil Additional context for the LSP
+--- server process. May contain:
+--- - {cwd} (string) Working directory for the LSP server process
+--- - {env} (table) Additional environment variables for LSP server process
+---@returns Client RPC object.
+---
+---@returns Methods:
+--- - `notify()` |vim.lsp.rpc.notify()|
+--- - `request()` |vim.lsp.rpc.request()|
+--- - `is_closing()` returns a boolean indicating if the RPC is closing.
+--- - `terminate()` terminates the RPC client.
+local function start(cmd, cmd_args, dispatchers, extra_spawn_params)
+  local _ = log.info()
+    and log.info('Starting RPC client', { cmd = cmd, args = cmd_args, extra = extra_spawn_params })
+  validate({
+    cmd = { cmd, 's' },
+    cmd_args = { cmd_args, 't' },
+    dispatchers = { dispatchers, 't', true },
+  })
 
-  ---@private
-  local function handle_body(body)
-    local ok, decoded = pcall(vim.json.decode, body, { luanil = { object = true } })
-    if not ok then
-      on_error(client_errors.INVALID_SERVER_JSON, decoded)
-      return
-    end
-    local _ = log.debug() and log.debug('rpc.receive', decoded)
-
-    if type(decoded.method) == 'string' and decoded.id then
-      local err
-      -- Schedule here so that the users functions don't trigger an error and
-      -- we can still use the result.
-      schedule(function()
-        local status, result
-        status, result, err = try_call(
-          client_errors.SERVER_REQUEST_HANDLER_ERROR,
-          dispatchers.server_request,
-          decoded.method,
-          decoded.params
-        )
-        local _ = log.debug()
-          and log.debug(
-            'server_request: callback result',
-            { status = status, result = result, err = err }
-          )
-        if status then
-          if not (result or err) then
-            -- TODO this can be a problem if `null` is sent for result. needs vim.NIL
-            error(
-              string.format(
-                'method %q: either a result or an error must be sent to the server in response',
-                decoded.method
-              )
-            )
-          end
-          if err then
-            assert(
-              type(err) == 'table',
-              'err must be a table. Use rpc_response_error to help format errors.'
-            )
-            local code_name = assert(
-              protocol.ErrorCodes[err.code],
-              'Errors must use protocol.ErrorCodes. Use rpc_response_error to help format errors.'
-            )
-            err.message = err.message or code_name
-          end
-        else
-          -- On an exception, result will contain the error message.
-          err = rpc_response_error(protocol.ErrorCodes.InternalError, result)
-          result = nil
-        end
-        send_response(decoded.id, err, result)
-      end)
-      -- This works because we are expecting vim.NIL here
-    elseif decoded.id and (decoded.result ~= vim.NIL or decoded.error ~= vim.NIL) then
-      -- We sent a number, so we expect a number.
-      local result_id = tonumber(decoded.id)
-
-      -- Notify the user that a response was received for the request
-      local notify_reply_callback = notify_reply_callbacks and notify_reply_callbacks[result_id]
-      if notify_reply_callback then
-        validate({
-          notify_reply_callback = { notify_reply_callback, 'f' },
-        })
-        notify_reply_callback(result_id)
-        notify_reply_callbacks[result_id] = nil
-      end
+  if extra_spawn_params and extra_spawn_params.cwd then
+    assert(is_dir(extra_spawn_params.cwd), 'cwd must be a directory')
+  end
 
-      -- Do not surface RequestCancelled to users, it is RPC-internal.
-      if decoded.error then
-        local mute_error = false
-        if decoded.error.code == protocol.ErrorCodes.RequestCancelled then
-          local _ = log.debug() and log.debug('Received cancellation ack', decoded)
-          mute_error = true
-        end
+  dispatchers = merge_dispatchers(dispatchers)
+  local stdin = uv.new_pipe(false)
+  local stdout = uv.new_pipe(false)
+  local stderr = uv.new_pipe(false)
+  local handle, pid
 
-        if mute_error then
-          -- Clear any callback since this is cancelled now.
-          -- This is safe to do assuming that these conditions hold:
-          -- - The server will not send a result callback after this cancellation.
-          -- - If the server sent this cancellation ACK after sending the result, the user of this RPC
-          -- client will ignore the result themselves.
-          if result_id and message_callbacks then
-            message_callbacks[result_id] = nil
-          end
-          return
-        end
+  local client = new_client(dispatchers, {
+    write = function(msg)
+      stdin:write(msg)
+    end,
+    is_closing = function()
+      return handle == nil or handle:is_closing()
+    end,
+    terminate = function()
+      if handle then
+        handle:kill(15)
       end
+    end,
+  })
 
-      local callback = message_callbacks and message_callbacks[result_id]
-      if callback then
-        message_callbacks[result_id] = nil
-        validate({
-          callback = { callback, 'f' },
-        })
-        if decoded.error then
-          decoded.error = setmetatable(decoded.error, {
-            __tostring = format_rpc_error,
-          })
-        end
-        try_call(
-          client_errors.SERVER_RESULT_CALLBACK_ERROR,
-          callback,
-          decoded.error,
-          decoded.result
-        )
-      else
-        on_error(client_errors.NO_RESULT_CALLBACK_FOUND, decoded)
-        local _ = log.error()
-          and log.error('No callback found for server response id ' .. result_id)
-      end
-    elseif type(decoded.method) == 'string' then
-      -- Notification
-      try_call(
-        client_errors.NOTIFICATION_HANDLER_ERROR,
-        dispatchers.notification,
-        decoded.method,
-        decoded.params
-      )
+  ---@private
+  --- Callback for |vim.loop.spawn()| Closes all streams and runs the `on_exit` dispatcher.
+  ---@param code (number) Exit code
+  ---@param signal (number) Signal that was used to terminate (if any)
+  local function onexit(code, signal)
+    stdin:close()
+    stdout:close()
+    stderr:close()
+    handle:close()
+    dispatchers.on_exit(code, signal)
+  end
+  local spawn_params = {
+    args = cmd_args,
+    stdio = { stdin, stdout, stderr },
+    detached = not is_win,
+  }
+  if extra_spawn_params then
+    spawn_params.cwd = extra_spawn_params.cwd
+    spawn_params.env = env_merge(extra_spawn_params.env)
+    if extra_spawn_params.detached ~= nil then
+      spawn_params.detached = extra_spawn_params.detached
+    end
+  end
+  handle, pid = uv.spawn(cmd, spawn_params, onexit)
+  if handle == nil then
+    stdin:close()
+    stdout:close()
+    stderr:close()
+    local msg = string.format('Spawning language server with cmd: `%s` failed', cmd)
+    if string.match(pid, 'ENOENT') then
+      msg = msg
+        .. '. The language server is either not installed, missing from PATH, or not executable.'
     else
-      -- Invalid server message
-      on_error(client_errors.INVALID_SERVER_MESSAGE, decoded)
+      msg = msg .. string.format(' with error message: %s', pid)
     end
+    vim.notify(msg, vim.log.levels.WARN)
+    return
   end
 
-  local request_parser = coroutine.wrap(request_parser_loop)
-  request_parser()
-  stdout:read_start(function(err, chunk)
-    if err then
-      -- TODO better handling. Can these be intermittent errors?
-      on_error(client_errors.READ_ERROR, err)
-      return
-    end
-    -- This should signal that we are done reading from the client.
-    if not chunk then
-      return
-    end
-    -- Flush anything in the parser by looping until we don't get a result
-    -- anymore.
-    while true do
-      local headers, body = request_parser(chunk)
-      -- If we successfully parsed, then handle the response.
-      if headers then
-        handle_body(body)
-        -- Set chunk to empty so that we can call request_parser to get
-        -- anything existing in the parser to flush.
-        chunk = ''
-      else
-        break
-      end
+  stderr:read_start(function(_, chunk)
+    if chunk then
+      local _ = log.error() and log.error('rpc', cmd, 'stderr', chunk)
     end
   end)
 
-  return {
-    pid = pid,
-    handle = handle,
-    request = request,
-    notify = notify,
-  }
+  local handle_body = function(body)
+    client:handle_body(body)
+  end
+  stdout:read_start(create_read_loop(handle_body, nil, function(err)
+    client:on_error(client_errors.READ_ERROR, err)
+  end))
+
+  return public_client(client)
 end
 
 return {
   start = start,
+  connect = connect,
   rpc_response_error = rpc_response_error,
   format_rpc_error = format_rpc_error,
   client_errors = client_errors,
+  create_read_loop = create_read_loop,
 }
 -- vim:sw=2 ts=2 et
diff --git a/runtime/lua/vim/lsp/util.lua b/runtime/lua/vim/lsp/util.lua
index eac21db386..283099bbcf 100644
--- a/runtime/lua/vim/lsp/util.lua
+++ b/runtime/lua/vim/lsp/util.lua
@@ -113,7 +113,7 @@ end
 --- Convert byte index to `encoding` index.
 --- Convenience wrapper around vim.str_utfindex
 ---@param line string line to be indexed
----@param index number byte index (utf-8), or `nil` for length
+---@param index number|nil byte index (utf-8), or `nil` for length
 ---@param encoding string utf-8|utf-16|utf-32|nil defaults to utf-16
 ---@return number `encoding` index of `index` in `line`
 function M._str_utfindex_enc(line, index, encoding)
diff --git a/runtime/lua/vim/shared.lua b/runtime/lua/vim/shared.lua
index d6c3e25b3b..e1b4ed4ea9 100644
--- a/runtime/lua/vim/shared.lua
+++ b/runtime/lua/vim/shared.lua
@@ -526,7 +526,7 @@ function vim.trim(s)
   return s:match('^%s*(.*%S)') or ''
 end
 
---- Escapes magic chars in a Lua pattern.
+--- Escapes magic chars in |lua-patterns|.
 ---
 ---@see https://github.com/rxi/lume
 ---@param s string String to escape
diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua
index 70f2c425ed..6431162799 100644
--- a/runtime/lua/vim/treesitter.lua
+++ b/runtime/lua/vim/treesitter.lua
@@ -118,4 +118,127 @@ function M.get_string_parser(str, lang, opts)
   return LanguageTree.new(str, lang, opts)
 end
 
+--- Determines whether a node is the ancestor of another
+---
+---@param dest table the possible ancestor
+---@param source table the possible descendant node
+---
+---@returns (boolean) True if dest is an ancestor of source
+function M.is_ancestor(dest, source)
+  if not (dest and source) then
+    return false
+  end
+
+  local current = source
+  while current ~= nil do
+    if current == dest then
+      return true
+    end
+
+    current = current:parent()
+  end
+
+  return false
+end
+
+--- Get the node's range or unpack a range table
+---
+---@param node_or_range table
+---
+---@returns start_row, start_col, end_row, end_col
+function M.get_node_range(node_or_range)
+  if type(node_or_range) == 'table' then
+    return unpack(node_or_range)
+  else
+    return node_or_range:range()
+  end
+end
+
+---Determines whether (line, col) position is in node range
+---
+---@param node Node defining the range
+---@param line A line (0-based)
+---@param col A column (0-based)
+function M.is_in_node_range(node, line, col)
+  local start_line, start_col, end_line, end_col = M.get_node_range(node)
+  if line >= start_line and line <= end_line then
+    if line == start_line and line == end_line then
+      return col >= start_col and col < end_col
+    elseif line == start_line then
+      return col >= start_col
+    elseif line == end_line then
+      return col < end_col
+    else
+      return true
+    end
+  else
+    return false
+  end
+end
+
+---Determines if a node contains a range
+---@param node table The node
+---@param range table The range
+---
+---@returns (boolean) True if the node contains the range
+function M.node_contains(node, range)
+  local start_row, start_col, end_row, end_col = node:range()
+  local start_fits = start_row < range[1] or (start_row == range[1] and start_col <= range[2])
+  local end_fits = end_row > range[3] or (end_row == range[3] and end_col >= range[4])
+
+  return start_fits and end_fits
+end
+
+---Gets a list of captures for a given cursor position
+---@param bufnr number The buffer number
+---@param row number The position row
+---@param col number The position column
+---
+---@returns (table) A table of captures
+function M.get_captures_at_position(bufnr, row, col)
+  if bufnr == 0 then
+    bufnr = a.nvim_get_current_buf()
+  end
+  local buf_highlighter = M.highlighter.active[bufnr]
+
+  if not buf_highlighter then
+    return {}
+  end
+
+  local matches = {}
+
+  buf_highlighter.tree:for_each_tree(function(tstree, tree)
+    if not tstree then
+      return
+    end
+
+    local root = tstree:root()
+    local root_start_row, _, root_end_row, _ = root:range()
+
+    -- Only worry about trees within the line range
+    if root_start_row > row or root_end_row < row then
+      return
+    end
+
+    local q = buf_highlighter:get_query(tree:lang())
+
+    -- Some injected languages may not have highlight queries.
+    if not q:query() then
+      return
+    end
+
+    local iter = q:query():iter_captures(root, buf_highlighter.bufnr, row, row + 1)
+
+    for capture, node, metadata in iter do
+      if M.is_in_node_range(node, row, col) then
+        local c = q._query.captures[capture] -- name of the capture in the query
+        if c ~= nil then
+          table.insert(matches, { capture = c, priority = metadata.priority })
+        end
+      end
+    end
+  end, true)
+  return matches
+end
+
 return M
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
index e27a5fa9c3..1f242b0fdd 100644
--- a/runtime/lua/vim/treesitter/highlighter.lua
+++ b/runtime/lua/vim/treesitter/highlighter.lua
@@ -12,105 +12,18 @@ TSHighlighterQuery.__index = TSHighlighterQuery
 
 local ns = a.nvim_create_namespace('treesitter/highlighter')
 
-local _default_highlights = {}
-local _link_default_highlight_once = function(from, to)
-  if not _default_highlights[from] then
-    _default_highlights[from] = true
-    a.nvim_set_hl(0, from, { link = to, default = true })
-  end
-
-  return from
-end
-
--- If @definition.special does not exist use @definition instead
-local subcapture_fallback = {
-  __index = function(self, capture)
-    local rtn
-    local shortened = capture
-    while not rtn and shortened do
-      shortened = shortened:match('(.*)%.')
-      rtn = shortened and rawget(self, shortened)
-    end
-    rawset(self, capture, rtn or '__notfound')
-    return rtn
-  end,
-}
-
-TSHighlighter.hl_map = setmetatable({
-  ['error'] = 'Error',
-  ['text.underline'] = 'Underlined',
-  ['todo'] = 'Todo',
-  ['debug'] = 'Debug',
-
-  -- Miscs
-  ['comment'] = 'Comment',
-  ['punctuation.delimiter'] = 'Delimiter',
-  ['punctuation.bracket'] = 'Delimiter',
-  ['punctuation.special'] = 'Delimiter',
-
-  -- Constants
-  ['constant'] = 'Constant',
-  ['constant.builtin'] = 'Special',
-  ['constant.macro'] = 'Define',
-  ['define'] = 'Define',
-  ['macro'] = 'Macro',
-  ['string'] = 'String',
-  ['string.regex'] = 'String',
-  ['string.escape'] = 'SpecialChar',
-  ['character'] = 'Character',
-  ['character.special'] = 'SpecialChar',
-  ['number'] = 'Number',
-  ['boolean'] = 'Boolean',
-  ['float'] = 'Float',
-
-  -- Functions
-  ['function'] = 'Function',
-  ['function.special'] = 'Function',
-  ['function.builtin'] = 'Special',
-  ['function.macro'] = 'Macro',
-  ['parameter'] = 'Identifier',
-  ['method'] = 'Function',
-  ['field'] = 'Identifier',
-  ['property'] = 'Identifier',
-  ['constructor'] = 'Special',
-
-  -- Keywords
-  ['conditional'] = 'Conditional',
-  ['repeat'] = 'Repeat',
-  ['label'] = 'Label',
-  ['operator'] = 'Operator',
-  ['keyword'] = 'Keyword',
-  ['exception'] = 'Exception',
-
-  ['type'] = 'Type',
-  ['type.builtin'] = 'Type',
-  ['type.qualifier'] = 'Type',
-  ['type.definition'] = 'Typedef',
-  ['storageclass'] = 'StorageClass',
-  ['structure'] = 'Structure',
-  ['include'] = 'Include',
-  ['preproc'] = 'PreProc',
-}, subcapture_fallback)
-
----@private
-local function is_highlight_name(capture_name)
-  local firstc = string.sub(capture_name, 1, 1)
-  return firstc ~= string.lower(firstc)
-end
-
 ---@private
 function TSHighlighterQuery.new(lang, query_string)
   local self = setmetatable({}, { __index = TSHighlighterQuery })
 
   self.hl_cache = setmetatable({}, {
     __index = function(table, capture)
-      local hl, is_vim_highlight = self:_get_hl_from_capture(capture)
-      if not is_vim_highlight then
-        hl = _link_default_highlight_once(lang .. hl, hl)
+      local name = self._query.captures[capture]
+      local id = 0
+      if not vim.startswith(name, '_') then
+        id = a.nvim_get_hl_id_by_name('@' .. name .. '.' .. lang)
       end
 
-      local id = a.nvim_get_hl_id_by_name(hl)
-
       rawset(table, capture, id)
       return id
     end,
@@ -130,20 +43,6 @@ function TSHighlighterQuery:query()
   return self._query
 end
 
----@private
---- Get the hl from capture.
---- Returns a tuple { highlight_name: string, is_builtin: bool }
-function TSHighlighterQuery:_get_hl_from_capture(capture)
-  local name = self._query.captures[capture]
-
-  if is_highlight_name(name) then
-    -- From "Normal.left" only keep "Normal"
-    return vim.split(name, '.', true)[1], true
-  else
-    return TSHighlighter.hl_map[name] or 0, false
-  end
-end
-
 --- Creates a new highlighter using @param tree
 ---
 ---@param tree The language tree to use for highlighting
diff --git a/runtime/lua/vim/treesitter/language.lua b/runtime/lua/vim/treesitter/language.lua
index dfb6f5be84..d14b825603 100644
--- a/runtime/lua/vim/treesitter/language.lua
+++ b/runtime/lua/vim/treesitter/language.lua
@@ -6,10 +6,11 @@ local M = {}
 ---
 --- Parsers are searched in the `parser` runtime directory.
 ---
----@param lang The language the parser should parse
----@param path Optional path the parser is located at
----@param silent Don't throw an error if language not found
-function M.require_language(lang, path, silent)
+---@param lang string The language the parser should parse
+---@param path string|nil Optional path the parser is located at
+---@param silent boolean|nil Don't throw an error if language not found
+---@param symbol_name string|nil Internal symbol name for the language to load
+function M.require_language(lang, path, silent, symbol_name)
   if vim._ts_has_language(lang) then
     return true
   end
@@ -21,7 +22,6 @@ function M.require_language(lang, path, silent)
         return false
       end
 
-      -- TODO(bfredl): help tag?
       error("no parser for '" .. lang .. "' language, see :help treesitter-parsers")
     end
     path = paths[1]
@@ -29,10 +29,10 @@ function M.require_language(lang, path, silent)
 
   if silent then
     return pcall(function()
-      vim._ts_add_language(path, lang)
+      vim._ts_add_language(path, lang, symbol_name)
     end)
   else
-    vim._ts_add_language(path, lang)
+    vim._ts_add_language(path, lang, symbol_name)
   end
 
   return true
diff --git a/runtime/lua/vim/treesitter/languagetree.lua b/runtime/lua/vim/treesitter/languagetree.lua
index 4d3b0631a2..70317a9f94 100644
--- a/runtime/lua/vim/treesitter/languagetree.lua
+++ b/runtime/lua/vim/treesitter/languagetree.lua
@@ -299,7 +299,7 @@ function LanguageTree:included_regions()
 end
 
 ---@private
-local function get_node_range(node, id, metadata)
+local function get_range_from_metadata(node, id, metadata)
   if metadata[id] and metadata[id].range then
     return metadata[id].range
   end
@@ -362,7 +362,7 @@ function LanguageTree:_get_injections()
         elseif name == 'combined' then
           combined = true
         elseif name == 'content' and #ranges == 0 then
-          table.insert(ranges, get_node_range(node, id, metadata))
+          table.insert(ranges, get_range_from_metadata(node, id, metadata))
           -- Ignore any tags that start with "_"
           -- Allows for other tags to be used in matches
         elseif string.sub(name, 1, 1) ~= '_' then
@@ -371,7 +371,7 @@ function LanguageTree:_get_injections()
           end
 
           if #ranges == 0 then
-            table.insert(ranges, get_node_range(node, id, metadata))
+            table.insert(ranges, get_range_from_metadata(node, id, metadata))
           end
         end
       end
@@ -549,6 +549,44 @@ function LanguageTree:contains(range)
   return false
 end
 
+--- Gets the tree that contains {range}
+---
+---@param range table A text range
+---@param opts table Options table
+---@param opts.ignore_injections boolean (default true) Ignore injected languages.
+function LanguageTree:tree_for_range(range, opts)
+  opts = opts or {}
+  local ignore = vim.F.if_nil(opts.ignore_injections, true)
+
+  if not ignore then
+    for _, child in pairs(self._children) do
+      for _, tree in pairs(child:trees()) do
+        if tree_contains(tree, range) then
+          return tree
+        end
+      end
+    end
+  end
+
+  for _, tree in pairs(self._trees) do
+    if tree_contains(tree, range) then
+      return tree
+    end
+  end
+
+  return nil
+end
+
+--- Gets the smallest named node that contains {range}
+---
+---@param range table A text range
+---@param opts table Options table
+---@param opts.ignore_injections boolean (default true) Ignore injected languages.
+function LanguageTree:named_node_for_range(range, opts)
+  local tree = self:tree_for_range(range, opts)
+  return tree:root():named_descendant_for_range(unpack(range))
+end
+
 --- Gets the appropriate language that contains {range}
 ---
 ---@param range A text range, see |LanguageTree:contains|
diff --git a/runtime/lua/vim/treesitter/query.lua b/runtime/lua/vim/treesitter/query.lua
index 103e85abfd..697e2e7691 100644
--- a/runtime/lua/vim/treesitter/query.lua
+++ b/runtime/lua/vim/treesitter/query.lua
@@ -181,9 +181,14 @@ end
 
 --- Gets the text corresponding to a given node
 ---
----@param node the node
----@param source The buffer or string from which the node is extracted
-function M.get_node_text(node, source)
+---@param node table The node
+---@param source table The buffer or string from which the node is extracted
+---@param opts table Optional parameters.
+---          - concat: (boolean default true) Concatenate result in a string
+function M.get_node_text(node, source, opts)
+  opts = opts or {}
+  local concat = vim.F.if_nil(opts.concat, true)
+
   local start_row, start_col, start_byte = node:start()
   local end_row, end_col, end_byte = node:end_()
 
@@ -210,7 +215,7 @@ function M.get_node_text(node, source)
       end
     end
 
-    return table.concat(lines, '\n')
+    return concat and table.concat(lines, '\n') or lines
   elseif type(source) == 'string' then
     return source:sub(start_byte + 1, end_byte)
   end