Merge remote-tracking branch 'upstream/master' into colorcolcharcolorcolchar

9 files changed, 1501 insertions, 275 deletions

diff --git a/test/functional/plugin/lsp/completion_spec.lua b/test/functional/plugin/lsp/completion_spec.lua
new file mode 100644
index 0000000000..9354654afe
--- /dev/null
+++ b/test/functional/plugin/lsp/completion_spec.lua
@@ -0,0 +1,239 @@
+---@diagnostic disable: no-unknown
+local helpers = require('test.functional.helpers')(after_each)
+local eq = helpers.eq
+local exec_lua = helpers.exec_lua
+
+
+--- Convert completion results.
+---
+---@param line string line contents. Mark cursor position with `|`
+---@param candidates lsp.CompletionList|lsp.CompletionItem[]
+---@param lnum? integer 0-based, defaults to 0
+---@return {items: table[], server_start_boundary: integer?}
+local function complete(line, candidates, lnum)
+  lnum = lnum or 0
+  -- nvim_win_get_cursor returns 0 based column, line:find returns 1 based
+  local cursor_col = line:find("|") - 1
+  line = line:gsub("|", "")
+  return exec_lua([[
+    local line, cursor_col, lnum, result = ...
+    local line_to_cursor = line:sub(1, cursor_col)
+    local client_start_boundary = vim.fn.match(line_to_cursor, '\\k*$')
+    local items, server_start_boundary = require("vim.lsp._completion")._convert_results(
+      line,
+      lnum,
+      cursor_col,
+      client_start_boundary,
+      nil,
+      result,
+      "utf-16"
+    )
+    return {
+      items = items,
+      server_start_boundary = server_start_boundary
+    }
+  ]], line, cursor_col, lnum, candidates)
+end
+
+
+describe("vim.lsp._completion", function()
+  before_each(helpers.clear)
+
+  -- https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_completion
+  it('prefers textEdit over label as word', function()
+    local range0 = {
+      start = { line = 0, character = 0 },
+      ["end"] = { line = 0, character = 0 },
+    }
+    local completion_list = {
+      -- resolves into label
+      { label = 'foobar', sortText = 'a', documentation = 'documentation' },
+      {
+        label = 'foobar',
+        sortText = 'b',
+        documentation = { value = 'documentation' },
+      },
+      -- resolves into insertText
+      { label = 'foocar', sortText = 'c', insertText = 'foobar' },
+      { label = 'foocar', sortText = 'd', insertText = 'foobar' },
+      -- resolves into textEdit.newText
+      { label = 'foocar', sortText = 'e', insertText = 'foodar', textEdit = { newText = 'foobar', range = range0 } },
+      { label = 'foocar', sortText = 'f', textEdit = { newText = 'foobar', range = range0 } },
+      -- real-world snippet text
+      {
+        label = 'foocar',
+        sortText = 'g',
+        insertText = 'foodar',
+        insertTextFormat = 2,
+        textEdit = { newText = 'foobar(${1:place holder}, ${2:more ...holder{\\}})', range = range0 },
+      },
+      {
+        label = 'foocar',
+        sortText = 'h',
+        insertText = 'foodar(${1:var1} typ1, ${2:var2} *typ2) {$0\\}',
+        insertTextFormat = 2,
+      },
+      -- nested snippet tokens
+      {
+        label = 'foocar',
+        sortText = 'i',
+        insertText = 'foodar(${1:${2|typ1,typ2|}}) {$0\\}',
+        insertTextFormat = 2,
+      },
+      -- braced tabstop
+      { label = 'foocar', sortText = 'j', insertText = 'foodar()${0}', insertTextFormat = 2},
+      -- plain text
+      {
+        label = 'foocar',
+        sortText = 'k',
+        insertText = 'foodar(${1:var1})',
+        insertTextFormat = 1,
+      },
+    }
+    local expected = {
+      {
+        abbr = 'foobar',
+        word = 'foobar',
+      },
+      {
+        abbr = 'foobar',
+        word = 'foobar',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foobar',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foobar',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foobar',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foobar',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foobar(place holder, more ...holder{})',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foodar(var1 typ1, var2 *typ2) {}',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foodar(typ1) {}',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foodar()',
+      },
+      {
+        abbr = 'foocar',
+        word = 'foodar(${1:var1})',
+      },
+    }
+    local result = complete('|', completion_list)
+    result = vim.tbl_map(function(x)
+      return {
+        abbr = x.abbr,
+        word = x.word
+      }
+    end, result.items)
+    eq(expected, result)
+  end)
+  it("uses correct start boundary", function()
+    local completion_list = {
+      isIncomplete = false,
+      items = {
+        {
+          filterText = "this_thread",
+          insertText = "this_thread",
+          insertTextFormat = 1,
+          kind = 9,
+          label = " this_thread",
+          score = 1.3205767869949,
+          sortText = "4056f757this_thread",
+          textEdit = {
+            newText = "this_thread",
+            range = {
+              start = { line = 0, character = 7 },
+              ["end"] = { line = 0, character = 11 },
+            },
+          }
+        },
+      }
+    }
+    local expected = {
+      abbr = ' this_thread',
+      dup = 1,
+      empty = 1,
+      icase = 1,
+      kind = 'Module',
+      menu = '',
+      word = 'this_thread',
+    }
+    local result = complete("  std::this|", completion_list)
+    eq(7, result.server_start_boundary)
+    local item = result.items[1]
+    item.user_data = nil
+    eq(expected, item)
+  end)
+
+  it("should search from start boundary to cursor position", function()
+    local completion_list = {
+      isIncomplete = false,
+      items = {
+        {
+          filterText = "this_thread",
+          insertText = "this_thread",
+          insertTextFormat = 1,
+          kind = 9,
+          label = " this_thread",
+          score = 1.3205767869949,
+          sortText = "4056f757this_thread",
+          textEdit = {
+            newText = "this_thread",
+            range = {
+              start = { line = 0, character = 7 },
+              ["end"] = { line = 0, character = 11 },
+            },
+          }
+        },
+        {
+          filterText = "notthis_thread",
+          insertText = "notthis_thread",
+          insertTextFormat = 1,
+          kind = 9,
+          label = " notthis_thread",
+          score = 1.3205767869949,
+          sortText = "4056f757this_thread",
+          textEdit = {
+            newText = "notthis_thread",
+            range = {
+              start = { line = 0, character = 7 },
+              ["end"] = { line = 0, character = 11 },
+            },
+          }
+        },
+      }
+    }
+    local expected = {
+      abbr = ' this_thread',
+      dup = 1,
+      empty = 1,
+      icase = 1,
+      kind = 'Module',
+      menu = '',
+      word = 'this_thread',
+    }
+    local result = complete("  std::this|is", completion_list)
+    eq(1, #result.items)
+    local item = result.items[1]
+    item.user_data = nil
+    eq(expected, item)
+  end)
+end)
diff --git a/test/functional/plugin/lsp/diagnostic_spec.lua b/test/functional/plugin/lsp/diagnostic_spec.lua
index f73ffc29b0..1da0222114 100644
--- a/test/functional/plugin/lsp/diagnostic_spec.lua
+++ b/test/functional/plugin/lsp/diagnostic_spec.lua
@@ -1,10 +1,13 @@
 local helpers = require('test.functional.helpers')(after_each)
+local lsp_helpers = require('test.functional.plugin.lsp.helpers')
 
 local clear = helpers.clear
 local exec_lua = helpers.exec_lua
 local eq = helpers.eq
 local neq = require('test.helpers').neq
 
+local create_server_definition = lsp_helpers.create_server_definition
+
 describe('vim.lsp.diagnostic', function()
   local fake_uri
 
@@ -81,6 +84,7 @@ describe('vim.lsp.diagnostic', function()
       local lines = {"1st line of text", "2nd line of text", "wow", "cool", "more", "lines"}
       vim.fn.bufload(diagnostic_bufnr)
       vim.api.nvim_buf_set_lines(diagnostic_bufnr, 0, 1, false, lines)
+      vim.api.nvim_win_set_buf(0, diagnostic_bufnr)
       return diagnostic_bufnr
     ]], fake_uri)
   end)
@@ -97,7 +101,6 @@ describe('vim.lsp.diagnostic', function()
         }
 
         diagnostics[1].code = 42
-        diagnostics[1].tags = {"foo", "bar"}
         diagnostics[1].data = "Hello world"
 
         vim.lsp.diagnostic.on_publish_diagnostics(nil, {
@@ -110,10 +113,9 @@ describe('vim.lsp.diagnostic', function()
           vim.lsp.diagnostic.get_line_diagnostics(diagnostic_bufnr, 1)[1],
         }
       ]]
-      eq({code = 42, tags = {"foo", "bar"}, data = "Hello world"}, result[1].user_data.lsp)
+      eq({code = 42, data = "Hello world"}, result[1].user_data.lsp)
       eq(42, result[1].code)
       eq(42, result[2].code)
-      eq({"foo", "bar"}, result[2].tags)
       eq("Hello world", result[2].data)
     end)
   end)
@@ -267,4 +269,155 @@ describe('vim.lsp.diagnostic', function()
       eq(exec_lua([[return #vim.diagnostic.get(...)]], bufnr), 0)
     end)
   end)
+
+  describe('vim.lsp.diagnostic.on_diagnostic', function()
+    before_each(function()
+      exec_lua(create_server_definition)
+      exec_lua([[
+        server = _create_server({
+          capabilities = {
+            diagnosticProvider = {
+            }
+          }
+        })
+
+        function get_extmarks(bufnr, client_id)
+          local namespace = vim.lsp.diagnostic.get_namespace(client_id, true)
+          local ns = vim.diagnostic.get_namespace(namespace)
+          local extmarks = {}
+          if ns.user_data.virt_text_ns then
+            for _, e in pairs(vim.api.nvim_buf_get_extmarks(bufnr, ns.user_data.virt_text_ns, 0, -1, {details=true})) do
+              table.insert(extmarks, e)
+            end
+          end
+          if ns.user_data.underline_ns then
+            for _, e in pairs(vim.api.nvim_buf_get_extmarks(bufnr, ns.user_data.underline_ns, 0, -1, {details=true})) do
+              table.insert(extmarks, e)
+            end
+          end
+          return extmarks
+        end
+
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+    end)
+
+    it('adds diagnostics to vim.diagnostics', function()
+      local diags = exec_lua([[
+        vim.lsp.diagnostic.on_diagnostic(nil,
+          {
+            kind = 'full',
+            items = {
+              make_error('Pull Diagnostic', 4, 4, 4, 4),
+            }
+          },
+          {
+            params = {
+              textDocument = { uri = fake_uri },
+            },
+            uri = fake_uri,
+            client_id = client_id,
+          },
+          {}
+        )
+
+        return vim.diagnostic.get(diagnostic_bufnr)
+      ]])
+      eq(1, #diags)
+      eq('Pull Diagnostic', diags[1].message)
+    end)
+
+    it('allows configuring the virtual text via vim.lsp.with', function()
+      local expected_spacing = 10
+      local extmarks = exec_lua(
+        [[
+        Diagnostic = vim.lsp.with(vim.lsp.diagnostic.on_diagnostic, {
+          virtual_text = {
+            spacing = ...,
+          },
+        })
+
+        Diagnostic(nil,
+          {
+            kind = 'full',
+            items = {
+              make_error('Pull Diagnostic', 4, 4, 4, 4),
+            }
+          },
+          {
+            params = {
+              textDocument = { uri = fake_uri },
+            },
+            uri = fake_uri,
+            client_id = client_id,
+          },
+          {}
+        )
+
+        return get_extmarks(diagnostic_bufnr, client_id)
+      ]],
+        expected_spacing
+      )
+      eq(2, #extmarks)
+      eq(expected_spacing, #extmarks[1][4].virt_text[1][1])
+    end)
+
+    it('clears diagnostics when client detaches', function()
+      exec_lua([[
+        vim.lsp.diagnostic.on_diagnostic(nil,
+          {
+            kind = 'full',
+            items = {
+              make_error('Pull Diagnostic', 4, 4, 4, 4),
+            }
+          },
+          {
+            params = {
+              textDocument = { uri = fake_uri },
+            },
+            uri = fake_uri,
+            client_id = client_id,
+          },
+          {}
+        )
+      ]])
+      local diags = exec_lua([[return vim.diagnostic.get(diagnostic_bufnr)]])
+      eq(1, #diags)
+
+      exec_lua([[ vim.lsp.stop_client(client_id) ]])
+
+      diags = exec_lua([[return vim.diagnostic.get(diagnostic_bufnr)]])
+      eq(0, #diags)
+    end)
+
+    it('keeps diagnostics when one client detaches and others still are attached', function()
+      exec_lua([[
+        client_id2 = vim.lsp.start({ name = 'dummy2', cmd = server.cmd })
+
+        vim.lsp.diagnostic.on_diagnostic(nil,
+          {
+            kind = 'full',
+            items = {
+              make_error('Pull Diagnostic', 4, 4, 4, 4),
+            }
+          },
+          {
+            params = {
+              textDocument = { uri = fake_uri },
+            },
+            uri = fake_uri,
+            client_id = client_id,
+          },
+          {}
+        )
+      ]])
+      local diags = exec_lua([[return vim.diagnostic.get(diagnostic_bufnr)]])
+      eq(1, #diags)
+
+      exec_lua([[ vim.lsp.stop_client(client_id2) ]])
+
+      diags = exec_lua([[return vim.diagnostic.get(diagnostic_bufnr)]])
+      eq(1, #diags)
+    end)
+  end)
 end)
diff --git a/test/functional/plugin/lsp/helpers.lua b/test/functional/plugin/lsp/helpers.lua
index 028ccb9e2c..15e6a62781 100644
--- a/test/functional/plugin/lsp/helpers.lua
+++ b/test/functional/plugin/lsp/helpers.lua
@@ -13,6 +13,7 @@ function M.clear_notrace()
   -- solution: don't look too closely for dragons
   clear {env={
     NVIM_LUA_NOTRACK="1";
+    NVIM_APPNAME="nvim_lsp_test";
     VIMRUNTIME=os.getenv"VIMRUNTIME";
   }}
 end
@@ -80,17 +81,15 @@ M.fake_lsp_logfile = 'Xtest-fake-lsp.log'
 local function fake_lsp_server_setup(test_name, timeout_ms, options, settings)
   exec_lua([=[
     lsp = require('vim.lsp')
-    local test_name, fixture_filename, logfile, timeout, options, settings = ...
+    local test_name, fake_lsp_code, fake_lsp_logfile, timeout, options, settings = ...
     TEST_RPC_CLIENT_ID = lsp.start_client {
       cmd_env = {
-        NVIM_LOG_FILE = logfile;
+        NVIM_LOG_FILE = fake_lsp_logfile;
         NVIM_LUA_NOTRACK = "1";
+        NVIM_APPNAME = "nvim_lsp_test";
       };
       cmd = {
-        vim.v.progpath, '-Es', '-u', 'NONE', '--headless',
-        "-c", string.format("lua TEST_NAME = %q", test_name),
-        "-c", string.format("lua TIMEOUT = %d", timeout),
-        "-c", "luafile "..fixture_filename,
+        vim.v.progpath, '-l', fake_lsp_code, test_name, tostring(timeout),
       };
       handlers = setmetatable({}, {
         __index = function(t, method)
@@ -100,7 +99,7 @@ local function fake_lsp_server_setup(test_name, timeout_ms, options, settings)
         end;
       });
       workspace_folders = {{
-          uri = 'file://' .. vim.loop.cwd(),
+          uri = 'file://' .. vim.uv.cwd(),
           name = 'test_folder',
       }};
       on_init = function(client, result)
diff --git a/test/functional/plugin/lsp/incremental_sync_spec.lua b/test/functional/plugin/lsp/incremental_sync_spec.lua
index 4985da9cd7..724b3efb97 100644
--- a/test/functional/plugin/lsp/incremental_sync_spec.lua
+++ b/test/functional/plugin/lsp/incremental_sync_spec.lua
@@ -21,7 +21,7 @@ before_each(function ()
     --   ["mac"] = '\r',
     -- }
 
-    -- local line_ending = format_line_ending[vim.api.nvim_buf_get_option(0, 'fileformat')]
+    -- local line_ending = format_line_ending[vim.api.nvim_get_option_value('fileformat', {})]
 
 
     function test_register(bufnr, id, offset_encoding, line_ending)
diff --git a/test/functional/plugin/lsp/inlay_hint_spec.lua b/test/functional/plugin/lsp/inlay_hint_spec.lua
new file mode 100644
index 0000000000..d0d55df72b
--- /dev/null
+++ b/test/functional/plugin/lsp/inlay_hint_spec.lua
@@ -0,0 +1,204 @@
+local helpers = require('test.functional.helpers')(after_each)
+local lsp_helpers = require('test.functional.plugin.lsp.helpers')
+local Screen = require('test.functional.ui.screen')
+
+local eq = helpers.eq
+local dedent = helpers.dedent
+local exec_lua = helpers.exec_lua
+local insert = helpers.insert
+
+local clear_notrace = lsp_helpers.clear_notrace
+local create_server_definition = lsp_helpers.create_server_definition
+
+local text = dedent([[
+auto add(int a, int b) { return a + b; }
+
+int main() {
+    int x = 1;
+    int y = 2;
+    return add(x,y);
+}
+}]])
+
+local response = [==[
+[
+{"kind":1,"paddingLeft":false,"label":"-> int","position":{"character":22,"line":0},"paddingRight":false},
+{"kind":2,"paddingLeft":false,"label":"a:","position":{"character":15,"line":5},"paddingRight":true},
+{"kind":2,"paddingLeft":false,"label":"b:","position":{"character":17,"line":5},"paddingRight":true}
+]
+]==]
+
+local grid_without_inlay_hints = [[
+  auto add(int a, int b) { return a + b; }          |
+                                                    |
+  int main() {                                      |
+      int x = 1;                                    |
+      int y = 2;                                    |
+      return add(x,y);                              |
+  }                                                 |
+  ^}                                                 |
+                                                    |
+]]
+
+local grid_with_inlay_hints =  [[
+  auto add(int a, int b)-> int { return a + b; }    |
+                                                    |
+  int main() {                                      |
+      int x = 1;                                    |
+      int y = 2;                                    |
+      return add(a: x,b: y);                        |
+  }                                                 |
+  ^}                                                 |
+                                                    |
+]]
+
+--- @type test.functional.ui.screen
+local screen
+before_each(function()
+  clear_notrace()
+  screen = Screen.new(50, 9)
+  screen:attach()
+
+  exec_lua(create_server_definition)
+  exec_lua([[
+    local response = ...
+    server = _create_server({
+      capabilities = {
+        inlayHintProvider = true,
+      },
+      handlers = {
+        ['textDocument/inlayHint'] = function()
+          return vim.json.decode(response)
+        end,
+      }
+    })
+
+    bufnr = vim.api.nvim_get_current_buf()
+    vim.api.nvim_win_set_buf(0, bufnr)
+
+    client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+  ]], response)
+
+  insert(text)
+  exec_lua([[vim.lsp.inlay_hint.enable(bufnr)]])
+  screen:expect({ grid = grid_with_inlay_hints })
+end)
+
+after_each(function()
+  exec_lua("vim.api.nvim_exec_autocmds('VimLeavePre', { modeline = false })")
+end)
+
+describe('vim.lsp.inlay_hint', function()
+  it('clears inlay hints when sole client detaches', function()
+    exec_lua([[vim.lsp.stop_client(client_id)]])
+    screen:expect({ grid = grid_without_inlay_hints, unchanged = true })
+  end)
+
+  it('does not clear inlay hints when one of several clients detaches', function()
+    exec_lua([[
+      server2 = _create_server({
+        capabilities = {
+          inlayHintProvider = true,
+        },
+        handlers = {
+          ['textDocument/inlayHint'] = function()
+            return {}
+          end,
+        }
+      })
+      client2 = vim.lsp.start({ name = 'dummy2', cmd = server2.cmd })
+      vim.lsp.inlay_hint.enable(bufnr)
+    ]])
+
+    exec_lua([[ vim.lsp.stop_client(client2) ]])
+    screen:expect({ grid = grid_with_inlay_hints, unchanged = true })
+  end)
+
+  describe('enable()', function()
+    it('clears/applies inlay hints when passed false/true/nil', function()
+      exec_lua([[vim.lsp.inlay_hint.enable(bufnr, false)]])
+      screen:expect({ grid = grid_without_inlay_hints, unchanged = true })
+
+      exec_lua([[vim.lsp.inlay_hint.enable(bufnr, true)]])
+      screen:expect({ grid = grid_with_inlay_hints, unchanged = true })
+
+      exec_lua([[vim.lsp.inlay_hint.enable(bufnr, not vim.lsp.inlay_hint.is_enabled(bufnr))]])
+      screen:expect({ grid = grid_without_inlay_hints, unchanged = true })
+
+      exec_lua([[vim.lsp.inlay_hint.enable(bufnr)]])
+      screen:expect({ grid = grid_with_inlay_hints, unchanged = true })
+    end)
+  end)
+
+  describe('get()', function()
+    it('returns filtered inlay hints', function()
+      --- @type lsp.InlayHint[]
+      local expected = vim.json.decode(response)
+      local expected2 = {
+        kind = 1,
+        paddingLeft = false,
+        label = ': int',
+        position = {
+          character = 10,
+          line = 2,
+        },
+        paddingRight = false,
+      }
+
+      exec_lua([[
+        local expected2 = ...
+        server2 = _create_server({
+          capabilities = {
+            inlayHintProvider = true,
+          },
+          handlers = {
+            ['textDocument/inlayHint'] = function()
+              return { expected2 }
+            end,
+          }
+        })
+        client2 = vim.lsp.start({ name = 'dummy2', cmd = server2.cmd })
+        vim.lsp.inlay_hint.enable(bufnr)
+      ]], expected2)
+
+      --- @type vim.lsp.inlay_hint.get.ret
+      local res = exec_lua([[return vim.lsp.inlay_hint.get()]])
+      eq(res, {
+        { bufnr = 1, client_id = 1, inlay_hint = expected[1] },
+        { bufnr = 1, client_id = 1, inlay_hint = expected[2] },
+        { bufnr = 1, client_id = 1, inlay_hint = expected[3] },
+        { bufnr = 1, client_id = 2, inlay_hint = expected2 },
+      })
+
+      --- @type vim.lsp.inlay_hint.get.ret
+      res = exec_lua([[return vim.lsp.inlay_hint.get({
+        range = {
+          start = { line = 2, character = 10 },
+          ["end"] = { line = 2, character = 10 },
+        },
+      })]])
+      eq(res, {
+        { bufnr = 1, client_id = 2, inlay_hint = expected2 },
+      })
+
+      --- @type vim.lsp.inlay_hint.get.ret
+      res = exec_lua([[return vim.lsp.inlay_hint.get({
+        bufnr = vim.api.nvim_get_current_buf(),
+        range = {
+          start = { line = 4, character = 18 },
+          ["end"] = { line = 5, character = 17 },
+        },
+      })]])
+      eq(res, {
+        { bufnr = 1, client_id = 1, inlay_hint = expected[2] },
+        { bufnr = 1, client_id = 1, inlay_hint = expected[3] },
+      })
+
+      --- @type vim.lsp.inlay_hint.get.ret
+      res = exec_lua([[return vim.lsp.inlay_hint.get({
+        bufnr = vim.api.nvim_get_current_buf() + 1,
+      })]])
+      eq(res, {})
+    end)
+  end)
+end)
diff --git a/test/functional/plugin/lsp/semantic_tokens_spec.lua b/test/functional/plugin/lsp/semantic_tokens_spec.lua
index 9c1ba86fe1..b7ac53f270 100644
--- a/test/functional/plugin/lsp/semantic_tokens_spec.lua
+++ b/test/functional/plugin/lsp/semantic_tokens_spec.lua
@@ -24,6 +24,27 @@ end)
 
 describe('semantic token highlighting', function()
 
+  local screen
+  before_each(function()
+    screen = Screen.new(40, 16)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = { bold = true, foreground = Screen.colors.Blue1 };
+      [2] = { foreground = Screen.colors.DarkCyan };
+      [3] = { foreground = Screen.colors.SlateBlue };
+      [4] = { bold = true, foreground = Screen.colors.SeaGreen };
+      [5] = { foreground = tonumber('0x6a0dad') };
+      [6] = { foreground = Screen.colors.Blue1 };
+      [7] = { bold = true, foreground = Screen.colors.DarkCyan };
+      [8] = { bold = true, foreground = Screen.colors.SlateBlue };
+      [9] = { bold = true, foreground = tonumber('0x6a0dad') };
+    }
+    command([[ hi link @lsp.type.namespace Type ]])
+    command([[ hi link @lsp.type.function Special ]])
+    command([[ hi link @lsp.type.comment Comment ]])
+    command([[ hi @lsp.mod.declaration gui=bold ]])
+  end)
+
   describe('general', function()
     local text = dedent([[
     #include <iostream>
@@ -58,24 +79,7 @@ describe('semantic token highlighting', function()
       "resultId":"2"
     }]]
 
-    local screen
     before_each(function()
-      screen = Screen.new(40, 16)
-      screen:attach()
-      screen:set_default_attr_ids {
-        [1] = { bold = true, foreground = Screen.colors.Blue1 };
-        [2] = { foreground = Screen.colors.DarkCyan };
-        [3] = { foreground = Screen.colors.SlateBlue };
-        [4] = { bold = true, foreground = Screen.colors.SeaGreen };
-        [5] = { foreground = tonumber('0x6a0dad') };
-        [6] = { foreground = Screen.colors.Blue1 };
-        [7] = { bold = true, foreground = Screen.colors.DarkCyan };
-        [8] = { bold = true, foreground = Screen.colors.SlateBlue };
-      }
-      command([[ hi link @namespace Type ]])
-      command([[ hi link @function Special ]])
-      command([[ hi @declaration gui=bold ]])
-
       exec_lua(create_server_definition)
       exec_lua([[
         local legend, response, edit_response = ...
@@ -102,6 +106,7 @@ describe('semantic token highlighting', function()
       exec_lua([[
         bufnr = vim.api.nvim_get_current_buf()
         vim.api.nvim_win_set_buf(0, bufnr)
+        vim.bo[bufnr].filetype = 'some-filetype'
         client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
       ]])
 
@@ -127,6 +132,46 @@ describe('semantic token highlighting', function()
       ]] }
     end)
 
+    it('use LspTokenUpdate and highlight_token', function()
+      exec_lua([[
+        vim.api.nvim_create_autocmd("LspTokenUpdate", {
+          callback = function(args)
+            local token = args.data.token
+            if token.type == "function" and token.modifiers.declaration then
+              vim.lsp.semantic_tokens.highlight_token(
+                token, args.buf, args.data.client_id, "Macro"
+              )
+            end
+          end,
+        })
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+
+      insert(text)
+
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {9:main}()                              |
+        {                                       |
+            int {7:x};                              |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {2:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]] }
+
+    end)
+
     it('buffer is unhighlighted when client is detached', function()
       exec_lua([[
         bufnr = vim.api.nvim_get_current_buf()
@@ -578,16 +623,33 @@ describe('semantic token highlighting', function()
         expected = {
           {
             line = 0,
-            modifiers = {
-              'declaration',
-              'globalScope',
-            },
+            modifiers = { declaration = true, globalScope = true },
             start_col = 6,
             end_col = 9,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
         },
+        expected_screen = function()
+          screen:expect{grid=[[
+            char* {7:foo} = "\n"^;                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'clangd-15 on C++',
@@ -613,67 +675,67 @@ int main()
         expected = {
           { -- main
             line = 1,
-            modifiers = { 'declaration', 'globalScope' },
+            modifiers = { declaration = true, globalScope = true },
             start_col = 4,
             end_col = 8,
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           { --  __cplusplus
             line = 3,
-            modifiers = { 'globalScope' },
+            modifiers = { globalScope = true },
             start_col = 9,
             end_col = 20,
             type = 'macro',
-            extmark_added = true,
+            marked = true,
           },
           { -- x
             line = 4,
-            modifiers = { 'declaration', 'readonly', 'functionScope' },
+            modifiers = { declaration = true, readonly = true, functionScope = true },
             start_col = 12,
             end_col = 13,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           { -- std
             line = 5,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 2,
             end_col = 5,
             type = 'namespace',
-            extmark_added = true,
+            marked = true,
           },
           { -- cout
             line = 5,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 7,
             end_col = 11,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           { -- x
             line = 5,
-            modifiers = { 'readonly', 'functionScope' },
+            modifiers = { readonly = true, functionScope = true },
             start_col = 15,
             end_col = 16,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           { -- std
             line = 5,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 20,
             end_col = 23,
             type = 'namespace',
-            extmark_added = true,
+            marked = true,
           },
           { -- endl
             line = 5,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 25,
             end_col = 29,
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           { -- #else comment #endif
             line = 6,
@@ -681,7 +743,7 @@ int main()
             start_col = 0,
             end_col = 7,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 7,
@@ -689,7 +751,7 @@ int main()
             start_col = 0,
             end_col = 11,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 8,
@@ -697,9 +759,29 @@ int main()
             start_col = 0,
             end_col = 8,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
         },
+        expected_screen = function()
+          screen:expect{grid=[[
+            #include <iostream>                     |
+            int {8:main}()                              |
+            {                                       |
+              #ifdef {5:__cplusplus}                    |
+              const int {7:x} = 1;                      |
+              {4:std}::{2:cout} << {2:x} << {4:std}::{3:endl};          |
+            {6:  #else}                                 |
+            {6:    comment}                             |
+            {6:  #endif}                                |
+            ^}                                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'sumneko_lua',
@@ -722,25 +804,45 @@ b = "as"]],
             start_col = 0,
             end_col = 10,
             type = 'comment', -- comment
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 1,
-            modifiers = { 'declaration' }, -- a
+            modifiers = { declaration = true }, -- a
             start_col = 6,
             end_col = 7,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 2,
-            modifiers = { 'static' }, -- b (global)
+            modifiers = { static = true }, -- b (global)
             start_col = 0,
             end_col = 1,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
         },
+        expected_screen = function()
+          screen:expect{grid=[[
+            {6:-- comment}                              |
+            local {7:a} = 1                             |
+            {2:b} = "as^"                                |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'rust-analyzer',
@@ -768,7 +870,7 @@ b = "as"]],
             start_col = 0,
             end_col = 3, -- pub
             type = 'keyword',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
@@ -776,15 +878,15 @@ b = "as"]],
             start_col = 4,
             end_col = 6, -- fn
             type = 'keyword',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
-            modifiers = { 'declaration', 'public' },
+            modifiers = { declaration = true, public = true },
             start_col = 7,
             end_col = 11, -- main
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
@@ -792,7 +894,7 @@ b = "as"]],
             start_col = 11,
             end_col = 12,
             type = 'parenthesis',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
@@ -800,7 +902,7 @@ b = "as"]],
             start_col = 12,
             end_col = 13,
             type = 'parenthesis',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
@@ -808,15 +910,15 @@ b = "as"]],
             start_col = 14,
             end_col = 15,
             type = 'brace',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 1,
-            modifiers = { 'controlFlow' },
+            modifiers = { controlFlow = true },
             start_col = 4,
             end_col = 9, -- break
             type = 'keyword',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 1,
@@ -824,7 +926,7 @@ b = "as"]],
             start_col = 10,
             end_col = 13, -- rust
             type = 'unresolvedReference',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 1,
@@ -832,15 +934,15 @@ b = "as"]],
             start_col = 13,
             end_col = 13,
             type = 'semicolon',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 2,
-            modifiers = { 'documentation' },
+            modifiers = { documentation = true },
             start_col = 4,
             end_col = 11,
             type = 'comment', -- /// what?
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 3,
@@ -848,9 +950,29 @@ b = "as"]],
             start_col = 0,
             end_col = 1,
             type = 'brace',
-            extmark_added = true,
+            marked = true,
           },
         },
+        expected_screen = function()
+          screen:expect{grid=[[
+            pub fn {8:main}() {                         |
+              break rust;                           |
+              //{6:/ what?}                             |
+            }                                       |
+            ^                                        |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
     }) do
       it(test.it, function()
@@ -877,6 +999,8 @@ b = "as"]],
 
         insert(test.text)
 
+        test.expected_screen()
+
         local highlights = exec_lua([[
           local semantic_tokens = vim.lsp.semantic_tokens
           return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
@@ -906,28 +1030,68 @@ b = "as"]],
           {
             line = 0,
             modifiers = {
-              'declaration',
-              'globalScope',
+              declaration = true,
+              globalScope = true,
             },
             start_col = 6,
             end_col = 9,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           }
         },
         expected2 = {
           {
             line = 1,
             modifiers = {
-              'declaration',
-              'globalScope',
+              declaration = true,
+              globalScope = true,
             },
             start_col = 6,
             end_col = 9,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           }
         },
+        expected_screen1 = function()
+        screen:expect{grid=[[
+          char* {7:foo} = "\n"^;                       |
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+                                                  |
+        ]]}
+        end,
+        expected_screen2 = function()
+          screen:expect{grid=[[
+            ^                                        |
+            char* {7:foo} = "\n";                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'response with multiple delta edits',
@@ -976,55 +1140,55 @@ int main()
             line = 2,
             start_col = 4,
             end_col = 8,
-            modifiers = { 'declaration', 'globalScope' },
+            modifiers = { declaration = true, globalScope = true },
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 4,
             start_col = 8,
             end_col = 9,
-            modifiers = { 'declaration', 'functionScope' },
+            modifiers = { declaration = true, functionScope = true },
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 5,
             start_col = 7,
             end_col = 18,
-            modifiers = { 'globalScope' },
+            modifiers = { globalScope = true },
             type = 'macro',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 6,
             start_col = 4,
             end_col = 7,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             type = 'namespace',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 6,
             start_col = 9,
             end_col = 13,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 6,
             start_col = 17,
             end_col = 18,
-            extmark_added = true,
-            modifiers = { 'functionScope' },
+            marked = true,
+            modifiers = { functionScope = true },
             type = 'variable',
           },
           {
             line = 7,
             start_col = 0,
             end_col = 5,
-            extmark_added = true,
+            marked = true,
             modifiers = {},
             type = 'comment',
           },
@@ -1034,7 +1198,7 @@ int main()
             modifiers = {},
             start_col = 0,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 9,
@@ -1042,7 +1206,7 @@ int main()
             end_col = 6,
             modifiers = {},
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           }
         },
         expected2 = {
@@ -1050,63 +1214,63 @@ int main()
             line = 2,
             start_col = 4,
             end_col = 8,
-            modifiers = { 'declaration', 'globalScope' },
+            modifiers = { declaration = true, globalScope = true },
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 4,
             start_col = 8,
             end_col = 9,
-            modifiers = { 'declaration', 'globalScope' },
+            modifiers = { declaration = true, globalScope = true },
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 5,
             end_col = 12,
             start_col = 11,
-            modifiers = { 'declaration', 'functionScope' },
+            modifiers = { declaration = true, functionScope = true },
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 6,
             start_col = 7,
             end_col = 18,
-            modifiers = { 'globalScope' },
+            modifiers = { globalScope = true },
             type = 'macro',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 7,
             start_col = 4,
             end_col = 7,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             type = 'namespace',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 7,
             start_col = 9,
             end_col = 13,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 7,
             start_col = 17,
             end_col = 18,
-            extmark_added = true,
-            modifiers = { 'globalScope' },
+            marked = true,
+            modifiers = { globalScope = true },
             type = 'function',
           },
           {
             line = 8,
             start_col = 0,
             end_col = 5,
-            extmark_added = true,
+            marked = true,
             modifiers = {},
             type = 'comment',
           },
@@ -1116,7 +1280,7 @@ int main()
             modifiers = {},
             start_col = 0,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 10,
@@ -1124,9 +1288,49 @@ int main()
             end_col = 6,
             modifiers = {},
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           }
         },
+        expected_screen1 = function()
+          screen:expect{grid=[[
+            #include <iostream>                     |
+                                                    |
+            int {8:main}()                              |
+            {                                       |
+                int {7:x};                              |
+            #ifdef {5:__cplusplus}                      |
+                {4:std}::{2:cout} << {2:x} << "\n";             |
+            {6:#else}                                   |
+            {6:    printf("%d\n", x);}                  |
+            {6:#endif}                                  |
+            ^}                                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
+        expected_screen2 = function()
+          screen:expect{grid=[[
+            #include <iostream>                     |
+                                                    |
+            int {8:main}()                              |
+            {                                       |
+                int {8:x}();                            |
+                double {7:y};                           |
+            #ifdef {5:__cplusplus}                      |
+                {4:std}::{2:cout} << {3:x} << "\n";             |
+            {6:#else}                                   |
+            {6:    printf("%d\n", x);}                  |
+            {6:^#endif}                                  |
+            }                                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'optional token_edit.data on deletion',
@@ -1146,16 +1350,56 @@ int main()
           {
             line = 0,
             modifiers = {
-              'declaration',
+              declaration = true,
             },
             start_col = 0,
             end_col = 6,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           }
         },
         expected2 = {
         },
+        expected_screen1 = function()
+          screen:expect{grid=[[
+            {7:string} = "test^"                         |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
+        expected_screen2 = function()
+          screen:expect{grid=[[
+            ^                                        |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
     }) do
       it(test.it, function()
@@ -1192,6 +1436,8 @@ int main()
 
         insert(test.text1)
 
+        test.expected_screen1()
+
         local highlights = exec_lua([[
           return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
         ]])
@@ -1204,10 +1450,12 @@ int main()
           exec_lua([[
             local text = ...
             vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, vim.fn.split(text, "\n"))
-            vim.wait(15) -- wait fot debounce
+            vim.wait(15) -- wait for debounce
           ]], test.text2)
         end
 
+        test.expected_screen2()
+
         highlights = exec_lua([[
           return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
         ]])
diff --git a/test/functional/plugin/lsp/snippet_spec.lua b/test/functional/plugin/lsp/snippet_spec.lua
index 7903885420..ba8bc7fe04 100644
--- a/test/functional/plugin/lsp/snippet_spec.lua
+++ b/test/functional/plugin/lsp/snippet_spec.lua
@@ -1,130 +1,71 @@
 local helpers = require('test.functional.helpers')(after_each)
-local snippet = require('vim.lsp._snippet')
+local snippet = require('vim.lsp._snippet_grammar')
+local type = snippet.NodeType
 
 local eq = helpers.eq
 local exec_lua = helpers.exec_lua
 
-describe('vim.lsp._snippet', function()
+describe('vim.lsp._snippet_grammar', function()
   before_each(helpers.clear)
   after_each(helpers.clear)
 
   local parse = function(...)
-    return exec_lua('return require("vim.lsp._snippet").parse(...)', ...)
+    local res = exec_lua('return require("vim.lsp._snippet_grammar").parse(...)', ...)
+    return res.data.children
   end
 
-  it('should parse only text', function()
+  it('parses only text', function()
     eq({
-      type = snippet.NodeType.SNIPPET,
-      children = {
-        {
-          type = snippet.NodeType.TEXT,
-          raw = 'TE\\$\\}XT',
-          esc = 'TE$}XT',
-        },
-      },
+      { type = type.Text, data = { text = 'TE$}XT' } },
     }, parse('TE\\$\\}XT'))
   end)
 
-  it('should parse tabstop', function()
+  it('parses tabstops', function()
     eq({
-      type = snippet.NodeType.SNIPPET,
-      children = {
-        {
-          type = snippet.NodeType.TABSTOP,
-          tabstop = 1,
-        },
-        {
-          type = snippet.NodeType.TABSTOP,
-          tabstop = 2,
-        },
-      },
+      { type = type.Tabstop, data = { tabstop = 1 } },
+      { type = type.Tabstop, data = { tabstop = 2 } },
     }, parse('$1${2}'))
   end)
 
-  it('should parse placeholders', function()
+  it('parses nested placeholders', function()
     eq({
-      type = snippet.NodeType.SNIPPET,
-      children = {
-        {
-          type = snippet.NodeType.PLACEHOLDER,
+      {
+        type = type.Placeholder,
+        data = {
           tabstop = 1,
-          children = {
-            {
-              type = snippet.NodeType.PLACEHOLDER,
+          value = {
+            type = type.Placeholder,
+            data = {
               tabstop = 2,
-              children = {
-                {
-                  type = snippet.NodeType.TEXT,
-                  raw = 'TE\\$\\}XT',
-                  esc = 'TE$}XT',
-                },
-                {
-                  type = snippet.NodeType.TABSTOP,
-                  tabstop = 3,
-                },
-                {
-                  type = snippet.NodeType.TABSTOP,
-                  tabstop = 1,
-                  transform = {
-                    type = snippet.NodeType.TRANSFORM,
-                    pattern = 'regex',
-                    option = 'i',
-                    format = {
-                      {
-                        type = snippet.NodeType.FORMAT,
-                        capture_index = 1,
-                        modifier = 'upcase',
-                      },
-                    },
-                  },
-                },
-                {
-                  type = snippet.NodeType.TEXT,
-                  raw = 'TE\\$\\}XT',
-                  esc = 'TE$}XT',
-                },
-              },
+              value = { type = type.Tabstop, data = { tabstop = 3 } },
             },
           },
         },
       },
-    }, parse('${1:${2:TE\\$\\}XT$3${1/regex/${1:/upcase}/i}TE\\$\\}XT}}'))
+    }, parse('${1:${2:${3}}}'))
   end)
 
-  it('should parse variables', function()
+  it('parses variables', function()
     eq({
-      type = snippet.NodeType.SNIPPET,
-      children = {
-        {
-          type = snippet.NodeType.VARIABLE,
-          name = 'VAR',
-        },
-        {
-          type = snippet.NodeType.VARIABLE,
+      { type = type.Variable, data = { name = 'VAR' } },
+      { type = type.Variable, data = { name = 'VAR' } },
+      {
+        type = type.Variable,
+        data = {
           name = 'VAR',
+          default = { type = type.Tabstop, data = { tabstop = 1 } },
         },
-        {
-          type = snippet.NodeType.VARIABLE,
+      },
+      {
+        type = type.Variable,
+        data = {
           name = 'VAR',
-          children = {
+          regex = 'regex',
+          options = '',
+          format = {
             {
-              type = snippet.NodeType.TABSTOP,
-              tabstop = 1,
-            },
-          },
-        },
-        {
-          type = snippet.NodeType.VARIABLE,
-          name = 'VAR',
-          transform = {
-            type = snippet.NodeType.TRANSFORM,
-            pattern = 'regex',
-            format = {
-              {
-                type = snippet.NodeType.FORMAT,
-                capture_index = 1,
-                modifier = 'upcase',
-              },
+              type = type.Format,
+              data = { capture = 1, modifier = 'upcase' },
             },
           },
         },
@@ -132,105 +73,99 @@ describe('vim.lsp._snippet', function()
     }, parse('$VAR${VAR}${VAR:$1}${VAR/regex/${1:/upcase}/}'))
   end)
 
-  it('should parse choice', function()
+  it('parses choice', function()
     eq({
-      type = snippet.NodeType.SNIPPET,
-      children = {
-        {
-          type = snippet.NodeType.CHOICE,
-          tabstop = 1,
-          items = {
-            ',',
-            '|',
-          },
-        },
+      {
+        type = type.Choice,
+        data = { tabstop = 1, values = { ',', '|' } },
       },
     }, parse('${1|\\,,\\||}'))
   end)
 
-  it('should parse format', function()
-    eq({
-      type = snippet.NodeType.SNIPPET,
-      children = {
+  it('parses format', function()
+    eq(
+      {
         {
-          type = snippet.NodeType.VARIABLE,
-          name = 'VAR',
-          transform = {
-            type = snippet.NodeType.TRANSFORM,
-            pattern = 'regex',
+          type = type.Variable,
+          data = {
+            name = 'VAR',
+            regex = 'regex',
+            options = '',
             format = {
               {
-                type = snippet.NodeType.FORMAT,
-                capture_index = 1,
-                modifier = 'upcase',
+                type = type.Format,
+                data = { capture = 1, modifier = 'upcase' },
               },
               {
-                type = snippet.NodeType.FORMAT,
-                capture_index = 1,
-                if_text = 'if_text',
-                else_text = '',
+                type = type.Format,
+                data = { capture = 1, if_text = 'if_text' },
               },
               {
-                type = snippet.NodeType.FORMAT,
-                capture_index = 1,
-                if_text = '',
-                else_text = 'else_text',
+                type = type.Format,
+                data = { capture = 1, else_text = 'else_text' },
               },
               {
-                type = snippet.NodeType.FORMAT,
-                capture_index = 1,
-                else_text = 'else_text',
-                if_text = 'if_text',
+                type = type.Format,
+                data = { capture = 1, if_text = 'if_text', else_text = 'else_text' },
               },
               {
-                type = snippet.NodeType.FORMAT,
-                capture_index = 1,
-                if_text = '',
-                else_text = 'else_text',
+                type = type.Format,
+                data = { capture = 1, else_text = 'else_text' },
               },
             },
           },
         },
       },
-    }, parse('${VAR/regex/${1:/upcase}${1:+if_text}${1:-else_text}${1:?if_text:else_text}${1:else_text}/}'))
+      parse(
+        '${VAR/regex/${1:/upcase}${1:+if_text}${1:-else_text}${1:?if_text:else_text}${1:else_text}/}'
+      )
+    )
   end)
 
-  it('should parse empty strings', function()
+  it('parses empty strings', function()
     eq({
-      children = {
-        {
-          children = { {
-            esc = '',
-            raw = '',
-            type = 7,
-          } },
+      {
+        type = type.Placeholder,
+        data = {
           tabstop = 1,
-          type = 2,
-        },
-        {
-          esc = ' ',
-          raw = ' ',
-          type = 7,
+          value = { type = type.Text, data = { text = '' } },
         },
-        {
+      },
+      {
+        type = type.Text,
+        data = { text = ' ' },
+      },
+      {
+        type = type.Variable,
+        data = {
           name = 'VAR',
-          transform = {
-            format = {
-              {
-                capture_index = 1,
-                else_text = '',
-                if_text = '',
-                type = 6,
-              },
+          regex = 'erg',
+          format = {
+            {
+              type = type.Format,
+              data = { capture = 1, if_text = '' },
             },
-            option = 'g',
-            pattern = 'erg',
-            type = 5,
           },
-          type = 3,
+          options = 'g',
         },
       },
-      type = 0,
-    }, parse('${1:} ${VAR/erg/${1:?:}/g}'))
+    }, parse('${1:} ${VAR/erg/${1:+}/g}'))
+  end)
+
+  it('parses closing curly brace as text', function()
+    eq(
+      {
+        { type = type.Text, data = { text = 'function ' } },
+        { type = type.Tabstop, data = { tabstop = 1 } },
+        { type = type.Text, data = { text = '() {\n  ' } },
+        { type = type.Tabstop, data = { tabstop = 0 } },
+        { type = type.Text, data = { text = '\n}' } },
+      },
+      parse(table.concat({
+        'function $1() {',
+        '  $0',
+        '}',
+      }, '\n'))
+    )
   end)
 end)
diff --git a/test/functional/plugin/lsp/utils_spec.lua b/test/functional/plugin/lsp/utils_spec.lua
new file mode 100644
index 0000000000..804dc32f0d
--- /dev/null
+++ b/test/functional/plugin/lsp/utils_spec.lua
@@ -0,0 +1,226 @@
+local helpers = require('test.functional.helpers')(after_each)
+local Screen = require('test.functional.ui.screen')
+local feed = helpers.feed
+
+local eq = helpers.eq
+local exec_lua = helpers.exec_lua
+
+describe('vim.lsp.util', function()
+  before_each(helpers.clear)
+
+  describe('stylize_markdown', function()
+    local stylize_markdown = function(content, opts)
+      return exec_lua([[
+        local bufnr = vim.uri_to_bufnr("file:///fake/uri")
+        vim.fn.bufload(bufnr)
+
+        local args = { ... }
+        local content = args[1]
+        local opts = args[2]
+        local stripped_content = vim.lsp.util.stylize_markdown(bufnr, content, opts)
+
+        return stripped_content
+      ]], content, opts)
+    end
+
+    it('code fences', function()
+      local lines = {
+        "```lua",
+        "local hello = 'world'",
+        "```",
+      }
+      local expected = {
+        "local hello = 'world'",
+      }
+      local opts = {}
+      eq(expected, stylize_markdown(lines, opts))
+    end)
+
+    it('code fences with whitespace surrounded info string', function()
+      local lines = {
+        "```   lua   ",
+        "local hello = 'world'",
+        "```",
+      }
+      local expected = {
+        "local hello = 'world'",
+      }
+      local opts = {}
+      eq(expected, stylize_markdown(lines, opts))
+    end)
+
+    it('adds separator after code block', function()
+      local lines = {
+        "```lua",
+        "local hello = 'world'",
+        "```",
+        "",
+        "something",
+      }
+      local expected = {
+        "local hello = 'world'",
+        "─────────────────────",
+        "something",
+      }
+      local opts = { separator = true }
+      eq(expected, stylize_markdown(lines, opts))
+    end)
+
+    it('replaces supported HTML entities', function()
+      local lines = {
+        "1 &lt; 2",
+        "3 &gt; 2",
+        "&quot;quoted&quot;",
+        "&apos;apos&apos;",
+        "&ensp; &emsp;",
+        "&amp;",
+      }
+      local expected = {
+        "1 < 2",
+        "3 > 2",
+        '"quoted"',
+        "'apos'",
+        "   ",
+        "&",
+      }
+      local opts = {}
+      eq(expected, stylize_markdown(lines, opts))
+    end)
+  end)
+
+  describe('normalize_markdown', function ()
+    it('collapses consecutive blank lines', function ()
+      local result = exec_lua [[
+        local lines = {
+          'foo',
+          '',
+          '',
+          '',
+          'bar',
+          '',
+          'baz'
+        }
+        return vim.lsp.util._normalize_markdown(lines)
+      ]]
+      local expected = {'foo', '', 'bar', '', 'baz'}
+      eq(expected, result)
+    end)
+
+    it('removes preceding and trailing empty lines', function ()
+      local result = exec_lua [[
+        local lines = {
+          '',
+          'foo',
+          'bar',
+          '',
+          ''
+        }
+        return vim.lsp.util._normalize_markdown(lines)
+      ]]
+      local expected = {'foo', 'bar'}
+      eq(expected, result)
+    end)
+  end)
+
+  describe("make_floating_popup_options", function ()
+
+    local function assert_anchor(anchor_bias, expected_anchor)
+        local opts = exec_lua([[
+          local args = { ... }
+          local anchor_bias = args[1]
+          return vim.lsp.util.make_floating_popup_options(30, 10, { anchor_bias = anchor_bias })
+        ]], anchor_bias)
+
+        eq(expected_anchor, string.sub(opts.anchor, 1, 1))
+    end
+
+    local screen
+    before_each(function ()
+      helpers.clear()
+      screen = Screen.new(80, 80)
+      screen:attach()
+      feed("79i<CR><Esc>") -- fill screen with empty lines
+    end)
+
+    describe('when on the first line it places window below', function ()
+      before_each(function ()
+        feed('gg')
+      end)
+
+      it('for anchor_bias = "auto"', function ()
+        assert_anchor('auto', 'N')
+      end)
+
+      it('for anchor_bias = "above"', function ()
+        assert_anchor('above', 'N')
+      end)
+
+      it('for anchor_bias = "below"', function ()
+        assert_anchor('below', 'N')
+      end)
+    end)
+
+    describe('when on the last line it places window above', function ()
+      before_each(function ()
+        feed('G')
+      end)
+
+      it('for anchor_bias = "auto"', function ()
+        assert_anchor('auto', 'S')
+      end)
+
+      it('for anchor_bias = "above"', function ()
+        assert_anchor('above', 'S')
+      end)
+
+      it('for anchor_bias = "below"', function ()
+        assert_anchor('below', 'S')
+      end)
+    end)
+
+    describe('with 20 lines above, 59 lines below', function ()
+      before_each(function ()
+        feed('gg20j')
+      end)
+
+      it('places window below for anchor_bias = "auto"', function ()
+        assert_anchor('auto', 'N')
+      end)
+
+      it('places window above for anchor_bias = "above"', function ()
+        assert_anchor('above', 'S')
+      end)
+
+      it('places window below for anchor_bias = "below"', function ()
+        assert_anchor('below', 'N')
+      end)
+    end)
+
+    describe('with 59 lines above, 20 lines below', function ()
+      before_each(function ()
+        feed('G20k')
+      end)
+
+      it('places window above for anchor_bias = "auto"', function ()
+        assert_anchor('auto', 'S')
+      end)
+
+      it('places window above for anchor_bias = "above"', function ()
+        assert_anchor('above', 'S')
+      end)
+
+      it('places window below for anchor_bias = "below"', function ()
+        assert_anchor('below', 'N')
+      end)
+
+      it('bordered window truncates dimensions correctly', function ()
+        local opts = exec_lua([[
+          return vim.lsp.util.make_floating_popup_options(100, 100, { border = 'single' })
+        ]])
+
+        eq(56, opts.height)
+      end)
+    end)
+  end)
+
+end)
diff --git a/test/functional/plugin/lsp/watchfiles_spec.lua b/test/functional/plugin/lsp/watchfiles_spec.lua
new file mode 100644
index 0000000000..a8260e0c98
--- /dev/null
+++ b/test/functional/plugin/lsp/watchfiles_spec.lua
@@ -0,0 +1,222 @@
+local helpers = require('test.functional.helpers')(after_each)
+
+local eq = helpers.eq
+local exec_lua = helpers.exec_lua
+
+describe('vim.lsp._watchfiles', function()
+  before_each(helpers.clear)
+  after_each(helpers.clear)
+
+  local match = function(...)
+    return exec_lua('return require("vim.lsp._watchfiles")._match(...)', ...)
+  end
+
+  describe('glob matching', function()
+    it('should match literal strings', function()
+      eq(true, match('', ''))
+      eq(false, match('', 'a'))
+      eq(true, match('a', 'a'))
+      eq(true, match('/', '/'))
+      eq(true, match('abc', 'abc'))
+      eq(false, match('abc', 'abcdef'))
+      eq(false, match('abc', 'a'))
+      eq(false, match('abc', 'bc'))
+      eq(false, match('a', 'b'))
+      eq(false, match('.', 'a'))
+      eq(true, match('$', '$'))
+      eq(true, match('/dir', '/dir'))
+      eq(true, match('dir/', 'dir/'))
+      eq(true, match('dir/subdir', 'dir/subdir'))
+      eq(false, match('dir/subdir', 'subdir'))
+      eq(false, match('dir/subdir', 'dir/subdir/file'))
+      eq(true, match('🤠', '🤠'))
+    end)
+
+    it('should match * wildcards', function()
+      eq(false, match('*', ''))
+      eq(true, match('*', 'a'))
+      eq(false, match('*', '/'))
+      eq(false, match('*', '/a'))
+      eq(false, match('*', 'a/'))
+      eq(true, match('*', 'aaa'))
+      eq(true, match('*a', 'aa'))
+      eq(true, match('*a', 'abca'))
+      eq(true, match('*.txt', 'file.txt'))
+      eq(false, match('*.txt', 'file.txtxt'))
+      eq(false, match('*.txt', 'dir/file.txt'))
+      eq(false, match('*.txt', '/dir/file.txt'))
+      eq(false, match('*.txt', 'C:/dir/file.txt'))
+      eq(false, match('*.dir', 'test.dir/file'))
+      eq(true, match('file.*', 'file.txt'))
+      eq(false, match('file.*', 'not-file.txt'))
+      eq(true, match('*/file.txt', 'dir/file.txt'))
+      eq(false, match('*/file.txt', 'dir/subdir/file.txt'))
+      eq(false, match('*/file.txt', '/dir/file.txt'))
+      eq(true, match('dir/*', 'dir/file.txt'))
+      eq(false, match('dir/*', 'dir'))
+      eq(false, match('dir/*.txt', 'file.txt'))
+      eq(true, match('dir/*.txt', 'dir/file.txt'))
+      eq(false, match('dir/*.txt', 'dir/subdir/file.txt'))
+      eq(false, match('dir/*/file.txt', 'dir/file.txt'))
+      eq(true, match('dir/*/file.txt', 'dir/subdir/file.txt'))
+      eq(false, match('dir/*/file.txt', 'dir/subdir/subdir/file.txt'))
+
+      -- TODO: The spec does not describe this, but VSCode only interprets ** when it's by
+      -- itself in a path segment, and otherwise interprets ** as consecutive * directives.
+      -- The following tests show how this behavior should work, but is not yet fully implemented.
+      -- Currently, "a**" parses incorrectly as "a" "**" and "**a" parses correctly as "*" "*" "a".
+      -- see: https://github.com/microsoft/vscode/blob/eef30e7165e19b33daa1e15e92fa34ff4a5df0d3/src/vs/base/common/glob.ts#L112
+      eq(true, match('a**', 'abc')) -- '**' should parse as two '*'s when not by itself in a path segment
+      eq(true, match('**c', 'abc'))
+      -- eq(false, match('a**', 'ab')) -- each '*' should still represent at least one character
+      eq(false, match('**c', 'bc'))
+      eq(true, match('a**', 'abcd'))
+      eq(true, match('**d', 'abcd'))
+      -- eq(false, match('a**', 'abc/d'))
+      eq(false, match('**d', 'abc/d'))
+    end)
+
+    it('should match ? wildcards', function()
+      eq(false, match('?', ''))
+      eq(true, match('?', 'a'))
+      eq(false, match('??', 'a'))
+      eq(false, match('?', 'ab'))
+      eq(true, match('??', 'ab'))
+      eq(true, match('a?c', 'abc'))
+      eq(false, match('a?c', 'a/c'))
+    end)
+
+    it('should match ** wildcards', function()
+      eq(true, match('**', ''))
+      eq(true, match('**', 'a'))
+      eq(true, match('**', '/'))
+      eq(true, match('**', 'a/'))
+      eq(true, match('**', '/a'))
+      eq(true, match('**', 'C:/a'))
+      eq(true, match('**', 'a/a'))
+      eq(true, match('**', 'a/a/a'))
+      eq(false, match('/**', '')) -- /** matches leading / literally
+      eq(true, match('/**', '/'))
+      eq(true, match('/**', '/a/b/c'))
+      eq(true, match('**/', '')) -- **/ absorbs trailing /
+      eq(true, match('**/', '/a/b/c'))
+      eq(true, match('**/**', ''))
+      eq(true, match('**/**', 'a'))
+      eq(false, match('a/**', ''))
+      eq(false, match('a/**', 'a'))
+      eq(true, match('a/**', 'a/b'))
+      eq(true, match('a/**', 'a/b/c'))
+      eq(false, match('a/**', 'b/a'))
+      eq(false, match('a/**', '/a'))
+      eq(false, match('**/a', ''))
+      eq(true, match('**/a', 'a'))
+      eq(false, match('**/a', 'a/b'))
+      eq(true, match('**/a', '/a'))
+      eq(true, match('**/a', '/b/a'))
+      eq(true, match('**/a', '/c/b/a'))
+      eq(true, match('**/a', '/a/a'))
+      eq(true, match('**/a', '/abc/a'))
+      eq(false, match('a/**/c', 'a'))
+      eq(false, match('a/**/c', 'c'))
+      eq(true, match('a/**/c', 'a/c'))
+      eq(true, match('a/**/c', 'a/b/c'))
+      eq(true, match('a/**/c', 'a/b/b/c'))
+      eq(false, match('**/a/**', 'a'))
+      eq(true, match('**/a/**', 'a/'))
+      eq(false, match('**/a/**', '/dir/a'))
+      eq(false, match('**/a/**', 'dir/a'))
+      eq(true, match('**/a/**', 'dir/a/'))
+      eq(true, match('**/a/**', 'a/dir'))
+      eq(true, match('**/a/**', 'dir/a/dir'))
+      eq(true, match('**/a/**', '/a/dir'))
+      eq(true, match('**/a/**', 'C:/a/dir'))
+      eq(false, match('**/a/**', 'a.txt'))
+    end)
+
+    it('should match {} groups', function()
+      eq(true, match('{}', ''))
+      eq(false, match('{}', 'a'))
+      eq(true, match('a{}', 'a'))
+      eq(true, match('{}a', 'a'))
+      eq(true, match('{,}', ''))
+      eq(true, match('{a,}', ''))
+      eq(true, match('{a,}', 'a'))
+      eq(true, match('{a}', 'a'))
+      eq(false, match('{a}', 'aa'))
+      eq(false, match('{a}', 'ab'))
+      eq(true, match('{a?c}', 'abc'))
+      eq(false, match('{ab}', 'a'))
+      eq(false, match('{ab}', 'b'))
+      eq(true, match('{ab}', 'ab'))
+      eq(true, match('{a,b}', 'a'))
+      eq(true, match('{a,b}', 'b'))
+      eq(false, match('{a,b}', 'ab'))
+      eq(true, match('{ab,cd}', 'ab'))
+      eq(false, match('{ab,cd}', 'a'))
+      eq(true, match('{ab,cd}', 'cd'))
+      eq(true, match('{a,b,c}', 'c'))
+      eq(true, match('{a,{b,c}}', 'c'))
+    end)
+
+    it('should match [] groups', function()
+      eq(true, match('[]', '[]')) -- empty [] is a literal
+      eq(false, match('[a-z]', ''))
+      eq(true, match('[a-z]', 'a'))
+      eq(false, match('[a-z]', 'ab'))
+      eq(true, match('[a-z]', 'z'))
+      eq(true, match('[a-z]', 'j'))
+      eq(false, match('[a-f]', 'j'))
+      eq(false, match('[a-z]', '`')) -- 'a' - 1
+      eq(false, match('[a-z]', '{')) -- 'z' + 1
+      eq(false, match('[a-z]', 'A'))
+      eq(false, match('[a-z]', '5'))
+      eq(true, match('[A-Z]', 'A'))
+      eq(true, match('[A-Z]', 'Z'))
+      eq(true, match('[A-Z]', 'J'))
+      eq(false, match('[A-Z]', '@')) -- 'A' - 1
+      eq(false, match('[A-Z]', '[')) -- 'Z' + 1
+      eq(false, match('[A-Z]', 'a'))
+      eq(false, match('[A-Z]', '5'))
+      eq(true, match('[a-zA-Z0-9]', 'z'))
+      eq(true, match('[a-zA-Z0-9]', 'Z'))
+      eq(true, match('[a-zA-Z0-9]', '9'))
+      eq(false, match('[a-zA-Z0-9]', '&'))
+    end)
+
+    it('should match [!...] groups', function()
+      eq(true, match('[!]', '[!]')) -- [!] is a literal
+      eq(false, match('[!a-z]', ''))
+      eq(false, match('[!a-z]', 'a'))
+      eq(false, match('[!a-z]', 'z'))
+      eq(false, match('[!a-z]', 'j'))
+      eq(true, match('[!a-f]', 'j'))
+      eq(false, match('[!a-f]', 'jj'))
+      eq(true, match('[!a-z]', '`')) -- 'a' - 1
+      eq(true, match('[!a-z]', '{')) -- 'z' + 1
+      eq(false, match('[!a-zA-Z0-9]', 'a'))
+      eq(false, match('[!a-zA-Z0-9]', 'A'))
+      eq(false, match('[!a-zA-Z0-9]', '0'))
+      eq(true, match('[!a-zA-Z0-9]', '!'))
+    end)
+
+    it('should match complex patterns', function()
+      eq(false, match('**/*.{c,h}', ''))
+      eq(false, match('**/*.{c,h}', 'c'))
+      eq(false, match('**/*.{c,h}', 'file.m'))
+      eq(true, match('**/*.{c,h}', 'file.c'))
+      eq(true, match('**/*.{c,h}', 'file.h'))
+      eq(true, match('**/*.{c,h}', '/file.c'))
+      eq(true, match('**/*.{c,h}', 'dir/subdir/file.c'))
+      eq(true, match('**/*.{c,h}', 'dir/subdir/file.h'))
+      eq(true, match('**/*.{c,h}', '/dir/subdir/file.c'))
+      eq(true, match('**/*.{c,h}', 'C:/dir/subdir/file.c'))
+      eq(true, match('/dir/**/*.{c,h}', '/dir/file.c'))
+      eq(false, match('/dir/**/*.{c,h}', 'dir/file.c'))
+      eq(true, match('/dir/**/*.{c,h}', '/dir/subdir/subdir/file.c'))
+
+      eq(true, match('{[0-9],[a-z]}', '0'))
+      eq(true, match('{[0-9],[a-z]}', 'a'))
+      eq(false, match('{[0-9],[a-z]}', 'A'))
+    end)
+  end)
+end)