diff options
| -rw-r--r-- | runtime/doc/lsp.txt | 49 | ||||
| -rw-r--r-- | runtime/doc/lua.txt | 1 | ||||
| -rw-r--r-- | runtime/doc/news.txt | 8 | ||||
| -rw-r--r-- | runtime/lua/vim/highlight.lua | 1 | ||||
| -rw-r--r-- | runtime/lua/vim/lsp.lua | 9 | ||||
| -rw-r--r-- | runtime/lua/vim/lsp/protocol.lua | 55 | ||||
| -rw-r--r-- | runtime/lua/vim/lsp/semantic_tokens.lua | 644 | ||||
| -rwxr-xr-x | scripts/gen_vimdoc.py | 1 | ||||
| -rw-r--r-- | src/nvim/highlight_group.c | 13 | ||||
| -rw-r--r-- | test/functional/plugin/lsp/helpers.lua | 178 | ||||
| -rw-r--r-- | test/functional/plugin/lsp/semantic_tokens_spec.lua | 910 | ||||
| -rw-r--r-- | test/functional/plugin/lsp_spec.lua | 181 |
12 files changed, 1876 insertions, 174 deletions
diff --git a/runtime/doc/lsp.txt b/runtime/doc/lsp.txt index 4c4403c38f..22593d8331 100644 --- a/runtime/doc/lsp.txt +++ b/runtime/doc/lsp.txt @@ -1320,6 +1320,55 @@ save({lenses}, {bufnr}, {client_id}) *vim.lsp.codelens.save()* ============================================================================== +Lua module: vim.lsp.semantic_tokens *lsp-semantic_tokens* + +force_refresh({bufnr}) *vim.lsp.semantic_tokens.force_refresh()* + Force a refresh of all semantic tokens + + Only has an effect if the buffer is currently active for semantic token + highlighting (|vim.lsp.semantic_tokens.start()| has been called for it) + + Parameters: ~ + • {bufnr} (nil|number) default: current buffer + +start({bufnr}, {client_id}, {opts}) *vim.lsp.semantic_tokens.start()* + Start the semantic token highlighting engine for the given buffer with the + given client. The client must already be attached to the buffer. + + NOTE: This is currently called automatically by + |vim.lsp.buf_attach_client()|. To opt-out of semantic highlighting with a + server that supports it, you can delete the semanticTokensProvider table + from the {server_capabilities} of your client in your |LspAttach| callback + or your configuration's `on_attach` callback. + + >lua + + client.server_capabilities.semanticTokensProvider = nil +< + + Parameters: ~ + • {bufnr} (number) + • {client_id} (number) + • {opts} (nil|table) Optional keyword arguments + • debounce (number, default: 200): Debounce token + requests to the server by the given number in + milliseconds + +stop({bufnr}, {client_id}) *vim.lsp.semantic_tokens.stop()* + Stop the semantic token highlighting engine for the given buffer with the + given client. + + NOTE: This is automatically called by a |LspDetach| autocmd that is set up + as part of `start()`, so you should only need this function to manually + disengage the semantic token engine without fully detaching the LSP client + from the buffer. + + Parameters: ~ + • {bufnr} (number) + • {client_id} (number) + + +============================================================================== Lua module: vim.lsp.handlers *lsp-handlers* hover({_}, {result}, {ctx}, {config}) *vim.lsp.handlers.hover()* diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt index 2682725167..5a1c186192 100644 --- a/runtime/doc/lua.txt +++ b/runtime/doc/lua.txt @@ -604,6 +604,7 @@ vim.highlight.priorities *vim.highlight.priorities* Table with default priorities used for highlighting: • `syntax`: `50`, used for standard syntax highlighting • `treesitter`: `100`, used for tree-sitter-based highlighting + • `semantic_tokens`: `125`, used for LSP semantic token highlighting • `diagnostics`: `150`, used for code analysis such as diagnostics • `user`: `200`, used for user-triggered highlights such as LSP document symbols or `on_yank` autocommands diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt index 881faaa84e..bd0d1cfc5b 100644 --- a/runtime/doc/news.txt +++ b/runtime/doc/news.txt @@ -39,6 +39,14 @@ NEW FEATURES *news-features* The following new APIs or features were added. +• Added support for semantic token highlighting to the LSP client. This + functionality is enabled by default when a client that supports this feature + is attached to a buffer. Opt-out can be performed by deleting the + `semanticTokensProvider` from the LSP client's {server_capabilities} in the + `LspAttach` callback. + + See |lsp-semantic_tokens| for more information. + • |vim.treesitter.show_tree()| opens a split window showing a text representation of the nodes in a language tree for the current buffer. diff --git a/runtime/lua/vim/highlight.lua b/runtime/lua/vim/highlight.lua index 0fde515bd9..20ad48dd27 100644 --- a/runtime/lua/vim/highlight.lua +++ b/runtime/lua/vim/highlight.lua @@ -5,6 +5,7 @@ local M = {} M.priorities = { syntax = 50, treesitter = 100, + semantic_tokens = 125, diagnostics = 150, user = 200, } diff --git a/runtime/lua/vim/lsp.lua b/runtime/lua/vim/lsp.lua index 9c42e9df52..3d3c856fcb 100644 --- a/runtime/lua/vim/lsp.lua +++ b/runtime/lua/vim/lsp.lua @@ -4,6 +4,7 @@ local lsp_rpc = require('vim.lsp.rpc') local protocol = require('vim.lsp.protocol') local util = require('vim.lsp.util') local sync = require('vim.lsp.sync') +local semantic_tokens = require('vim.lsp.semantic_tokens') local api = vim.api local nvim_err_writeln, nvim_buf_get_lines, nvim_command, nvim_buf_get_option, nvim_exec_autocmds = @@ -25,6 +26,7 @@ local lsp = { buf = require('vim.lsp.buf'), diagnostic = require('vim.lsp.diagnostic'), codelens = require('vim.lsp.codelens'), + semantic_tokens = semantic_tokens, util = util, -- Allow raw RPC access. @@ -56,6 +58,8 @@ lsp._request_name_to_capability = { ['textDocument/formatting'] = { 'documentFormattingProvider' }, ['textDocument/completion'] = { 'completionProvider' }, ['textDocument/documentHighlight'] = { 'documentHighlightProvider' }, + ['textDocument/semanticTokens/full'] = { 'semanticTokensProvider' }, + ['textDocument/semanticTokens/full/delta'] = { 'semanticTokensProvider' }, } -- TODO improve handling of scratch buffers with LSP attached. @@ -1526,6 +1530,11 @@ function lsp.start_client(config) -- TODO(ashkan) handle errors. pcall(config.on_attach, client, bufnr) end + + if vim.tbl_get(client.server_capabilities, 'semanticTokensProvider', 'full') then + semantic_tokens.start(bufnr, client.id) + end + client.attached_buffers[bufnr] = true end diff --git a/runtime/lua/vim/lsp/protocol.lua b/runtime/lua/vim/lsp/protocol.lua index 925115d056..dfbd01b8f8 100644 --- a/runtime/lua/vim/lsp/protocol.lua +++ b/runtime/lua/vim/lsp/protocol.lua @@ -629,6 +629,58 @@ export interface WorkspaceClientCapabilities { function protocol.make_client_capabilities() return { textDocument = { + semanticTokens = { + dynamicRegistration = false, + tokenTypes = { + 'namespace', + 'type', + 'class', + 'enum', + 'interface', + 'struct', + 'typeParameter', + 'parameter', + 'variable', + 'property', + 'enumMember', + 'event', + 'function', + 'method', + 'macro', + 'keyword', + 'modifier', + 'comment', + 'string', + 'number', + 'regexp', + 'operator', + 'decorator', + }, + tokenModifiers = { + 'declaration', + 'definition', + 'readonly', + 'static', + 'deprecated', + 'abstract', + 'async', + 'modification', + 'documentation', + 'defaultLibrary', + }, + formats = { 'relative' }, + requests = { + -- TODO(jdrouhard): Add support for this + range = false, + full = { delta = true }, + }, + + overlappingTokenSupport = true, + -- TODO(jdrouhard): Add support for this + multilineTokenSupport = false, + serverCancelSupport = false, + augmentsSyntaxTokens = true, + }, synchronization = { dynamicRegistration = false, @@ -772,6 +824,9 @@ function protocol.make_client_capabilities() workspaceEdit = { resourceOperations = { 'rename', 'create', 'delete' }, }, + semanticTokens = { + refreshSupport = true, + }, }, callHierarchy = { dynamicRegistration = false, diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua new file mode 100644 index 0000000000..99cdc20f54 --- /dev/null +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -0,0 +1,644 @@ +local api = vim.api +local handlers = require('vim.lsp.handlers') +local util = require('vim.lsp.util') + +--- @class STTokenRange +--- @field line number line number 0-based +--- @field start_col number start column 0-based +--- @field end_col number end column 0-based +--- @field type string token type as string +--- @field modifiers string[] token modifiers as strings +--- @field extmark_added boolean whether this extmark has been added to the buffer yet +--- +--- @class STCurrentResult +--- @field version number document version associated with this result +--- @field result_id string resultId from the server; used with delta requests +--- @field highlights STTokenRange[] cache of highlight ranges for this document version +--- @field tokens number[] raw token array as received by the server. used for calculating delta responses +--- @field namespace_cleared boolean whether the namespace was cleared for this result yet +--- +--- @class STActiveRequest +--- @field request_id number the LSP request ID of the most recent request sent to the server +--- @field version number the document version associated with the most recent request +--- +--- @class STClientState +--- @field namespace number +--- @field active_request STActiveRequest +--- @field current_result STCurrentResult + +---@class STHighlighter +---@field active table<number, STHighlighter> +---@field bufnr number +---@field augroup number augroup for buffer events +---@field debounce number milliseconds to debounce requests for new tokens +---@field timer table uv_timer for debouncing requests for new tokens +---@field client_state table<number, STClientState> +local STHighlighter = { active = {} } + +---@private +local function binary_search(tokens, line) + local lo = 1 + local hi = #tokens + while lo < hi do + local mid = math.floor((lo + hi) / 2) + if tokens[mid].line < line then + lo = mid + 1 + else + hi = mid + end + end + return lo +end + +--- Extracts modifier strings from the encoded number in the token array +--- +---@private +---@return string[] +local function modifiers_from_number(x, modifiers_table) + ---@private + local function _get_bit(n, k) + --TODO(jdrouhard): remove once `bit` module is available for non-LuaJIT + if _G.bit then + return _G.bit.band(_G.bit.rshift(n, k), 1) + else + return math.floor((n / math.pow(2, k)) % 2) + end + end + + local modifiers = {} + for i = 0, #modifiers_table - 1 do + local b = _get_bit(x, i) + if b == 1 then + modifiers[#modifiers + 1] = modifiers_table[i + 1] + end + end + + return modifiers +end + +--- Converts a raw token list to a list of highlight ranges used by the on_win callback +--- +---@private +---@return STTokenRange[] +local function tokens_to_ranges(data, bufnr, client) + local legend = client.server_capabilities.semanticTokensProvider.legend + local token_types = legend.tokenTypes + local token_modifiers = legend.tokenModifiers + local ranges = {} + + local line + local start_char = 0 + for i = 1, #data, 5 do + local delta_line = data[i] + line = line and line + delta_line or delta_line + local delta_start = data[i + 1] + start_char = delta_line == 0 and start_char + delta_start or delta_start + + -- data[i+3] +1 because Lua tables are 1-indexed + local token_type = token_types[data[i + 3] + 1] + local modifiers = modifiers_from_number(data[i + 4], token_modifiers) + + ---@private + local function _get_byte_pos(char_pos) + return util._get_line_byte_from_position(bufnr, { + line = line, + character = char_pos, + }, client.offset_encoding) + end + + local start_col = _get_byte_pos(start_char) + local end_col = _get_byte_pos(start_char + data[i + 2]) + + if token_type then + ranges[#ranges + 1] = { + line = line, + start_col = start_col, + end_col = end_col, + type = token_type, + modifiers = modifiers, + extmark_added = false, + } + end + end + + return ranges +end + +--- Construct a new STHighlighter for the buffer +--- +---@private +---@param bufnr number +function STHighlighter.new(bufnr) + local self = setmetatable({}, { __index = STHighlighter }) + + self.bufnr = bufnr + self.augroup = api.nvim_create_augroup('vim_lsp_semantic_tokens:' .. bufnr, { clear = true }) + self.client_state = {} + + STHighlighter.active[bufnr] = self + + api.nvim_buf_attach(bufnr, false, { + on_lines = function(_, buf) + local highlighter = STHighlighter.active[buf] + if not highlighter then + return true + end + highlighter:on_change() + end, + on_reload = function(_, buf) + local highlighter = STHighlighter.active[buf] + if highlighter then + highlighter:reset() + highlighter:send_request() + end + end, + on_detach = function(_, buf) + local highlighter = STHighlighter.active[buf] + if highlighter then + highlighter:destroy() + end + end, + }) + + api.nvim_create_autocmd({ 'BufWinEnter', 'InsertLeave' }, { + buffer = self.bufnr, + group = self.augroup, + callback = function() + self:send_request() + end, + }) + + api.nvim_create_autocmd('LspDetach', { + buffer = self.bufnr, + group = self.augroup, + callback = function(args) + self:detach(args.data.client_id) + if vim.tbl_isempty(self.client_state) then + self:destroy() + end + end, + }) + + return self +end + +---@private +function STHighlighter:destroy() + for client_id, _ in pairs(self.client_state) do + self:detach(client_id) + end + + api.nvim_del_augroup_by_id(self.augroup) + STHighlighter.active[self.bufnr] = nil +end + +---@private +function STHighlighter:attach(client_id) + local state = self.client_state[client_id] + if not state then + state = { + namespace = api.nvim_create_namespace('vim_lsp_semantic_tokens:' .. client_id), + active_request = {}, + current_result = {}, + } + self.client_state[client_id] = state + end +end + +---@private +function STHighlighter:detach(client_id) + local state = self.client_state[client_id] + if state then + --TODO: delete namespace if/when that becomes possible + api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1) + self.client_state[client_id] = nil + end +end + +--- This is the entry point for getting all the tokens in a buffer. +--- +--- For the given clients (or all attached, if not provided), this sends a request +--- to ask for semantic tokens. If the server supports delta requests, that will +--- be prioritized if we have a previous requestId and token array. +--- +--- This function will skip servers where there is an already an active request in +--- flight for the same version. If there is a stale request in flight, that is +--- cancelled prior to sending a new one. +--- +--- Finally, if the request was successful, the requestId and document version +--- are saved to facilitate document synchronization in the response. +--- +---@private +function STHighlighter:send_request() + local version = util.buf_versions[self.bufnr] + + self:reset_timer() + + for client_id, state in pairs(self.client_state) do + local client = vim.lsp.get_client_by_id(client_id) + + local current_result = state.current_result + local active_request = state.active_request + + -- Only send a request for this client if the current result is out of date and + -- there isn't a current a request in flight for this version + if client and current_result.version ~= version and active_request.version ~= version then + -- cancel stale in-flight request + if active_request.request_id then + client.cancel_request(active_request.request_id) + active_request = {} + state.active_request = active_request + end + + local spec = client.server_capabilities.semanticTokensProvider.full + local hasEditProvider = type(spec) == 'table' and spec.delta + + local params = { textDocument = util.make_text_document_params(self.bufnr) } + local method = 'textDocument/semanticTokens/full' + + if hasEditProvider and current_result.result_id then + method = method .. '/delta' + params.previousResultId = current_result.result_id + end + local success, request_id = client.request(method, params, function(err, response, ctx) + -- look client up again using ctx.client_id instead of using a captured + -- client object + local c = vim.lsp.get_client_by_id(ctx.client_id) + local highlighter = STHighlighter.active[ctx.bufnr] + if not err and c and highlighter then + highlighter:process_response(response, c, version) + end + end, self.bufnr) + + if success then + active_request.request_id = request_id + active_request.version = version + end + end + end +end + +--- This function will parse the semantic token responses and set up the cache +--- (current_result). It also performs document synchronization by checking the +--- version of the document associated with the resulting request_id and only +--- performing work if the response is not out-of-date. +--- +--- Delta edits are applied if necessary, and new highlight ranges are calculated +--- and stored in the buffer state. +--- +--- Finally, a redraw command is issued to force nvim to redraw the screen to +--- pick up changed highlight tokens. +--- +---@private +function STHighlighter:process_response(response, client, version) + local state = self.client_state[client.id] + if not state then + return + end + + -- ignore stale responses + if state.active_request.version and version ~= state.active_request.version then + return + end + + -- reset active request + state.active_request = {} + + -- if we have a response to a delta request, update the state of our tokens + -- appropriately. if it's a full response, just use that + local tokens + local token_edits = response.edits + if token_edits then + table.sort(token_edits, function(a, b) + return a.start < b.start + end) + + ---@private + local function _splice(list, start, remove_count, data) + local ret = vim.list_slice(list, 1, start) + vim.list_extend(ret, data) + vim.list_extend(ret, list, start + remove_count + 1) + return ret + end + + tokens = state.current_result.tokens + for _, token_edit in ipairs(token_edits) do + tokens = _splice(tokens, token_edit.start, token_edit.deleteCount, token_edit.data) + end + else + tokens = response.data + end + + -- Update the state with the new results + local current_result = state.current_result + current_result.version = version + current_result.result_id = response.resultId + current_result.tokens = tokens + current_result.highlights = tokens_to_ranges(tokens, self.bufnr, client) + current_result.namespace_cleared = false + + api.nvim_command('redraw!') +end + +--- on_win handler for the decoration provider (see |nvim_set_decoration_provider|) +--- +--- If there is a current result for the buffer and the version matches the +--- current document version, then the tokens are valid and can be applied. As +--- the buffer is drawn, this function will add extmark highlights for every +--- token in the range of visible lines. Once a highlight has been added, it +--- sticks around until the document changes and there's a new set of matching +--- highlight tokens available. +--- +--- If this is the first time a buffer is being drawn with a new set of +--- highlights for the current document version, the namespace is cleared to +--- remove extmarks from the last version. It's done here instead of the response +--- handler to avoid the "blink" that occurs due to the timing between the +--- response handler and the actual redraw. +--- +---@private +function STHighlighter:on_win(topline, botline) + for _, state in pairs(self.client_state) do + local current_result = state.current_result + if current_result.version and current_result.version == util.buf_versions[self.bufnr] then + if not current_result.namespace_cleared then + api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1) + current_result.namespace_cleared = true + end + + -- We can't use ephemeral extmarks because the buffer updates are not in + -- sync with the list of semantic tokens. There's a delay between the + -- buffer changing and when the LSP server can respond with updated + -- tokens, and we don't want to "blink" the token highlights while + -- updates are in flight, and we don't want to use stale tokens because + -- they likely won't line up right with the actual buffer. + -- + -- Instead, we have to use normal extmarks that can attach to locations + -- in the buffer and are persisted between redraws. + local highlights = current_result.highlights + local idx = binary_search(highlights, topline) + + for i = idx, #highlights do + local token = highlights[i] + + if token.line > botline then + break + end + + if not token.extmark_added then + -- `strict = false` is necessary here for the 1% of cases where the + -- current result doesn't actually match the buffer contents. Some + -- LSP servers can respond with stale tokens on requests if they are + -- still processing changes from a didChange notification. + -- + -- LSP servers that do this _should_ follow up known stale responses + -- with a refresh notification once they've finished processing the + -- didChange notification, which would re-synchronize the tokens from + -- our end. + -- + -- The server I know of that does this is clangd when the preamble of + -- a file changes and the token request is processed with a stale + -- preamble while the new one is still being built. Once the preamble + -- finishes, clangd sends a refresh request which lets the client + -- re-synchronize the tokens. + api.nvim_buf_set_extmark(self.bufnr, state.namespace, token.line, token.start_col, { + hl_group = '@' .. token.type, + end_col = token.end_col, + priority = vim.highlight.priorities.semantic_tokens, + strict = false, + }) + + --TODO(jdrouhard): do something with the modifiers + + token.extmark_added = true + end + end + end + end +end + +--- Reset the buffer's highlighting state and clears the extmark highlights. +--- +---@private +function STHighlighter:reset() + for client_id, state in pairs(self.client_state) do + api.nvim_buf_clear_namespace(self.bufnr, state.namespace, 0, -1) + state.current_result = {} + if state.active_request.request_id then + local client = vim.lsp.get_client_by_id(client_id) + assert(client) + client.cancel_request(state.active_request.request_id) + state.active_request = {} + end + end +end + +--- Mark a client's results as dirty. This method will cancel any active +--- requests to the server and pause new highlights from being added +--- in the on_win callback. The rest of the current results are saved +--- in case the server supports delta requests. +--- +---@private +---@param client_id number +function STHighlighter:mark_dirty(client_id) + local state = self.client_state[client_id] + assert(state) + + -- if we clear the version from current_result, it'll cause the + -- next request to be sent and will also pause new highlights + -- from being added in on_win until a new result comes from + -- the server + if state.current_result then + state.current_result.version = nil + end + + if state.active_request.request_id then + local client = vim.lsp.get_client_by_id(client_id) + assert(client) + client.cancel_request(state.active_request.request_id) + state.active_request = {} + end +end + +---@private +function STHighlighter:on_change() + self:reset_timer() + if self.debounce > 0 then + self.timer = vim.defer_fn(function() + self:send_request() + end, self.debounce) + else + self:send_request() + end +end + +---@private +function STHighlighter:reset_timer() + local timer = self.timer + if timer then + self.timer = nil + if not timer:is_closing() then + timer:stop() + timer:close() + end + end +end + +local M = {} + +--- Start the semantic token highlighting engine for the given buffer with the +--- given client. The client must already be attached to the buffer. +--- +--- NOTE: This is currently called automatically by |vim.lsp.buf_attach_client()|. To +--- opt-out of semantic highlighting with a server that supports it, you can +--- delete the semanticTokensProvider table from the {server_capabilities} of +--- your client in your |LspAttach| callback or your configuration's +--- `on_attach` callback. +--- +--- <pre>lua +--- client.server_capabilities.semanticTokensProvider = nil +--- </pre> +--- +---@param bufnr number +---@param client_id number +---@param opts (nil|table) Optional keyword arguments +--- - debounce (number, default: 200): Debounce token requests +--- to the server by the given number in milliseconds +function M.start(bufnr, client_id, opts) + vim.validate({ + bufnr = { bufnr, 'n', false }, + client_id = { client_id, 'n', false }, + }) + + opts = opts or {} + assert( + (not opts.debounce or type(opts.debounce) == 'number'), + 'opts.debounce must be a number with the debounce time in milliseconds' + ) + + local client = vim.lsp.get_client_by_id(client_id) + if not client then + vim.notify('[LSP] No client with id ' .. client_id, vim.log.levels.ERROR) + return + end + + if not vim.lsp.buf_is_attached(bufnr, client_id) then + vim.notify( + '[LSP] Client with id ' .. client_id .. ' not attached to buffer ' .. bufnr, + vim.log.levels.WARN + ) + return + end + + if not vim.tbl_get(client.server_capabilities, 'semanticTokensProvider', 'full') then + vim.notify('[LSP] Server does not support semantic tokens', vim.log.levels.WARN) + return + end + + local highlighter = STHighlighter.active[bufnr] + + if not highlighter then + highlighter = STHighlighter.new(bufnr) + highlighter.debounce = opts.debounce or 200 + else + highlighter.debounce = math.max(highlighter.debounce, opts.debounce or 200) + end + + highlighter:attach(client_id) + highlighter:send_request() +end + +--- Stop the semantic token highlighting engine for the given buffer with the +--- given client. +--- +--- NOTE: This is automatically called by a |LspDetach| autocmd that is set up as part +--- of `start()`, so you should only need this function to manually disengage the semantic +--- token engine without fully detaching the LSP client from the buffer. +--- +---@param bufnr number +---@param client_id number +function M.stop(bufnr, client_id) + vim.validate({ + bufnr = { bufnr, 'n', false }, + client_id = { client_id, 'n', false }, + }) + + local highlighter = STHighlighter.active[bufnr] + if not highlighter then + return + end + + highlighter:detach(client_id) + + if vim.tbl_isempty(highlighter.client_state) then + highlighter:destroy() + end +end + +--- Force a refresh of all semantic tokens +--- +--- Only has an effect if the buffer is currently active for semantic token +--- highlighting (|vim.lsp.semantic_tokens.start()| has been called for it) +--- +---@param bufnr (nil|number) default: current buffer +function M.force_refresh(bufnr) + vim.validate({ + bufnr = { bufnr, 'n', true }, + }) + + if bufnr == nil or bufnr == 0 then + bufnr = api.nvim_get_current_buf() + end + + local highlighter = STHighlighter.active[bufnr] + if not highlighter then + return + end + + highlighter:reset() + highlighter:send_request() +end + +--- |lsp-handler| for the method `workspace/semanticTokens/refresh` +--- +--- Refresh requests are sent by the server to indicate a project-wide change +--- that requires all tokens to be re-requested by the client. This handler will +--- invalidate the current results of all buffers and automatically kick off a +--- new request for buffers that are displayed in a window. For those that aren't, a +--- the BufWinEnter event should take care of it next time it's displayed. +--- +---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#semanticTokens_refreshRequest +handlers['workspace/semanticTokens/refresh'] = function(err, _, ctx) + if err then + return vim.NIL + end + + for _, bufnr in ipairs(vim.lsp.get_buffers_by_client_id(ctx.client_id)) do + local highlighter = STHighlighter.active[bufnr] + if highlighter and highlighter.client_state[ctx.client_id] then + highlighter:mark_dirty(ctx.client_id) + + if not vim.tbl_isempty(vim.fn.win_findbuf(bufnr)) then + highlighter:send_request() + end + end + end + + return vim.NIL +end + +local namespace = api.nvim_create_namespace('vim_lsp_semantic_tokens') +api.nvim_set_decoration_provider(namespace, { + on_win = function(_, _, bufnr, topline, botline) + local highlighter = STHighlighter.active[bufnr] + if highlighter then + highlighter:on_win(topline, botline) + end + end, +}) + +--- for testing only! there is no guarantee of API stability with this! +--- +---@private +M.__STHighlighter = STHighlighter + +return M diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py index 78f3c458eb..b18179b498 100755 --- a/scripts/gen_vimdoc.py +++ b/scripts/gen_vimdoc.py @@ -183,6 +183,7 @@ CONFIG = { 'diagnostic.lua', 'codelens.lua', 'tagfunc.lua', + 'semantic_tokens.lua', 'handlers.lua', 'util.lua', 'log.lua', diff --git a/src/nvim/highlight_group.c b/src/nvim/highlight_group.c index 2e60d6a570..c4649ac91c 100644 --- a/src/nvim/highlight_group.c +++ b/src/nvim/highlight_group.c @@ -258,12 +258,23 @@ static const char *highlight_init_both[] = { "default link @type Type", "default link @type.definition Typedef", "default link @storageclass StorageClass", - "default link @structure Structure", "default link @namespace Identifier", "default link @include Include", "default link @preproc PreProc", "default link @debug Debug", "default link @tag Tag", + + // LSP semantic tokens + "default link @class Structure", + "default link @struct Structure", + "default link @enum Type", + "default link @enumMember Constant", + "default link @event Identifier", + "default link @interface Identifier", + "default link @modifier Identifier", + "default link @regexp SpecialChar", + "default link @typeParameter Type", + "default link @decorator Identifier", NULL }; diff --git a/test/functional/plugin/lsp/helpers.lua b/test/functional/plugin/lsp/helpers.lua new file mode 100644 index 0000000000..1363ab894d --- /dev/null +++ b/test/functional/plugin/lsp/helpers.lua @@ -0,0 +1,178 @@ +local helpers = require('test.functional.helpers')(nil) + +local clear = helpers.clear +local exec_lua = helpers.exec_lua +local run = helpers.run +local stop = helpers.stop +local NIL = helpers.NIL + +local M = {} + +function M.clear_notrace() + -- problem: here be dragons + -- solution: don't look too closely for dragons + clear {env={ + NVIM_LUA_NOTRACK="1"; + VIMRUNTIME=os.getenv"VIMRUNTIME"; + }} +end + +M.create_server_definition = [[ + function _create_server(opts) + opts = opts or {} + local server = {} + server.messages = {} + + function server.cmd(dispatchers) + local closing = false + local handlers = opts.handlers or {} + local srv = {} + + function srv.request(method, params, callback) + table.insert(server.messages, { + method = method, + params = params, + }) + local handler = handlers[method] + if handler then + local response, err = handler(method, params) + if response then + callback(err, response) + end + elseif method == 'initialize' then + callback(nil, { + capabilities = opts.capabilities or {} + }) + elseif method == 'shutdown' then + callback(nil, nil) + end + local request_id = #server.messages + return true, request_id + end + + function srv.notify(method, params) + table.insert(server.messages, { + method = method, + params = params + }) + if method == 'exit' then + dispatchers.on_exit(0, 15) + end + end + + function srv.is_closing() + return closing + end + + function srv.terminate() + closing = true + end + + return srv + end + + return server + end +]] + +-- Fake LSP server. +M.fake_lsp_code = 'test/functional/fixtures/fake-lsp-server.lua' +M.fake_lsp_logfile = 'Xtest-fake-lsp.log' + +local function fake_lsp_server_setup(test_name, timeout_ms, options, settings) + exec_lua([=[ + lsp = require('vim.lsp') + local test_name, fixture_filename, logfile, timeout, options, settings = ... + TEST_RPC_CLIENT_ID = lsp.start_client { + cmd_env = { + NVIM_LOG_FILE = logfile; + NVIM_LUA_NOTRACK = "1"; + }; + cmd = { + vim.v.progpath, '-Es', '-u', 'NONE', '--headless', + "-c", string.format("lua TEST_NAME = %q", test_name), + "-c", string.format("lua TIMEOUT = %d", timeout), + "-c", "luafile "..fixture_filename, + }; + handlers = setmetatable({}, { + __index = function(t, method) + return function(...) + return vim.rpcrequest(1, 'handler', ...) + end + end; + }); + workspace_folders = {{ + uri = 'file://' .. vim.loop.cwd(), + name = 'test_folder', + }}; + on_init = function(client, result) + TEST_RPC_CLIENT = client + vim.rpcrequest(1, "init", result) + end; + flags = { + allow_incremental_sync = options.allow_incremental_sync or false; + debounce_text_changes = options.debounce_text_changes or 0; + }; + settings = settings; + on_exit = function(...) + vim.rpcnotify(1, "exit", ...) + end; + } + ]=], test_name, M.fake_lsp_code, M.fake_lsp_logfile, timeout_ms or 1e3, options or {}, settings or {}) +end + +function M.test_rpc_server(config) + if config.test_name then + M.clear_notrace() + fake_lsp_server_setup(config.test_name, config.timeout_ms or 1e3, config.options, config.settings) + end + local client = setmetatable({}, { + __index = function(_, name) + -- Workaround for not being able to yield() inside __index for Lua 5.1 :( + -- Otherwise I would just return the value here. + return function(...) + return exec_lua([=[ + local name = ... + if type(TEST_RPC_CLIENT[name]) == 'function' then + return TEST_RPC_CLIENT[name](select(2, ...)) + else + return TEST_RPC_CLIENT[name] + end + ]=], name, ...) + end + end; + }) + local code, signal + local function on_request(method, args) + if method == "init" then + if config.on_init then + config.on_init(client, unpack(args)) + end + return NIL + end + if method == 'handler' then + if config.on_handler then + config.on_handler(unpack(args)) + end + end + return NIL + end + local function on_notify(method, args) + if method == 'exit' then + code, signal = unpack(args) + return stop() + end + end + -- TODO specify timeout? + -- run(on_request, on_notify, config.on_setup, 1000) + run(on_request, on_notify, config.on_setup) + if config.on_exit then + config.on_exit(code, signal) + end + stop() + if config.test_name then + exec_lua("vim.api.nvim_exec_autocmds('VimLeavePre', { modeline = false })") + end +end + +return M diff --git a/test/functional/plugin/lsp/semantic_tokens_spec.lua b/test/functional/plugin/lsp/semantic_tokens_spec.lua new file mode 100644 index 0000000000..1646108416 --- /dev/null +++ b/test/functional/plugin/lsp/semantic_tokens_spec.lua @@ -0,0 +1,910 @@ +local helpers = require('test.functional.helpers')(after_each) +local lsp_helpers = require('test.functional.plugin.lsp.helpers') +local Screen = require('test.functional.ui.screen') + +local command = helpers.command +local dedent = helpers.dedent +local eq = helpers.eq +local exec_lua = helpers.exec_lua +local feed = helpers.feed +local feed_command = helpers.feed_command +local insert = helpers.insert +local matches = helpers.matches + +local clear_notrace = lsp_helpers.clear_notrace +local create_server_definition = lsp_helpers.create_server_definition + +before_each(function() + clear_notrace() +end) + +after_each(function() + exec_lua("vim.api.nvim_exec_autocmds('VimLeavePre', { modeline = false })") +end) + +describe('semantic token highlighting', function() + + describe('general', function() + local text = dedent([[ + #include <iostream> + + int main() + { + int x; + #ifdef __cplusplus + std::cout << x << "\n"; + #else + printf("%d\n", x); + #endif + } + }]]) + + local legend = [[{ + "tokenTypes": [ + "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment" + ], + "tokenModifiers": [ + "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope" + ] + }]] + + local response = [[{ + "data": [ 2, 4, 4, 3, 8193, 2, 8, 1, 1, 1025, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1024, 1, 0, 5, 20, 0, 1, 0, 22, 20, 0, 1, 0, 6, 20, 0 ], + "resultId": 1 + }]] + + local edit_response = [[{ + "edits": [ {"data": [ 2, 8, 1, 3, 8193, 1, 7, 11, 19, 8192, 1, 4, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 3, 8192 ], "deleteCount": 25, "start": 5 } ], + "resultId":"2" + }]] + + local screen + before_each(function() + screen = Screen.new(40, 16) + screen:attach() + screen:set_default_attr_ids { + [1] = { bold = true, foreground = Screen.colors.Blue1 }; + [2] = { foreground = Screen.colors.DarkCyan }; + [3] = { foreground = Screen.colors.SlateBlue }; + [4] = { bold = true, foreground = Screen.colors.SeaGreen }; + [5] = { foreground = tonumber('0x6a0dad') }; + [6] = { foreground = Screen.colors.Blue1 }; + } + command([[ hi link @namespace Type ]]) + command([[ hi link @function Special ]]) + + exec_lua(create_server_definition) + exec_lua([[ + local legend, response, edit_response = ... + server = _create_server({ + capabilities = { + semanticTokensProvider = { + full = { delta = true }, + legend = vim.fn.json_decode(legend), + }, + }, + handlers = { + ['textDocument/semanticTokens/full'] = function() + return vim.fn.json_decode(response) + end, + ['textDocument/semanticTokens/full/delta'] = function() + return vim.fn.json_decode(edit_response) + end, + } + }) + ]], legend, response, edit_response) + end) + + it('buffer is highlighted when attached', function() + exec_lua([[ + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd }) + ]]) + + insert(text) + + screen:expect { grid = [[ + #include <iostream> | + | + int {3:main}() | + { | + int {2:x}; | + #ifdef {5:__cplusplus} | + {4:std}::{2:cout} << {2:x} << "\n"; | + {6:#else} | + {6: printf("%d\n", x);} | + {6:#endif} | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]] } + end) + + it('buffer is unhighlighted when client is detached', function() + exec_lua([[ + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd }) + ]]) + + insert(text) + + exec_lua([[ + vim.notify = function() end + vim.lsp.buf_detach_client(bufnr, client_id) + ]]) + + screen:expect { grid = [[ + #include <iostream> | + | + int main() | + { | + int x; | + #ifdef __cplusplus | + std::cout << x << "\n"; | + #else | + printf("%d\n", x); | + #endif | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]] } + end) + + it('buffer is highlighted and unhighlighted when semantic token highlighting is started and stopped' + , function() + exec_lua([[ + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd }) + ]]) + + insert(text) + + exec_lua([[ + vim.notify = function() end + vim.lsp.semantic_tokens.stop(bufnr, client_id) + ]]) + + screen:expect { grid = [[ + #include <iostream> | + | + int main() | + { | + int x; | + #ifdef __cplusplus | + std::cout << x << "\n"; | + #else | + printf("%d\n", x); | + #endif | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]] } + + exec_lua([[ + vim.lsp.semantic_tokens.start(bufnr, client_id) + ]]) + + screen:expect { grid = [[ + #include <iostream> | + | + int {3:main}() | + { | + int {2:x}; | + #ifdef {5:__cplusplus} | + {4:std}::{2:cout} << {2:x} << "\n"; | + {6:#else} | + {6: printf("%d\n", x);} | + {6:#endif} | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]] } + end) + + it('buffer is re-highlighted when force refreshed', function() + exec_lua([[ + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd }) + ]]) + + insert(text) + + screen:expect { grid = [[ + #include <iostream> | + | + int {3:main}() | + { | + int {2:x}; | + #ifdef {5:__cplusplus} | + {4:std}::{2:cout} << {2:x} << "\n"; | + {6:#else} | + {6: printf("%d\n", x);} | + {6:#endif} | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]] } + + exec_lua([[ + vim.lsp.semantic_tokens.force_refresh(bufnr) + ]]) + + screen:expect { grid = [[ + #include <iostream> | + | + int {3:main}() | + { | + int {2:x}; | + #ifdef {5:__cplusplus} | + {4:std}::{2:cout} << {2:x} << "\n"; | + {6:#else} | + {6: printf("%d\n", x);} | + {6:#endif} | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]], unchanged = true } + + local messages = exec_lua('return server.messages') + local token_request_count = 0 + for _, message in ipairs(messages) do + assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received') + if message.method == 'textDocument/semanticTokens/full' then + token_request_count = token_request_count + 1 + end + end + eq(2, token_request_count) + end) + + it('destroys the highlighter if the buffer is deleted', function() + exec_lua([[ + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd }) + ]]) + + insert(text) + + local highlighters = exec_lua([[ + vim.api.nvim_buf_delete(bufnr, { force = true }) + local semantic_tokens = vim.lsp.semantic_tokens + return semantic_tokens.__STHighlighter.active + ]]) + + eq({}, highlighters) + end) + + it('updates highlights with delta request on buffer change', function() + exec_lua([[ + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd }) + ]]) + + insert(text) + feed_command('%s/int x/int x()/') + feed_command('noh') + screen:expect { grid = [[ + #include <iostream> | + | + int {3:main}() | + { | + ^int {3:x}(); | + #ifdef {5:__cplusplus} | + {4:std}::{2:cout} << {3:x} << "\n"; | + {6:#else} | + {6: printf("%d\n", x);} | + {6:#endif} | + } | + } | + {1:~ }| + {1:~ }| + {1:~ }| + :noh | + ]] } + end) + + it('prevents starting semantic token highlighting with invalid conditions', function() + exec_lua([[ + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start_client({ name = 'dummy', cmd = server.cmd }) + notifications = {} + vim.notify = function(...) table.insert(notifications, 1, {...}) end + ]]) + eq(false, exec_lua("return vim.lsp.buf_is_attached(bufnr, client_id)")) + + insert(text) + + local notifications = exec_lua([[ + vim.lsp.semantic_tokens.start(bufnr, client_id) + return notifications + ]]) + matches('%[LSP%] Client with id %d not attached to buffer %d', notifications[1][1]) + + notifications = exec_lua([[ + vim.lsp.semantic_tokens.start(bufnr, client_id + 1) + return notifications + ]]) + matches('%[LSP%] No client with id %d', notifications[1][1]) + end) + + it('opt-out: does not activate semantic token highlighting if disabled in client attach', + function() + exec_lua([[ + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ + name = 'dummy', + cmd = server.cmd, + on_attach = function(client, bufnr) + client.server_capabilities.semanticTokensProvider = nil + end, + }) + ]]) + eq(true, exec_lua("return vim.lsp.buf_is_attached(bufnr, client_id)")) + + insert(text) + + screen:expect { grid = [[ + #include <iostream> | + | + int main() | + { | + int x; | + #ifdef __cplusplus | + std::cout << x << "\n"; | + #else | + printf("%d\n", x); | + #endif | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]] } + + local notifications = exec_lua([[ + local notifications = {} + vim.notify = function(...) table.insert(notifications, 1, {...}) end + vim.lsp.semantic_tokens.start(bufnr, client_id) + return notifications + ]]) + eq('[LSP] Server does not support semantic tokens', notifications[1][1]) + + screen:expect { grid = [[ + #include <iostream> | + | + int main() | + { | + int x; | + #ifdef __cplusplus | + std::cout << x << "\n"; | + #else | + printf("%d\n", x); | + #endif | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]], unchanged = true } + end) + + it('does not send delta requests if not supported by server', function() + exec_lua([[ + local legend, response, edit_response = ... + server2 = _create_server({ + capabilities = { + semanticTokensProvider = { + full = { delta = false }, + legend = vim.fn.json_decode(legend), + }, + }, + handlers = { + ['textDocument/semanticTokens/full'] = function() + return vim.fn.json_decode(response) + end, + ['textDocument/semanticTokens/full/delta'] = function() + return vim.fn.json_decode(edit_response) + end, + } + }) + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server2.cmd }) + ]], legend, response, edit_response) + + insert(text) + screen:expect { grid = [[ + #include <iostream> | + | + int {3:main}() | + { | + int {2:x}; | + #ifdef {5:__cplusplus} | + {4:std}::{2:cout} << {2:x} << "\n"; | + {6:#else} | + {6: printf("%d\n", x);} | + {6:#endif} | + } | + ^} | + {1:~ }| + {1:~ }| + {1:~ }| + | + ]] } + feed_command('%s/int x/int x()/') + feed_command('noh') + + -- the highlights don't change because our fake server sent the exact + -- same result for the same method (the full request). "x" would have + -- changed to highlight index 3 had we sent a delta request + screen:expect { grid = [[ + #include <iostream> | + | + int {3:main}() | + { | + ^int {2:x}(); | + #ifdef {5:__cplusplus} | + {4:std}::{2:cout} << {2:x} << "\n"; | + {6:#else} | + {6: printf("%d\n", x);} | + {6:#endif} | + } | + } | + {1:~ }| + {1:~ }| + {1:~ }| + :noh | + ]] } + local messages = exec_lua('return server2.messages') + local token_request_count = 0 + for _, message in ipairs(messages) do + assert(message.method ~= 'textDocument/semanticTokens/full/delta', 'delta request received') + if message.method == 'textDocument/semanticTokens/full' then + token_request_count = token_request_count + 1 + end + end + eq(2, token_request_count) + end) + end) + + describe('token array decoding', function() + for _, test in ipairs({ + { + it = 'clangd-15 on C', + text = [[char* foo = "\n";]], + response = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]], + legend = [[{ + "tokenTypes": [ + "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment" + ], + "tokenModifiers": [ + "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope" + ] + }]], + expected = { + { + line = 0, + modifiers = { + 'declaration', + 'globalScope', + }, + start_col = 6, + end_col = 9, + type = 'variable', + extmark_added = true, + }, + }, + }, + { + it = 'clangd-15 on C++', + text = [[#include <iostream> +int main() +{ + #ifdef __cplusplus + const int x = 1; + std::cout << x << std::endl; + #else + comment + #endif +}]] , + response = [[{"data": [1, 4, 4, 3, 8193, 2, 9, 11, 19, 8192, 1, 12, 1, 1, 1033, 1, 2, 3, 15, 8448, 0, 5, 4, 0, 8448, 0, 8, 1, 1, 1032, 0, 5, 3, 15, 8448, 0, 5, 4, 3, 8448, 1, 0, 7, 20, 0, 1, 0, 11, 20, 0, 1, 0, 8, 20, 0], "resultId": "1"}]], + legend = [[{ + "tokenTypes": [ + "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment" + ], + "tokenModifiers": [ + "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope" + ] + }]], + expected = { + { -- main + line = 1, + modifiers = { 'declaration', 'globalScope' }, + start_col = 4, + end_col = 8, + type = 'function', + extmark_added = true, + }, + { -- __cplusplus + line = 3, + modifiers = { 'globalScope' }, + start_col = 9, + end_col = 20, + type = 'macro', + extmark_added = true, + }, + { -- x + line = 4, + modifiers = { 'declaration', 'readonly', 'functionScope' }, + start_col = 12, + end_col = 13, + type = 'variable', + extmark_added = true, + }, + { -- std + line = 5, + modifiers = { 'defaultLibrary', 'globalScope' }, + start_col = 2, + end_col = 5, + type = 'namespace', + extmark_added = true, + }, + { -- cout + line = 5, + modifiers = { 'defaultLibrary', 'globalScope' }, + start_col = 7, + end_col = 11, + type = 'variable', + extmark_added = true, + }, + { -- x + line = 5, + modifiers = { 'readonly', 'functionScope' }, + start_col = 15, + end_col = 16, + type = 'variable', + extmark_added = true, + }, + { -- std + line = 5, + modifiers = { 'defaultLibrary', 'globalScope' }, + start_col = 20, + end_col = 23, + type = 'namespace', + extmark_added = true, + }, + { -- endl + line = 5, + modifiers = { 'defaultLibrary', 'globalScope' }, + start_col = 25, + end_col = 29, + type = 'function', + extmark_added = true, + }, + { -- #else comment #endif + line = 6, + modifiers = {}, + start_col = 0, + end_col = 7, + type = 'comment', + extmark_added = true, + }, + { + line = 7, + modifiers = {}, + start_col = 0, + end_col = 11, + type = 'comment', + extmark_added = true, + }, + { + line = 8, + modifiers = {}, + start_col = 0, + end_col = 8, + type = 'comment', + extmark_added = true, + }, + }, + }, + { + it = 'sumneko_lua', + text = [[-- comment +local a = 1 +b = "as"]], + response = [[{"data": [0, 0, 10, 17, 0, 1, 6, 1, 8, 1, 1, 0, 1, 8, 8]}]], + legend = [[{ + "tokenTypes": [ + "namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "variable", "property", "enumMember", "event", "function", "method", "macro", "keyword", "modifier", "comment", "string", "number", "regexp", "operator" + ], + "tokenModifiers": [ + "declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary" + ] + }]], + expected = { + { + line = 0, + modifiers = {}, + start_col = 0, + end_col = 10, + type = 'comment', -- comment + extmark_added = true, + }, + { + line = 1, + modifiers = { 'declaration' }, -- a + start_col = 6, + end_col = 7, + type = 'variable', + extmark_added = true, + }, + { + line = 2, + modifiers = { 'static' }, -- b (global) + start_col = 0, + end_col = 1, + type = 'variable', + extmark_added = true, + }, + }, + }, + { + it = 'rust-analyzer', + text = [[pub fn main() { + break rust; + /// what? +} +]] , + response = [[{"data": [0, 0, 3, 1, 0, 0, 4, 2, 1, 0, 0, 3, 4, 14, 524290, 0, 4, 1, 45, 0, 0, 1, 1, 45, 0, 0, 2, 1, 26, 0, 1, 4, 5, 1, 8192, 0, 6, 4, 52, 0, 0, 4, 1, 48, 0, 1, 4, 9, 0, 1, 1, 0, 1, 26, 0], "resultId": "1"}]], + legend = [[{ + "tokenTypes": [ + "comment", "keyword", "string", "number", "regexp", "operator", "namespace", "type", "struct", "class", "interface", "enum", "enumMember", "typeParameter", "function", "method", "property", "macro", "variable", + "parameter", "angle", "arithmetic", "attribute", "attributeBracket", "bitwise", "boolean", "brace", "bracket", "builtinAttribute", "builtinType", "character", "colon", "comma", "comparison", "constParameter", "derive", + "dot", "escapeSequence", "formatSpecifier", "generic", "label", "lifetime", "logical", "macroBang", "operator", "parenthesis", "punctuation", "selfKeyword", "semicolon", "typeAlias", "toolModule", "union", "unresolvedReference" + ], + "tokenModifiers": [ + "documentation", "declaration", "definition", "static", "abstract", "deprecated", "readonly", "defaultLibrary", "async", "attribute", "callable", "constant", "consuming", "controlFlow", "crateRoot", "injected", "intraDocLink", + "library", "mutable", "public", "reference", "trait", "unsafe" + ] + }]], + expected = { + { + line = 0, + modifiers = {}, + start_col = 0, + end_col = 3, -- pub + type = 'keyword', + extmark_added = true, + }, + { + line = 0, + modifiers = {}, + start_col = 4, + end_col = 6, -- fn + type = 'keyword', + extmark_added = true, + }, + { + line = 0, + modifiers = { 'declaration', 'public' }, + start_col = 7, + end_col = 11, -- main + type = 'function', + extmark_added = true, + }, + { + line = 0, + modifiers = {}, + start_col = 11, + end_col = 12, + type = 'parenthesis', + extmark_added = true, + }, + { + line = 0, + modifiers = {}, + start_col = 12, + end_col = 13, + type = 'parenthesis', + extmark_added = true, + }, + { + line = 0, + modifiers = {}, + start_col = 14, + end_col = 15, + type = 'brace', + extmark_added = true, + }, + { + line = 1, + modifiers = { 'controlFlow' }, + start_col = 4, + end_col = 9, -- break + type = 'keyword', + extmark_added = true, + }, + { + line = 1, + modifiers = {}, + start_col = 10, + end_col = 13, -- rust + type = 'unresolvedReference', + extmark_added = true, + }, + { + line = 1, + modifiers = {}, + start_col = 13, + end_col = 13, + type = 'semicolon', + extmark_added = true, + }, + { + line = 2, + modifiers = { 'documentation' }, + start_col = 4, + end_col = 11, + type = 'comment', -- /// what? + extmark_added = true, + }, + { + line = 3, + modifiers = {}, + start_col = 0, + end_col = 1, + type = 'brace', + extmark_added = true, + }, + }, + }, + }) do + it(test.it, function() + exec_lua(create_server_definition) + exec_lua([[ + local legend, resp = ... + server = _create_server({ + capabilities = { + semanticTokensProvider = { + full = { delta = false }, + legend = vim.fn.json_decode(legend), + }, + }, + handlers = { + ['textDocument/semanticTokens/full'] = function() + return vim.fn.json_decode(resp) + end, + } + }) + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd }) + ]], test.legend, test.response) + + insert(test.text) + + local highlights = exec_lua([[ + local semantic_tokens = vim.lsp.semantic_tokens + return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights + ]]) + eq(test.expected, highlights) + end) + end + end) + describe('token decoding with deltas', function() + for _, test in ipairs({ + { + it = 'semantic_tokens_delta: clangd-15 on C', + name = 'semantic_tokens_delta', + legend = [[{ + "tokenTypes": [ + "variable", "variable", "parameter", "function", "method", "function", "property", "variable", "class", "interface", "enum", "enumMember", "type", "type", "unknown", "namespace", "typeParameter", "concept", "type", "macro", "comment" + ], + "tokenModifiers": [ + "declaration", "deprecated", "deduced", "readonly", "static", "abstract", "virtual", "dependentName", "defaultLibrary", "usedAsMutableReference", "functionScope", "classScope", "fileScope", "globalScope" + ] + }]], + text = [[char* foo = "\n";]], + edit = [[ggO<Esc>]], + response1 = [[{"data": [0, 6, 3, 0, 8193], "resultId": "1"}]], + response2 = [[{"edits": [{ "start": 0, "deleteCount": 1, "data": [1] }], "resultId": "2"}]], + expected1 = { + { + line = 0, + modifiers = { + 'declaration', + 'globalScope', + }, + start_col = 6, + end_col = 9, + type = 'variable', + extmark_added = true, + } + }, + expected2 = { + { + line = 1, + modifiers = { + 'declaration', + 'globalScope', + }, + start_col = 6, + end_col = 9, + type = 'variable', + extmark_added = true, + } + }, + } + }) do + it(test.it, function() + exec_lua(create_server_definition) + exec_lua([[ + local legend, resp1, resp2 = ... + server = _create_server({ + capabilities = { + semanticTokensProvider = { + full = { delta = true }, + legend = vim.fn.json_decode(legend), + }, + }, + handlers = { + ['textDocument/semanticTokens/full'] = function() + return vim.fn.json_decode(resp1) + end, + ['textDocument/semanticTokens/full/delta'] = function() + return vim.fn.json_decode(resp2) + end, + } + }) + bufnr = vim.api.nvim_get_current_buf() + vim.api.nvim_win_set_buf(0, bufnr) + client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd }) + semantic_tokens = vim.lsp.semantic_tokens + ]], test.legend, test.response1, test.response2) + + insert(test.text) + + local highlights = exec_lua([[ + return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights + ]]) + + eq(test.expected1, highlights) + + feed(test.edit) + + highlights = exec_lua([[ + return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights + ]]) + + eq(test.expected2, highlights) + end) + end + end) +end) diff --git a/test/functional/plugin/lsp_spec.lua b/test/functional/plugin/lsp_spec.lua index 163667d5f3..93fada8a50 100644 --- a/test/functional/plugin/lsp_spec.lua +++ b/test/functional/plugin/lsp_spec.lua @@ -1,8 +1,9 @@ local helpers = require('test.functional.helpers')(after_each) +local lsp_helpers = require('test.functional.plugin.lsp.helpers') local assert_log = helpers.assert_log -local clear = helpers.clear local buf_lines = helpers.buf_lines +local clear = helpers.clear local command = helpers.command local dedent = helpers.dedent local exec_lua = helpers.exec_lua @@ -14,6 +15,7 @@ local pesc = helpers.pesc local insert = helpers.insert local funcs = helpers.funcs local retry = helpers.retry +local stop = helpers.stop local NIL = helpers.NIL local read_file = require('test.helpers').read_file local write_file = require('test.helpers').write_file @@ -22,186 +24,19 @@ local meths = helpers.meths local is_os = helpers.is_os local skip = helpers.skip --- Use these to get access to a coroutine so that I can run async tests and use --- yield. -local run, stop = helpers.run, helpers.stop +local clear_notrace = lsp_helpers.clear_notrace +local create_server_definition = lsp_helpers.create_server_definition +local fake_lsp_code = lsp_helpers.fake_lsp_code +local fake_lsp_logfile = lsp_helpers.fake_lsp_logfile +local test_rpc_server = lsp_helpers.test_rpc_server -- TODO(justinmk): hangs on Windows https://github.com/neovim/neovim/pull/11837 if skip(is_os('win')) then return end --- Fake LSP server. -local fake_lsp_code = 'test/functional/fixtures/fake-lsp-server.lua' -local fake_lsp_logfile = 'Xtest-fake-lsp.log' - teardown(function() os.remove(fake_lsp_logfile) end) -local function clear_notrace() - -- problem: here be dragons - -- solution: don't look for dragons to closely - clear {env={ - NVIM_LUA_NOTRACK="1"; - VIMRUNTIME=os.getenv"VIMRUNTIME"; - }} -end - - -local create_server_definition = [[ - function _create_server(opts) - opts = opts or {} - local server = {} - server.messages = {} - - function server.cmd(dispatchers) - local closing = false - local handlers = opts.handlers or {} - local srv = {} - - function srv.request(method, params, callback) - table.insert(server.messages, { - method = method, - params = params, - }) - local handler = handlers[method] - if handler then - local response, err = handler(params) - if response then - callback(err, response) - end - elseif method == 'initialize' then - callback(nil, { - capabilities = opts.capabilities or {} - }) - elseif method == 'shutdown' then - callback(nil, nil) - end - local request_id = #server.messages - return true, request_id - end - - function srv.notify(method, params) - table.insert(server.messages, { - method = method, - params = params - }) - if method == 'exit' then - dispatchers.on_exit(0, 15) - end - end - - function srv.is_closing() - return closing - end - - function srv.terminate() - closing = true - end - - return srv - end - - return server - end -]] - - -local function fake_lsp_server_setup(test_name, timeout_ms, options, settings) - exec_lua([=[ - lsp = require('vim.lsp') - local test_name, fixture_filename, logfile, timeout, options, settings = ... - TEST_RPC_CLIENT_ID = lsp.start_client { - cmd_env = { - NVIM_LOG_FILE = logfile; - NVIM_LUA_NOTRACK = "1"; - }; - cmd = { - vim.v.progpath, '-Es', '-u', 'NONE', '--headless', - "-c", string.format("lua TEST_NAME = %q", test_name), - "-c", string.format("lua TIMEOUT = %d", timeout), - "-c", "luafile "..fixture_filename, - }; - handlers = setmetatable({}, { - __index = function(t, method) - return function(...) - return vim.rpcrequest(1, 'handler', ...) - end - end; - }); - workspace_folders = {{ - uri = 'file://' .. vim.loop.cwd(), - name = 'test_folder', - }}; - on_init = function(client, result) - TEST_RPC_CLIENT = client - vim.rpcrequest(1, "init", result) - end; - flags = { - allow_incremental_sync = options.allow_incremental_sync or false; - debounce_text_changes = options.debounce_text_changes or 0; - }; - settings = settings; - on_exit = function(...) - vim.rpcnotify(1, "exit", ...) - end; - } - ]=], test_name, fake_lsp_code, fake_lsp_logfile, timeout_ms or 1e3, options or {}, settings or {}) -end - -local function test_rpc_server(config) - if config.test_name then - clear_notrace() - fake_lsp_server_setup(config.test_name, config.timeout_ms or 1e3, config.options, config.settings) - end - local client = setmetatable({}, { - __index = function(_, name) - -- Workaround for not being able to yield() inside __index for Lua 5.1 :( - -- Otherwise I would just return the value here. - return function(...) - return exec_lua([=[ - local name = ... - if type(TEST_RPC_CLIENT[name]) == 'function' then - return TEST_RPC_CLIENT[name](select(2, ...)) - else - return TEST_RPC_CLIENT[name] - end - ]=], name, ...) - end - end; - }) - local code, signal - local function on_request(method, args) - if method == "init" then - if config.on_init then - config.on_init(client, unpack(args)) - end - return NIL - end - if method == 'handler' then - if config.on_handler then - config.on_handler(unpack(args)) - end - end - return NIL - end - local function on_notify(method, args) - if method == 'exit' then - code, signal = unpack(args) - return stop() - end - end - -- TODO specify timeout? - -- run(on_request, on_notify, config.on_setup, 1000) - run(on_request, on_notify, config.on_setup) - if config.on_exit then - config.on_exit(code, signal) - end - stop() - if config.test_name then - exec_lua("vim.api.nvim_exec_autocmds('VimLeavePre', { modeline = false })") - end -end - describe('LSP', function() before_each(function() clear_notrace() |