From f140175564001cc1ad84128a0147a5d2f7798a63 Mon Sep 17 00:00:00 2001 From: Mathias Fußenegger Date: Thu, 23 Feb 2023 13:35:46 +0100 Subject: refactor(lsp): remove workaround for missing bit module (#22373) --- runtime/lua/vim/lsp/semantic_tokens.lua | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index b1bc48dac6..00b4757ea9 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -1,6 +1,7 @@ local api = vim.api local handlers = require('vim.lsp.handlers') local util = require('vim.lsp.util') +local bit = require('bit') --- @class STTokenRange --- @field line number line number 0-based @@ -58,18 +59,10 @@ local function modifiers_from_number(x, modifiers_table) local modifiers = {} local idx = 1 while x > 0 do - if _G.bit then - if _G.bit.band(x, 1) == 1 then - modifiers[#modifiers + 1] = modifiers_table[idx] - end - x = _G.bit.rshift(x, 1) - else - --TODO(jdrouhard): remove this branch once `bit` module is available for non-LuaJIT (#21222) - if x % 2 == 1 then - modifiers[#modifiers + 1] = modifiers_table[idx] - end - x = math.floor(x / 2) + if bit.band(x, 1) == 1 then + modifiers[#modifiers + 1] = modifiers_table[idx] end + x = bit.rshift(x, 1) idx = idx + 1 end -- cgit From 7e19cabeb192d2e7f20d7bb965a3f62e1543d2ac Mon Sep 17 00:00:00 2001 From: bfredl Date: Tue, 28 Feb 2023 12:38:33 +0100 Subject: perf(lsp): only redraw the windows containing LSP tokens redraw! redraws the entire screen instead of just the windows with the buffer which were actually changed. I considered trying to calculating the range for the delta but it looks tricky. Could a follow-up. --- runtime/lua/vim/lsp/semantic_tokens.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index 00b4757ea9..24b5c6c24e 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -334,7 +334,8 @@ function STHighlighter:process_response(response, client, version) current_result.highlights = tokens_to_ranges(tokens, self.bufnr, client) current_result.namespace_cleared = false - api.nvim_command('redraw!') + -- redraw all windows displaying buffer + api.nvim__buf_redraw_range(self.bufnr, 0, -1) end --- on_win handler for the decoration provider (see |nvim_set_decoration_provider|) -- cgit From 1cc23e1109ed88275df5c986c352f73b99a0301c Mon Sep 17 00:00:00 2001 From: swarn Date: Mon, 6 Mar 2023 12:03:13 -0600 Subject: feat(lsp)!: add rule-based sem token highlighting (#22022) feat(lsp)!: change semantic token highlighting Change the default highlights used, and add more highlights per token. Add an LspTokenUpdate event and a highlight_token function. :Inspect now shows any highlights applied by token highlighting rules, default or user-defined. BREAKING CHANGE: change the default highlight groups used by semantic token highlighting. --- runtime/lua/vim/lsp/semantic_tokens.lua | 183 ++++++++++++++++++++++---------- 1 file changed, 125 insertions(+), 58 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index 24b5c6c24e..7983d066b8 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -8,8 +8,8 @@ local bit = require('bit') --- @field start_col number start column 0-based --- @field end_col number end column 0-based --- @field type string token type as string ---- @field modifiers string[] token modifiers as strings ---- @field extmark_added boolean whether this extmark has been added to the buffer yet +--- @field modifiers table token modifiers as a set. E.g., { static = true, readonly = true } +--- @field marked boolean whether this token has had extmarks applied --- --- @class STCurrentResult --- @field version number document version associated with this result @@ -36,10 +36,13 @@ local bit = require('bit') ---@field client_state table local STHighlighter = { active = {} } +--- Do a binary search of the tokens in the half-open range [lo, hi). +--- +--- Return the index i in range such that tokens[j].line < line for all j < i, and +--- tokens[j].line >= line for all j >= i, or return hi if no such index is found. +--- ---@private -local function binary_search(tokens, line) - local lo = 1 - local hi = #tokens +local function lower_bound(tokens, line, lo, hi) while lo < hi do local mid = math.floor((lo + hi) / 2) if tokens[mid].line < line then @@ -51,16 +54,34 @@ local function binary_search(tokens, line) return lo end +--- Do a binary search of the tokens in the half-open range [lo, hi). +--- +--- Return the index i in range such that tokens[j].line <= line for all j < i, and +--- tokens[j].line > line for all j >= i, or return hi if no such index is found. +--- +---@private +local function upper_bound(tokens, line, lo, hi) + while lo < hi do + local mid = math.floor((lo + hi) / 2) + if line < tokens[mid].line then + hi = mid + else + lo = mid + 1 + end + end + return lo +end + --- Extracts modifier strings from the encoded number in the token array --- ---@private ----@return string[] +---@return table local function modifiers_from_number(x, modifiers_table) local modifiers = {} local idx = 1 while x > 0 do if bit.band(x, 1) == 1 then - modifiers[#modifiers + 1] = modifiers_table[idx] + modifiers[modifiers_table[idx]] = true end x = bit.rshift(x, 1) idx = idx + 1 @@ -109,7 +130,7 @@ local function tokens_to_ranges(data, bufnr, client) end_col = end_col, type = token_type, modifiers = modifiers, - extmark_added = false, + marked = false, } end end @@ -355,7 +376,7 @@ end --- ---@private function STHighlighter:on_win(topline, botline) - for _, state in pairs(self.client_state) do + for client_id, state in pairs(self.client_state) do local current_result = state.current_result if current_result.version and current_result.version == util.buf_versions[self.bufnr] then if not current_result.namespace_cleared then @@ -372,52 +393,55 @@ function STHighlighter:on_win(topline, botline) -- -- Instead, we have to use normal extmarks that can attach to locations -- in the buffer and are persisted between redraws. + -- + -- `strict = false` is necessary here for the 1% of cases where the + -- current result doesn't actually match the buffer contents. Some + -- LSP servers can respond with stale tokens on requests if they are + -- still processing changes from a didChange notification. + -- + -- LSP servers that do this _should_ follow up known stale responses + -- with a refresh notification once they've finished processing the + -- didChange notification, which would re-synchronize the tokens from + -- our end. + -- + -- The server I know of that does this is clangd when the preamble of + -- a file changes and the token request is processed with a stale + -- preamble while the new one is still being built. Once the preamble + -- finishes, clangd sends a refresh request which lets the client + -- re-synchronize the tokens. + + local set_mark = function(token, hl_group, delta) + vim.api.nvim_buf_set_extmark(self.bufnr, state.namespace, token.line, token.start_col, { + hl_group = hl_group, + end_col = token.end_col, + priority = vim.highlight.priorities.semantic_tokens + delta, + strict = false, + }) + end + + local ft = vim.bo[self.bufnr].filetype local highlights = current_result.highlights - local idx = binary_search(highlights, topline) + local first = lower_bound(highlights, topline, 1, #highlights + 1) + local last = upper_bound(highlights, botline, first, #highlights + 1) - 1 - for i = idx, #highlights do + for i = first, last do local token = highlights[i] - - if token.line > botline then - break - end - - if not token.extmark_added then - -- `strict = false` is necessary here for the 1% of cases where the - -- current result doesn't actually match the buffer contents. Some - -- LSP servers can respond with stale tokens on requests if they are - -- still processing changes from a didChange notification. - -- - -- LSP servers that do this _should_ follow up known stale responses - -- with a refresh notification once they've finished processing the - -- didChange notification, which would re-synchronize the tokens from - -- our end. - -- - -- The server I know of that does this is clangd when the preamble of - -- a file changes and the token request is processed with a stale - -- preamble while the new one is still being built. Once the preamble - -- finishes, clangd sends a refresh request which lets the client - -- re-synchronize the tokens. - api.nvim_buf_set_extmark(self.bufnr, state.namespace, token.line, token.start_col, { - hl_group = '@' .. token.type, - end_col = token.end_col, - priority = vim.highlight.priorities.semantic_tokens, - strict = false, - }) - - -- TODO(bfredl) use single extmark when hl_group supports table - if #token.modifiers > 0 then - for _, modifier in pairs(token.modifiers) do - api.nvim_buf_set_extmark(self.bufnr, state.namespace, token.line, token.start_col, { - hl_group = '@' .. modifier, - end_col = token.end_col, - priority = vim.highlight.priorities.semantic_tokens + 1, - strict = false, - }) - end + if not token.marked then + set_mark(token, string.format('@lsp.type.%s.%s', token.type, ft), 0) + for modifier, _ in pairs(token.modifiers) do + set_mark(token, string.format('@lsp.mod.%s.%s', modifier, ft), 1) + set_mark(token, string.format('@lsp.typemod.%s.%s.%s', token.type, modifier, ft), 2) end - - token.extmark_added = true + token.marked = true + + api.nvim_exec_autocmds('LspTokenUpdate', { + pattern = vim.api.nvim_buf_get_name(self.bufnr), + modeline = false, + data = { + token = token, + client_id = client_id, + }, + }) end end end @@ -588,7 +612,13 @@ end ---@param row number|nil Position row (default cursor position) ---@param col number|nil Position column (default cursor position) --- ----@return table|nil (table|nil) List of tokens at position +---@return table|nil (table|nil) List of tokens at position. Each token has +--- the following fields: +--- - line (number) line number, 0-based +--- - start_col (number) start column, 0-based +--- - end_col (number) end column, 0-based +--- - type (string) token type as string, e.g. "variable" +--- - modifiers (table) token modifiers as a set. E.g., { static = true, readonly = true } function M.get_at_pos(bufnr, row, col) if bufnr == nil or bufnr == 0 then bufnr = api.nvim_get_current_buf() @@ -608,7 +638,7 @@ function M.get_at_pos(bufnr, row, col) for client_id, client in pairs(highlighter.client_state) do local highlights = client.current_result.highlights if highlights then - local idx = binary_search(highlights, row) + local idx = lower_bound(highlights, row, 1, #highlights + 1) for i = idx, #highlights do local token = highlights[i] @@ -631,23 +661,60 @@ end --- Only has an effect if the buffer is currently active for semantic token --- highlighting (|vim.lsp.semantic_tokens.start()| has been called for it) --- ----@param bufnr (nil|number) default: current buffer +---@param bufnr (number|nil) filter by buffer. All buffers if nil, current +--- buffer if 0 function M.force_refresh(bufnr) vim.validate({ bufnr = { bufnr, 'n', true }, }) - if bufnr == nil or bufnr == 0 then - bufnr = api.nvim_get_current_buf() + local buffers = bufnr == nil and vim.tbl_keys(STHighlighter.active) + or bufnr == 0 and { api.nvim_get_current_buf() } + or { bufnr } + + for _, buffer in ipairs(buffers) do + local highlighter = STHighlighter.active[buffer] + if highlighter then + highlighter:reset() + highlighter:send_request() + end end +end +--- Highlight a semantic token. +--- +--- Apply an extmark with a given highlight group for a semantic token. The +--- mark will be deleted by the semantic token engine when appropriate; for +--- example, when the LSP sends updated tokens. This function is intended for +--- use inside |LspTokenUpdate| callbacks. +---@param token (table) a semantic token, found as `args.data.token` in +--- |LspTokenUpdate|. +---@param bufnr (number) the buffer to highlight +---@param client_id (number) The ID of the |vim.lsp.client| +---@param hl_group (string) Highlight group name +---@param opts (table|nil) Optional parameters. +--- - priority: (number|nil) Priority for the applied extmark. Defaults +--- to `vim.highlight.priorities.semantic_tokens + 3` +function M.highlight_token(token, bufnr, client_id, hl_group, opts) local highlighter = STHighlighter.active[bufnr] if not highlighter then return end - highlighter:reset() - highlighter:send_request() + local state = highlighter.client_state[client_id] + if not state then + return + end + + opts = opts or {} + local priority = opts.priority or vim.highlight.priorities.semantic_tokens + 3 + + vim.api.nvim_buf_set_extmark(bufnr, state.namespace, token.line, token.start_col, { + hl_group = hl_group, + end_col = token.end_col, + priority = priority, + strict = false, + }) end --- |lsp-handler| for the method `workspace/semanticTokens/refresh` -- cgit From 706bcab75eaad2c370d61bf828531054439d3a3e Mon Sep 17 00:00:00 2001 From: Jaehwang Jung Date: Tue, 7 Mar 2023 15:17:52 +0900 Subject: docs(lsp): change type annotations from number → integer (#22510) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- runtime/lua/vim/lsp/semantic_tokens.lua | 60 ++++++++++++++++----------------- 1 file changed, 30 insertions(+), 30 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index 7983d066b8..9eaccd539f 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -4,36 +4,36 @@ local util = require('vim.lsp.util') local bit = require('bit') --- @class STTokenRange ---- @field line number line number 0-based ---- @field start_col number start column 0-based ---- @field end_col number end column 0-based +--- @field line integer line number 0-based +--- @field start_col integer start column 0-based +--- @field end_col integer end column 0-based --- @field type string token type as string --- @field modifiers table token modifiers as a set. E.g., { static = true, readonly = true } --- @field marked boolean whether this token has had extmarks applied --- --- @class STCurrentResult ---- @field version number document version associated with this result +--- @field version integer document version associated with this result --- @field result_id string resultId from the server; used with delta requests --- @field highlights STTokenRange[] cache of highlight ranges for this document version ---- @field tokens number[] raw token array as received by the server. used for calculating delta responses +--- @field tokens integer[] raw token array as received by the server. used for calculating delta responses --- @field namespace_cleared boolean whether the namespace was cleared for this result yet --- --- @class STActiveRequest ---- @field request_id number the LSP request ID of the most recent request sent to the server ---- @field version number the document version associated with the most recent request +--- @field request_id integer the LSP request ID of the most recent request sent to the server +--- @field version integer the document version associated with the most recent request --- --- @class STClientState ---- @field namespace number +--- @field namespace integer --- @field active_request STActiveRequest --- @field current_result STCurrentResult ---@class STHighlighter ----@field active table ----@field bufnr number ----@field augroup number augroup for buffer events ----@field debounce number milliseconds to debounce requests for new tokens +---@field active table +---@field bufnr integer +---@field augroup integer augroup for buffer events +---@field debounce integer milliseconds to debounce requests for new tokens ---@field timer table uv_timer for debouncing requests for new tokens ----@field client_state table +---@field client_state table local STHighlighter = { active = {} } --- Do a binary search of the tokens in the half-open range [lo, hi). @@ -141,7 +141,7 @@ end --- Construct a new STHighlighter for the buffer --- ---@private ----@param bufnr number +---@param bufnr integer function STHighlighter.new(bufnr) local self = setmetatable({}, { __index = STHighlighter }) @@ -470,7 +470,7 @@ end --- in case the server supports delta requests. --- ---@private ----@param client_id number +---@param client_id integer function STHighlighter:mark_dirty(client_id) local state = self.client_state[client_id] assert(state) @@ -529,10 +529,10 @@ local M = {} --- client.server_capabilities.semanticTokensProvider = nil --- --- ----@param bufnr number ----@param client_id number +---@param bufnr integer +---@param client_id integer ---@param opts (nil|table) Optional keyword arguments ---- - debounce (number, default: 200): Debounce token requests +--- - debounce (integer, default: 200): Debounce token requests --- to the server by the given number in milliseconds function M.start(bufnr, client_id, opts) vim.validate({ @@ -585,8 +585,8 @@ end --- of `start()`, so you should only need this function to manually disengage the semantic --- token engine without fully detaching the LSP client from the buffer. --- ----@param bufnr number ----@param client_id number +---@param bufnr integer +---@param client_id integer function M.stop(bufnr, client_id) vim.validate({ bufnr = { bufnr, 'n', false }, @@ -608,15 +608,15 @@ end --- Return the semantic token(s) at the given position. --- If called without arguments, returns the token under the cursor. --- ----@param bufnr number|nil Buffer number (0 for current buffer, default) ----@param row number|nil Position row (default cursor position) ----@param col number|nil Position column (default cursor position) +---@param bufnr integer|nil Buffer number (0 for current buffer, default) +---@param row integer|nil Position row (default cursor position) +---@param col integer|nil Position column (default cursor position) --- ---@return table|nil (table|nil) List of tokens at position. Each token has --- the following fields: ---- - line (number) line number, 0-based ---- - start_col (number) start column, 0-based ---- - end_col (number) end column, 0-based +--- - line (integer) line number, 0-based +--- - start_col (integer) start column, 0-based +--- - end_col (integer) end column, 0-based --- - type (string) token type as string, e.g. "variable" --- - modifiers (table) token modifiers as a set. E.g., { static = true, readonly = true } function M.get_at_pos(bufnr, row, col) @@ -661,7 +661,7 @@ end --- Only has an effect if the buffer is currently active for semantic token --- highlighting (|vim.lsp.semantic_tokens.start()| has been called for it) --- ----@param bufnr (number|nil) filter by buffer. All buffers if nil, current +---@param bufnr (integer|nil) filter by buffer. All buffers if nil, current --- buffer if 0 function M.force_refresh(bufnr) vim.validate({ @@ -689,11 +689,11 @@ end --- use inside |LspTokenUpdate| callbacks. ---@param token (table) a semantic token, found as `args.data.token` in --- |LspTokenUpdate|. ----@param bufnr (number) the buffer to highlight ----@param client_id (number) The ID of the |vim.lsp.client| +---@param bufnr (integer) the buffer to highlight +---@param client_id (integer) The ID of the |vim.lsp.client| ---@param hl_group (string) Highlight group name ---@param opts (table|nil) Optional parameters. ---- - priority: (number|nil) Priority for the applied extmark. Defaults +--- - priority: (integer|nil) Priority for the applied extmark. Defaults --- to `vim.highlight.priorities.semantic_tokens + 3` function M.highlight_token(token, bufnr, client_id, hl_group, opts) local highlighter = STHighlighter.active[bufnr] -- cgit From 4385f8a7430f8181189f385a6cfb4e295d30b21e Mon Sep 17 00:00:00 2001 From: jdrouhard Date: Tue, 7 Mar 2023 10:35:12 -0600 Subject: fix(lsp): change LspTokenUpdate to use buffer instead of pattern (#22559) --- runtime/lua/vim/lsp/semantic_tokens.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index 9eaccd539f..a5e007a011 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -435,7 +435,7 @@ function STHighlighter:on_win(topline, botline) token.marked = true api.nvim_exec_autocmds('LspTokenUpdate', { - pattern = vim.api.nvim_buf_get_name(self.bufnr), + buffer = self.bufnr, modeline = false, data = { token = token, -- cgit From 75537768ef0b8cc35ef9c6aa906237e449640b46 Mon Sep 17 00:00:00 2001 From: Null Chilly <56817415+nullchilly@users.noreply.github.com> Date: Fri, 10 Mar 2023 20:10:38 +0700 Subject: perf(lsp): better binary search mid calculation in semantic token (#22607) This commit replaces the usage of math.floor((lo + hi) / 2) with the faster and equivalent bit.rshift(lo + hi, 1) for calculating the midpoint in binary search. --- runtime/lua/vim/lsp/semantic_tokens.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index a5e007a011..03532c33b7 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -44,7 +44,7 @@ local STHighlighter = { active = {} } ---@private local function lower_bound(tokens, line, lo, hi) while lo < hi do - local mid = math.floor((lo + hi) / 2) + local mid = bit.rshift(lo + hi, 1) -- Equivalent to floor((lo + hi) / 2). if tokens[mid].line < line then lo = mid + 1 else @@ -62,7 +62,7 @@ end ---@private local function upper_bound(tokens, line, lo, hi) while lo < hi do - local mid = math.floor((lo + hi) / 2) + local mid = bit.rshift(lo + hi, 1) -- Equivalent to floor((lo + hi) / 2). if line < tokens[mid].line then hi = mid else -- cgit From edf05b005f34f59dd40468c36cc139e217345a71 Mon Sep 17 00:00:00 2001 From: jdrouhard Date: Mon, 1 May 2023 00:15:32 -0500 Subject: perf(lsp): process semantic tokens response in a coroutine that yields every 5ms (#23375) --- runtime/lua/vim/lsp/semantic_tokens.lua | 45 +++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index 03532c33b7..b6b09c58b1 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -1,7 +1,8 @@ local api = vim.api +local bit = require('bit') local handlers = require('vim.lsp.handlers') local util = require('vim.lsp.util') -local bit = require('bit') +local uv = vim.loop --- @class STTokenRange --- @field line integer line number 0-based @@ -94,15 +95,38 @@ end --- ---@private ---@return STTokenRange[] -local function tokens_to_ranges(data, bufnr, client) +local function tokens_to_ranges(data, bufnr, client, request) local legend = client.server_capabilities.semanticTokensProvider.legend local token_types = legend.tokenTypes local token_modifiers = legend.tokenModifiers local ranges = {} + local start = uv.hrtime() + local ms_to_ns = 1000 * 1000 + local yield_interval_ns = 5 * ms_to_ns + local co, is_main = coroutine.running() + local line local start_char = 0 for i = 1, #data, 5 do + -- if this function is called from the main coroutine, let it run to completion with no yield + if not is_main then + local elapsed_ns = uv.hrtime() - start + + if elapsed_ns > yield_interval_ns then + vim.schedule(function() + coroutine.resume(co, util.buf_versions[bufnr]) + end) + if request.version ~= coroutine.yield() then + -- request became stale since the last time the coroutine ran. + -- abandon it by yielding without a way to resume + coroutine.yield() + end + + start = uv.hrtime() + end + end + local delta_line = data[i] line = line and line + delta_line or delta_line local delta_start = data[i + 1] @@ -280,7 +304,7 @@ function STHighlighter:send_request() local c = vim.lsp.get_client_by_id(ctx.client_id) local highlighter = STHighlighter.active[ctx.bufnr] if not err and c and highlighter then - highlighter:process_response(response, c, version) + coroutine.wrap(STHighlighter.process_response)(highlighter, response, c, version) end end, self.bufnr) @@ -315,11 +339,9 @@ function STHighlighter:process_response(response, client, version) return end - -- reset active request - state.active_request = {} - -- skip nil responses if response == nil then + state.active_request = {} return end @@ -347,12 +369,19 @@ function STHighlighter:process_response(response, client, version) tokens = response.data end - -- Update the state with the new results + -- convert token list to highlight ranges + -- this could yield and run over multiple event loop iterations + local highlights = tokens_to_ranges(tokens, self.bufnr, client, state.active_request) + + -- reset active request + state.active_request = {} + + -- update the state with the new results local current_result = state.current_result current_result.version = version current_result.result_id = response.resultId current_result.tokens = tokens - current_result.highlights = tokens_to_ranges(tokens, self.bufnr, client) + current_result.highlights = highlights current_result.namespace_cleared = false -- redraw all windows displaying buffer -- cgit From 648f777931d49b8013146f69d7e2776f69c52900 Mon Sep 17 00:00:00 2001 From: jdrouhard Date: Fri, 5 May 2023 00:41:36 -0500 Subject: perf(lsp): load buffer contents once when processing semantic tokens responses (#23484) perf(lsp): load buffer contents once when processing semantic token responses Using _get_line_byte_from_position() for each token's boundaries was a pretty huge bottleneck, since that function would load individual buffer lines via nvim_buf_get_lines() (plus a lot of extra overhead). So each token caused two calls to nvim_buf_get_lines() (once for the start position, and once for the end position). For semantic tokens, we only attach to buffers that have already been loaded, so we can safely just get all the lines for the entire buffer at once, and lift the rest of the _get_line_byte_from_position() implementation directly while bypassing the part that loads the buffer line. While I was looking at get_lines (used by _get_line_byte_from_position), I noticed that we were checking for non-file URIs before we even looked to see if we already had the buffer loaded. Moving the buffer-loaded check to be the first thing done in get_lines() more than halved the average time spent transforming the token list into highlight ranges vs when it was still using _get_line_byte_from_position. I ended up improving that loop more by not using get_lines, but figured the performance improvement it provided was worth leaving in. --- runtime/lua/vim/lsp/semantic_tokens.lua | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index b6b09c58b1..376cac19a7 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -99,6 +99,7 @@ local function tokens_to_ranges(data, bufnr, client, request) local legend = client.server_capabilities.semanticTokensProvider.legend local token_types = legend.tokenTypes local token_modifiers = legend.tokenModifiers + local lines = api.nvim_buf_get_lines(bufnr, 0, -1, false) local ranges = {} local start = uv.hrtime() @@ -137,11 +138,17 @@ local function tokens_to_ranges(data, bufnr, client, request) local modifiers = modifiers_from_number(data[i + 4], token_modifiers) ---@private - local function _get_byte_pos(char_pos) - return util._get_line_byte_from_position(bufnr, { - line = line, - character = char_pos, - }, client.offset_encoding) + local function _get_byte_pos(col) + if col > 0 then + local buf_line = lines[line + 1] or '' + local ok, result + ok, result = pcall(util._str_byteindex_enc, buf_line, col, client.offset_encoding) + if ok then + return result + end + return math.min(#buf_line, col) + end + return col end local start_col = _get_byte_pos(start_char) -- cgit From 2db719f6c2b677fcbc197b02fe52764a851523b2 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Sat, 3 Jun 2023 11:06:00 +0100 Subject: feat(lua): rename vim.loop -> vim.uv (#22846) --- runtime/lua/vim/lsp/semantic_tokens.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index 376cac19a7..191cc7b400 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -2,7 +2,7 @@ local api = vim.api local bit = require('bit') local handlers = require('vim.lsp.handlers') local util = require('vim.lsp.util') -local uv = vim.loop +local uv = vim.uv --- @class STTokenRange --- @field line integer line number 0-based -- cgit From be74807eef13ff8c90d55cf8b22b01d6d33b1641 Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Tue, 18 Jul 2023 15:42:30 +0100 Subject: docs(lua): more improvements (#24387) * docs(lua): teach lua2dox how to table * docs(lua): teach gen_vimdoc.py about local functions No more need to mark local functions with @private * docs(lua): mention @nodoc and @meta in dev-lua-doc * fixup! Co-authored-by: Justin M. Keyes --------- Co-authored-by: Justin M. Keyes --- runtime/lua/vim/lsp/semantic_tokens.lua | 7 ------- 1 file changed, 7 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index 191cc7b400..84723bbc05 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -41,8 +41,6 @@ local STHighlighter = { active = {} } --- --- Return the index i in range such that tokens[j].line < line for all j < i, and --- tokens[j].line >= line for all j >= i, or return hi if no such index is found. ---- ----@private local function lower_bound(tokens, line, lo, hi) while lo < hi do local mid = bit.rshift(lo + hi, 1) -- Equivalent to floor((lo + hi) / 2). @@ -59,8 +57,6 @@ end --- --- Return the index i in range such that tokens[j].line <= line for all j < i, and --- tokens[j].line > line for all j >= i, or return hi if no such index is found. ---- ----@private local function upper_bound(tokens, line, lo, hi) while lo < hi do local mid = bit.rshift(lo + hi, 1) -- Equivalent to floor((lo + hi) / 2). @@ -75,7 +71,6 @@ end --- Extracts modifier strings from the encoded number in the token array --- ----@private ---@return table local function modifiers_from_number(x, modifiers_table) local modifiers = {} @@ -93,7 +88,6 @@ end --- Converts a raw token list to a list of highlight ranges used by the on_win callback --- ----@private ---@return STTokenRange[] local function tokens_to_ranges(data, bufnr, client, request) local legend = client.server_capabilities.semanticTokensProvider.legend @@ -137,7 +131,6 @@ local function tokens_to_ranges(data, bufnr, client, request) local token_type = token_types[data[i + 3] + 1] local modifiers = modifiers_from_number(data[i + 4], token_modifiers) - ---@private local function _get_byte_pos(col) if col > 0 then local buf_line = lines[line + 1] or '' -- cgit From f1772272b4fda43c093fc495f54b5e7c11968d62 Mon Sep 17 00:00:00 2001 From: Raphael Date: Thu, 3 Aug 2023 19:03:48 +0800 Subject: refactor(lsp): use protocol.Methods instead of strings #24537 --- runtime/lua/vim/lsp/semantic_tokens.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index 84723bbc05..bab1b23ee4 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -1,6 +1,7 @@ local api = vim.api local bit = require('bit') local handlers = require('vim.lsp.handlers') +local ms = require('vim.lsp.protocol').Methods local util = require('vim.lsp.util') local uv = vim.uv @@ -292,7 +293,7 @@ function STHighlighter:send_request() local hasEditProvider = type(spec) == 'table' and spec.delta local params = { textDocument = util.make_text_document_params(self.bufnr) } - local method = 'textDocument/semanticTokens/full' + local method = ms.textDocument_semanticTokens_full if hasEditProvider and current_result.result_id then method = method .. '/delta' @@ -755,7 +756,7 @@ end --- the BufWinEnter event should take care of it next time it's displayed. --- ---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#semanticTokens_refreshRequest -handlers['workspace/semanticTokens/refresh'] = function(err, _, ctx) +handlers[ms.workspace_semanticTokens_refresh] = function(err, _, ctx) if err then return vim.NIL end -- cgit From c43c745a14dced87a23227d7be4f1c33d4455193 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Wed, 9 Aug 2023 11:06:13 +0200 Subject: fix(lua): improve annotations for stricter luals diagnostics (#24609) Problem: luals returns stricter diagnostics with bundled luarc.json Solution: Improve some function and type annotations: * use recognized uv.* types * disable diagnostic for global `vim` in shared.lua * docs: don't start comment lines with taglink (otherwise LuaLS will interpret it as a type) * add type alias for lpeg pattern * fix return annotation for `vim.secure.trust` * rename local Range object in vim.version (shadows `Range` in vim.treesitter) * fix some "missing fields" warnings * add missing required fields for test functions in eval.lua * rename lsp meta files for consistency --- runtime/lua/vim/lsp/semantic_tokens.lua | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'runtime/lua/vim/lsp/semantic_tokens.lua') diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua index bab1b23ee4..5b20344bd3 100644 --- a/runtime/lua/vim/lsp/semantic_tokens.lua +++ b/runtime/lua/vim/lsp/semantic_tokens.lua @@ -14,11 +14,11 @@ local uv = vim.uv --- @field marked boolean whether this token has had extmarks applied --- --- @class STCurrentResult ---- @field version integer document version associated with this result ---- @field result_id string resultId from the server; used with delta requests ---- @field highlights STTokenRange[] cache of highlight ranges for this document version ---- @field tokens integer[] raw token array as received by the server. used for calculating delta responses ---- @field namespace_cleared boolean whether the namespace was cleared for this result yet +--- @field version? integer document version associated with this result +--- @field result_id? string resultId from the server; used with delta requests +--- @field highlights? STTokenRange[] cache of highlight ranges for this document version +--- @field tokens? integer[] raw token array as received by the server. used for calculating delta responses +--- @field namespace_cleared? boolean whether the namespace was cleared for this result yet --- --- @class STActiveRequest --- @field request_id integer the LSP request ID of the most recent request sent to the server @@ -717,8 +717,7 @@ end --- mark will be deleted by the semantic token engine when appropriate; for --- example, when the LSP sends updated tokens. This function is intended for --- use inside |LspTokenUpdate| callbacks. ----@param token (table) a semantic token, found as `args.data.token` in ---- |LspTokenUpdate|. +---@param token (table) a semantic token, found as `args.data.token` in |LspTokenUpdate|. ---@param bufnr (integer) the buffer to highlight ---@param client_id (integer) The ID of the |vim.lsp.client| ---@param hl_group (string) Highlight group name -- cgit From 2e92065686f62851318150a315591c30b8306a4b Mon Sep 17 00:00:00 2001 From: Gregory Anders <8965202+gpanders@users.noreply.github.com> Date: Thu, 14 Sep 2023 08:23:01 -0500 Subject: docs: replace 
 with ``` (#25136)

---
 runtime/lua/vim/lsp/semantic_tokens.lua | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'runtime/lua/vim/lsp/semantic_tokens.lua')

diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua
index 5b20344bd3..a5831c0beb 100644
--- a/runtime/lua/vim/lsp/semantic_tokens.lua
+++ b/runtime/lua/vim/lsp/semantic_tokens.lua
@@ -555,9 +555,10 @@ local M = {}
 --- delete the semanticTokensProvider table from the {server_capabilities} of
 --- your client in your |LspAttach| callback or your configuration's
 --- `on_attach` callback:
---- 
lua
----   client.server_capabilities.semanticTokensProvider = nil
---- 

+---
+--- ```lua
+--- client.server_capabilities.semanticTokensProvider = nil
+--- ```
 ---
 ---@param bufnr integer
 ---@param client_id integer
-- 
cgit