Merge pull request #21393 from folke/highlight_show

feat(lsp): add function to get semantic tokens at cursor
feat: `vim.inspect_pos()`, `vim.show_pos()` and `:Inspect[!]`

13 files changed, 437 insertions, 6 deletions

diff --git a/runtime/doc/api.txt b/runtime/doc/api.txt
index d555cff443..8a33fc58a3 100644
--- a/runtime/doc/api.txt
+++ b/runtime/doc/api.txt
@@ -2683,7 +2683,7 @@ nvim_buf_set_extmark({buffer}, {ns_id}, {line}, {col}, {*opts})
         Id of the created/updated extmark
 
 nvim_create_namespace({name})                        *nvim_create_namespace()*
-    Creates a new *namespace* or gets an existing one.
+    Creates a new namespace or gets an existing one.               *namespace*
 
     Namespaces are used for buffer highlights and virtual text, see
     |nvim_buf_add_highlight()| and |nvim_buf_set_extmark()|.
diff --git a/runtime/doc/lsp.txt b/runtime/doc/lsp.txt
index 75d5c067b1..b101740b03 100644
--- a/runtime/doc/lsp.txt
+++ b/runtime/doc/lsp.txt
@@ -1332,6 +1332,19 @@ force_refresh({bufnr})               *vim.lsp.semantic_tokens.force_refresh()*
     Parameters: ~
       • {bufnr}  (nil|number) default: current buffer
 
+                                        *vim.lsp.semantic_tokens.get_at_pos()*
+get_at_pos({bufnr}, {row}, {col})
+    Return the semantic token(s) at the given position. If called without
+    arguments, returns the token under the cursor.
+
+    Parameters: ~
+      • {bufnr}  (number|nil) Buffer number (0 for current buffer, default)
+      • {row}    (number|nil) Position row (default cursor position)
+      • {col}    (number|nil) Position column (default cursor position)
+
+    Return: ~
+        (table|nil) List of tokens at position
+
 start({bufnr}, {client_id}, {opts})          *vim.lsp.semantic_tokens.start()*
     Start the semantic token highlighting engine for the given buffer with the
     given client. The client must already be attached to the buffer.
diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
index 5a1c186192..9c98ed7771 100644
--- a/runtime/doc/lua.txt
+++ b/runtime/doc/lua.txt
@@ -1504,6 +1504,56 @@ schedule_wrap({cb})                                      *vim.schedule_wrap()*
         |vim.in_fast_event()|
 
 
+==============================================================================
+Lua module: inspector                                          *lua-inspector*
+
+inspect_pos({bufnr}, {row}, {col}, {filter})               *vim.inspect_pos()*
+    Get all the items at a given buffer position.
+
+    Can also be pretty-printed with `:Inspect!`.                   *:Inspect!*
+
+    Parameters: ~
+      • {bufnr}   (number|nil) defaults to the current buffer
+      • {row}     (number|nil) row to inspect, 0-based. Defaults to the row of
+                  the current cursor
+      • {col}     (number|nil) col to inspect, 0-based. Defaults to the col of
+                  the current cursor
+      • {filter}  (table|nil) a table with key-value pairs to filter the items
+                  • syntax (boolean): include syntax based highlight groups
+                    (defaults to true)
+                  • treesitter (boolean): include treesitter based highlight
+                    groups (defaults to true)
+                  • extmarks (boolean|"all"): include extmarks. When `all`,
+                    then extmarks without a `hl_group` will also be included
+                    (defaults to true)
+                  • semantic_tokens (boolean): include semantic tokens
+                    (defaults to true)
+
+    Return: ~
+        (table) a table with the following key-value pairs. Items are in
+        "traversal order":
+        • treesitter: a list of treesitter captures
+        • syntax: a list of syntax groups
+        • semantic_tokens: a list of semantic tokens
+        • extmarks: a list of extmarks
+        • buffer: the buffer used to get the items
+        • row: the row used to get the items
+        • col: the col used to get the items
+
+show_pos({bufnr}, {row}, {col}, {filter})                     *vim.show_pos()*
+    Show all the items at a given buffer position.
+
+    Can also be shown with `:Inspect`.                              *:Inspect*
+
+    Parameters: ~
+      • {bufnr}   (number|nil) defaults to the current buffer
+      • {row}     (number|nil) row to inspect, 0-based. Defaults to the row of
+                  the current cursor
+      • {col}     (number|nil) col to inspect, 0-based. Defaults to the col of
+                  the current cursor
+      • {filter}  (table|nil) see |vim.inspect_pos()|
+
+
 
 
 deep_equal({a}, {b})                                        *vim.deep_equal()*
diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt
index bd0d1cfc5b..e5336edb5a 100644
--- a/runtime/doc/news.txt
+++ b/runtime/doc/news.txt
@@ -39,6 +39,10 @@ NEW FEATURES                                                    *news-features*
 
 The following new APIs or features were added.
 
+• |vim.inspect_pos()|, |vim.show_pos()| and |:Inspect| allow a user to get or show items
+  at a given buffer postion. Currently this includes treesitter captures,
+  semantic tokens, syntax groups and extmarks.
+
 • Added support for semantic token highlighting to the LSP client. This
   functionality is enabled by default when a client that supports this feature
   is attached to a buffer. Opt-out can be performed by deleting the
diff --git a/runtime/lua/vim/_init_packages.lua b/runtime/lua/vim/_init_packages.lua
index 19c8608732..0c4ee8636d 100644
--- a/runtime/lua/vim/_init_packages.lua
+++ b/runtime/lua/vim/_init_packages.lua
@@ -56,6 +56,9 @@ setmetatable(vim, {
     if vim._submodules[key] then
       t[key] = require('vim.' .. key)
       return t[key]
+    elseif key == 'inspect_pos' or key == 'show_pos' then
+      require('vim._inspector')
+      return t[key]
     elseif vim.startswith(key, 'uri_') then
       local val = require('vim.uri')[key]
       if val ~= nil then
diff --git a/runtime/lua/vim/_inspector.lua b/runtime/lua/vim/_inspector.lua
new file mode 100644
index 0000000000..f46a525910
--- /dev/null
+++ b/runtime/lua/vim/_inspector.lua
@@ -0,0 +1,238 @@
+---@class InspectorFilter
+---@field syntax boolean include syntax based highlight groups (defaults to true)
+---@field treesitter boolean include treesitter based highlight groups (defaults to true)
+---@field extmarks boolean|"all" include extmarks. When `all`, then extmarks without a `hl_group` will also be included (defaults to true)
+---@field semantic_tokens boolean include semantic tokens (defaults to true)
+local defaults = {
+  syntax = true,
+  treesitter = true,
+  extmarks = true,
+  semantic_tokens = true,
+}
+
+---Get all the items at a given buffer position.
+---
+---Can also be pretty-printed with `:Inspect!`. *:Inspect!*
+---
+---@param bufnr? number defaults to the current buffer
+---@param row? number row to inspect, 0-based. Defaults to the row of the current cursor
+---@param col? number col to inspect, 0-based. Defaults to the col of the current cursor
+---@param filter? InspectorFilter (table|nil) a table with key-value pairs to filter the items
+---               - syntax (boolean): include syntax based highlight groups (defaults to true)
+---               - treesitter (boolean): include treesitter based highlight groups (defaults to true)
+---               - extmarks (boolean|"all"): include extmarks. When `all`, then extmarks without a `hl_group` will also be included (defaults to true)
+---               - semantic_tokens (boolean): include semantic tokens (defaults to true)
+---@return {treesitter:table,syntax:table,extmarks:table,semantic_tokens:table,buffer:number,col:number,row:number} (table) a table with the following key-value pairs. Items are in "traversal order":
+---               - treesitter: a list of treesitter captures
+---               - syntax: a list of syntax groups
+---               - semantic_tokens: a list of semantic tokens
+---               - extmarks: a list of extmarks
+---               - buffer: the buffer used to get the items
+---               - row: the row used to get the items
+---               - col: the col used to get the items
+function vim.inspect_pos(bufnr, row, col, filter)
+  filter = vim.tbl_deep_extend('force', defaults, filter or {})
+
+  bufnr = bufnr or 0
+  if row == nil or col == nil then
+    -- get the row/col from the first window displaying the buffer
+    local win = bufnr == 0 and vim.api.nvim_get_current_win() or vim.fn.bufwinid(bufnr)
+    if win == -1 then
+      error('row/col is required for buffers not visible in a window')
+    end
+    local cursor = vim.api.nvim_win_get_cursor(win)
+    row, col = cursor[1] - 1, cursor[2]
+  end
+  bufnr = bufnr == 0 and vim.api.nvim_get_current_buf() or bufnr
+
+  local results = {
+    treesitter = {},
+    syntax = {},
+    extmarks = {},
+    semantic_tokens = {},
+    buffer = bufnr,
+    row = row,
+    col = col,
+  }
+
+  -- resolve hl links
+  ---@private
+  local function resolve_hl(data)
+    if data.hl_group then
+      local hlid = vim.api.nvim_get_hl_id_by_name(data.hl_group)
+      local name = vim.fn.synIDattr(vim.fn.synIDtrans(hlid), 'name')
+      data.hl_group_link = name
+    end
+    return data
+  end
+
+  -- treesitter
+  if filter.treesitter then
+    for _, capture in pairs(vim.treesitter.get_captures_at_pos(bufnr, row, col)) do
+      capture.hl_group = '@' .. capture.capture
+      table.insert(results.treesitter, resolve_hl(capture))
+    end
+  end
+
+  -- syntax
+  if filter.syntax then
+    for _, i1 in ipairs(vim.fn.synstack(row + 1, col + 1)) do
+      table.insert(results.syntax, resolve_hl({ hl_group = vim.fn.synIDattr(i1, 'name') }))
+    end
+  end
+
+  -- semantic tokens
+  if filter.semantic_tokens then
+    for _, token in ipairs(vim.lsp.semantic_tokens.get_at_pos(bufnr, row, col) or {}) do
+      token.hl_groups = {
+        type = resolve_hl({ hl_group = '@' .. token.type }),
+        modifiers = vim.tbl_map(function(modifier)
+          return resolve_hl({ hl_group = '@' .. modifier })
+        end, token.modifiers or {}),
+      }
+      table.insert(results.semantic_tokens, token)
+    end
+  end
+
+  -- extmarks
+  if filter.extmarks then
+    for ns, nsid in pairs(vim.api.nvim_get_namespaces()) do
+      if ns:find('vim_lsp_semantic_tokens') ~= 1 then
+        local extmarks = vim.api.nvim_buf_get_extmarks(bufnr, nsid, 0, -1, { details = true })
+        for _, extmark in ipairs(extmarks) do
+          extmark = {
+            ns_id = nsid,
+            ns = ns,
+            id = extmark[1],
+            row = extmark[2],
+            col = extmark[3],
+            opts = resolve_hl(extmark[4]),
+          }
+          local end_row = extmark.opts.end_row or extmark.row -- inclusive
+          local end_col = extmark.opts.end_col or (extmark.col + 1) -- exclusive
+          if
+            (filter.extmarks == 'all' or extmark.opts.hl_group) -- filter hl_group
+            and (row >= extmark.row and row <= end_row) -- within the rows of the extmark
+            and (row > extmark.row or col >= extmark.col) -- either not the first row, or in range of the col
+            and (row < end_row or col < end_col) -- either not in the last row or in range of the col
+          then
+            table.insert(results.extmarks, extmark)
+          end
+        end
+      end
+    end
+  end
+  return results
+end
+
+---Show all the items at a given buffer position.
+---
+---Can also be shown with `:Inspect`. *:Inspect*
+---
+---@param bufnr? number defaults to the current buffer
+---@param row? number row to inspect, 0-based. Defaults to the row of the current cursor
+---@param col? number col to inspect, 0-based. Defaults to the col of the current cursor
+---@param filter? InspectorFilter (table|nil) see |vim.inspect_pos()|
+function vim.show_pos(bufnr, row, col, filter)
+  local items = vim.inspect_pos(bufnr, row, col, filter)
+
+  local lines = { {} }
+
+  ---@private
+  local function append(str, hl)
+    table.insert(lines[#lines], { str, hl })
+  end
+
+  ---@private
+  local function nl()
+    table.insert(lines, {})
+  end
+
+  ---@private
+  local function item(data, comment)
+    append('  - ')
+    append(data.hl_group, data.hl_group)
+    append(' ')
+    if data.hl_group ~= data.hl_group_link then
+      append('links to ', 'MoreMsg')
+      append(data.hl_group_link, data.hl_group_link)
+      append(' ')
+    end
+    if comment then
+      append(comment, 'Comment')
+    end
+    nl()
+  end
+
+  -- treesitter
+  if #items.treesitter > 0 then
+    append('Treesitter', 'Title')
+    nl()
+    for _, capture in ipairs(items.treesitter) do
+      item(capture, capture.lang)
+    end
+    nl()
+  end
+
+  if #items.semantic_tokens > 0 then
+    append('Semantic Tokens', 'Title')
+    nl()
+    for _, token in ipairs(items.semantic_tokens) do
+      local client = vim.lsp.get_client_by_id(token.client_id)
+      client = client and (' (' .. client.name .. ')') or ''
+      item(token.hl_groups.type, 'type' .. client)
+      for _, modifier in ipairs(token.hl_groups.modifiers) do
+        item(modifier, 'modifier' .. client)
+      end
+    end
+    nl()
+  end
+
+  -- syntax
+  if #items.syntax > 0 then
+    append('Syntax', 'Title')
+    nl()
+    for _, syn in ipairs(items.syntax) do
+      item(syn)
+    end
+    nl()
+  end
+  -- extmarks
+  if #items.extmarks > 0 then
+    append('Extmarks', 'Title')
+    nl()
+    for _, extmark in ipairs(items.extmarks) do
+      if extmark.opts.hl_group then
+        item(extmark.opts, extmark.ns)
+      else
+        append('  - ')
+        append(extmark.ns, 'Comment')
+        nl()
+      end
+    end
+    nl()
+  end
+
+  if #lines[#lines] == 0 then
+    table.remove(lines)
+  end
+
+  local chunks = {}
+  for _, line in ipairs(lines) do
+    vim.list_extend(chunks, line)
+    table.insert(chunks, { '\n' })
+  end
+  if #chunks == 0 then
+    chunks = {
+      {
+        'No items found at position '
+          .. items.row
+          .. ','
+          .. items.col
+          .. ' in buffer '
+          .. items.buffer,
+      },
+    }
+  end
+  vim.api.nvim_echo(chunks, false, {})
+end
diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua
index b7ffedab2b..e14d3e51cd 100644
--- a/runtime/lua/vim/lsp/semantic_tokens.lua
+++ b/runtime/lua/vim/lsp/semantic_tokens.lua
@@ -585,6 +585,51 @@ function M.stop(bufnr, client_id)
   end
 end
 
+--- Return the semantic token(s) at the given position.
+--- If called without arguments, returns the token under the cursor.
+---
+---@param bufnr number|nil Buffer number (0 for current buffer, default)
+---@param row number|nil Position row (default cursor position)
+---@param col number|nil Position column (default cursor position)
+---
+---@return table|nil (table|nil) List of tokens at position
+function M.get_at_pos(bufnr, row, col)
+  if bufnr == nil or bufnr == 0 then
+    bufnr = api.nvim_get_current_buf()
+  end
+
+  local highlighter = STHighlighter.active[bufnr]
+  if not highlighter then
+    return
+  end
+
+  if row == nil or col == nil then
+    local cursor = api.nvim_win_get_cursor(0)
+    row, col = cursor[1] - 1, cursor[2]
+  end
+
+  local tokens = {}
+  for client_id, client in pairs(highlighter.client_state) do
+    local highlights = client.current_result.highlights
+    if highlights then
+      local idx = binary_search(highlights, row)
+      for i = idx, #highlights do
+        local token = highlights[i]
+
+        if token.line > row then
+          break
+        end
+
+        if token.start_col <= col and token.end_col > col then
+          token.client_id = client_id
+          tokens[#tokens + 1] = token
+        end
+      end
+    end
+  end
+  return tokens
+end
+
 --- Force a refresh of all semantic tokens
 ---
 --- Only has an effect if the buffer is currently active for semantic token
diff --git a/runtime/lua/vim/treesitter.lua b/runtime/lua/vim/treesitter.lua
index 25f0e7dc5e..582922ecb6 100644
--- a/runtime/lua/vim/treesitter.lua
+++ b/runtime/lua/vim/treesitter.lua
@@ -240,7 +240,7 @@ function M.get_captures_at_pos(bufnr, row, col)
       if M.is_in_node_range(node, row, col) then
         local c = q._query.captures[capture] -- name of the capture in the query
         if c ~= nil then
-          table.insert(matches, { capture = c, metadata = metadata })
+          table.insert(matches, { capture = c, metadata = metadata, lang = tree:lang() })
         end
       end
     end
diff --git a/runtime/plugin/nvim.lua b/runtime/plugin/nvim.lua
new file mode 100644
index 0000000000..815886f896
--- /dev/null
+++ b/runtime/plugin/nvim.lua
@@ -0,0 +1,7 @@
+vim.api.nvim_create_user_command('Inspect', function(cmd)
+  if cmd.bang then
+    vim.pretty_print(vim.inspect_pos())
+  else
+    vim.show_pos()
+  end
+end, { desc = 'Inspect highlights and extmarks at the cursor', bang = true })
diff --git a/scripts/gen_vimdoc.py b/scripts/gen_vimdoc.py
index b18179b498..502c9161b6 100755
--- a/scripts/gen_vimdoc.py
+++ b/scripts/gen_vimdoc.py
@@ -125,6 +125,7 @@ CONFIG = {
         'filename': 'lua.txt',
         'section_order': [
             '_editor.lua',
+            '_inspector.lua',
             'shared.lua',
             'uri.lua',
             'ui.lua',
@@ -142,11 +143,13 @@ CONFIG = {
             'runtime/lua/vim/keymap.lua',
             'runtime/lua/vim/fs.lua',
             'runtime/lua/vim/secure.lua',
+            'runtime/lua/vim/_inspector.lua',
         ],
         'file_patterns': '*.lua',
         'fn_name_prefix': '',
         'section_name': {
             'lsp.lua': 'core',
+            '_inspector.lua': 'inspector',
         },
         'section_fmt': lambda name: (
             'Lua module: vim'
@@ -163,6 +166,7 @@ CONFIG = {
         'module_override': {
             # `shared` functions are exposed on the `vim` module.
             'shared': 'vim',
+            '_inspector': 'vim',
             'uri': 'vim',
             'ui': 'vim.ui',
             'filetype': 'vim.filetype',
@@ -346,6 +350,17 @@ def self_or_child(n):
     return n.childNodes[0]
 
 
+def align_tags(line):
+    tag_regex = r"\s(\*.+?\*)(?:\s|$)"
+    tags = re.findall(tag_regex, line)
+
+    if len(tags) > 0:
+        line = re.sub(tag_regex, "", line)
+        tags = " " + " ".join(tags)
+        line = line + (" " * (78 - len(line) - len(tags))) + tags
+    return line
+
+
 def clean_lines(text):
     """Removes superfluous lines.
 
@@ -950,7 +965,7 @@ def fmt_doxygen_xml_as_vimhelp(filename, target):
 
             start = end
 
-        func_doc = "\n".join(split_lines)
+        func_doc = "\n".join(map(align_tags, split_lines))
 
         if (name.startswith(CONFIG[target]['fn_name_prefix'])
            and name != "nvim_error_event"):
diff --git a/src/nvim/api/extmark.c b/src/nvim/api/extmark.c
index bdc0900dd9..f7c6b398d5 100644
--- a/src/nvim/api/extmark.c
+++ b/src/nvim/api/extmark.c
@@ -40,7 +40,7 @@ void api_extmark_free_all_mem(void)
   map_destroy(String, handle_T)(&namespace_ids);
 }
 
-/// Creates a new \*namespace\* or gets an existing one.
+/// Creates a new namespace or gets an existing one. \*namespace\*
 ///
 /// Namespaces are used for buffer highlights and virtual text, see
 /// |nvim_buf_add_highlight()| and |nvim_buf_set_extmark()|.
diff --git a/test/functional/lua/inspector_spec.lua b/test/functional/lua/inspector_spec.lua
new file mode 100644
index 0000000000..5e488bb082
--- /dev/null
+++ b/test/functional/lua/inspector_spec.lua
@@ -0,0 +1,56 @@
+local helpers = require('test.functional.helpers')(after_each)
+local exec_lua = helpers.exec_lua
+local eq = helpers.eq
+local eval = helpers.eval
+local clear = helpers.clear
+
+describe('vim.inspect_pos', function()
+  before_each(function()
+    clear()
+  end)
+
+  it('it returns items', function()
+    local ret = exec_lua([[
+      local buf = vim.api.nvim_create_buf(true, false)
+      vim.api.nvim_set_current_buf(buf)
+      vim.api.nvim_buf_set_lines(0, 0, -1, false, {"local a = 123"})
+      vim.api.nvim_buf_set_option(buf, "filetype", "lua")
+      vim.cmd("syntax on")
+      return {buf, vim.inspect_pos(0, 0, 10)}
+    ]])
+    local buf, items = unpack(ret)
+    eq('', eval('v:errmsg'))
+    eq({
+      buffer = buf,
+      col = 10,
+      row = 0,
+      extmarks = {},
+      treesitter = {},
+      semantic_tokens = {},
+      syntax = {
+        {
+          hl_group = 'luaNumber',
+          hl_group_link = 'Constant',
+        },
+      },
+    }, items)
+  end)
+end)
+
+describe('vim.show_pos', function()
+  before_each(function()
+    clear()
+  end)
+
+  it('it does not error', function()
+    exec_lua([[
+      local buf = vim.api.nvim_create_buf(true, false)
+      vim.api.nvim_set_current_buf(buf)
+      vim.api.nvim_buf_set_lines(0, 0, -1, false, {"local a = 123"})
+      vim.api.nvim_buf_set_option(buf, "filetype", "lua")
+      vim.cmd("syntax on")
+      return {buf, vim.show_pos(0, 0, 10)}
+    ]])
+    eq('', eval('v:errmsg'))
+  end)
+end)
diff --git a/test/functional/treesitter/highlight_spec.lua b/test/functional/treesitter/highlight_spec.lua
index ae3f42ff0a..2a2311c0fa 100644
--- a/test/functional/treesitter/highlight_spec.lua
+++ b/test/functional/treesitter/highlight_spec.lua
@@ -605,8 +605,8 @@ describe('treesitter highlighting', function()
     }}
 
     eq({
-      {capture='Error', metadata = { priority='101' }};
-      {capture='type', metadata = { } };
+      {capture='Error', metadata = { priority='101' }, lang='c' };
+      {capture='type', metadata = { }, lang='c' };
     }, exec_lua [[ return vim.treesitter.get_captures_at_pos(0, 0, 2) ]])
     end)