Merge remote-tracking branch 'upstream/master' into userreguserreg

1 files changed, 349 insertions, 101 deletions

diff --git a/test/functional/plugin/lsp/semantic_tokens_spec.lua b/test/functional/plugin/lsp/semantic_tokens_spec.lua
index 9c1ba86fe1..b7ac53f270 100644
--- a/test/functional/plugin/lsp/semantic_tokens_spec.lua
+++ b/test/functional/plugin/lsp/semantic_tokens_spec.lua
@@ -24,6 +24,27 @@ end)
 
 describe('semantic token highlighting', function()
 
+  local screen
+  before_each(function()
+    screen = Screen.new(40, 16)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = { bold = true, foreground = Screen.colors.Blue1 };
+      [2] = { foreground = Screen.colors.DarkCyan };
+      [3] = { foreground = Screen.colors.SlateBlue };
+      [4] = { bold = true, foreground = Screen.colors.SeaGreen };
+      [5] = { foreground = tonumber('0x6a0dad') };
+      [6] = { foreground = Screen.colors.Blue1 };
+      [7] = { bold = true, foreground = Screen.colors.DarkCyan };
+      [8] = { bold = true, foreground = Screen.colors.SlateBlue };
+      [9] = { bold = true, foreground = tonumber('0x6a0dad') };
+    }
+    command([[ hi link @lsp.type.namespace Type ]])
+    command([[ hi link @lsp.type.function Special ]])
+    command([[ hi link @lsp.type.comment Comment ]])
+    command([[ hi @lsp.mod.declaration gui=bold ]])
+  end)
+
   describe('general', function()
     local text = dedent([[
     #include <iostream>
@@ -58,24 +79,7 @@ describe('semantic token highlighting', function()
       "resultId":"2"
     }]]
 
-    local screen
     before_each(function()
-      screen = Screen.new(40, 16)
-      screen:attach()
-      screen:set_default_attr_ids {
-        [1] = { bold = true, foreground = Screen.colors.Blue1 };
-        [2] = { foreground = Screen.colors.DarkCyan };
-        [3] = { foreground = Screen.colors.SlateBlue };
-        [4] = { bold = true, foreground = Screen.colors.SeaGreen };
-        [5] = { foreground = tonumber('0x6a0dad') };
-        [6] = { foreground = Screen.colors.Blue1 };
-        [7] = { bold = true, foreground = Screen.colors.DarkCyan };
-        [8] = { bold = true, foreground = Screen.colors.SlateBlue };
-      }
-      command([[ hi link @namespace Type ]])
-      command([[ hi link @function Special ]])
-      command([[ hi @declaration gui=bold ]])
-
       exec_lua(create_server_definition)
       exec_lua([[
         local legend, response, edit_response = ...
@@ -102,6 +106,7 @@ describe('semantic token highlighting', function()
       exec_lua([[
         bufnr = vim.api.nvim_get_current_buf()
         vim.api.nvim_win_set_buf(0, bufnr)
+        vim.bo[bufnr].filetype = 'some-filetype'
         client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
       ]])
 
@@ -127,6 +132,46 @@ describe('semantic token highlighting', function()
       ]] }
     end)
 
+    it('use LspTokenUpdate and highlight_token', function()
+      exec_lua([[
+        vim.api.nvim_create_autocmd("LspTokenUpdate", {
+          callback = function(args)
+            local token = args.data.token
+            if token.type == "function" and token.modifiers.declaration then
+              vim.lsp.semantic_tokens.highlight_token(
+                token, args.buf, args.data.client_id, "Macro"
+              )
+            end
+          end,
+        })
+        bufnr = vim.api.nvim_get_current_buf()
+        vim.api.nvim_win_set_buf(0, bufnr)
+        client_id = vim.lsp.start({ name = 'dummy', cmd = server.cmd })
+      ]])
+
+      insert(text)
+
+      screen:expect { grid = [[
+        #include <iostream>                     |
+                                                |
+        int {9:main}()                              |
+        {                                       |
+            int {7:x};                              |
+        #ifdef {5:__cplusplus}                      |
+            {4:std}::{2:cout} << {2:x} << "\n";             |
+        {6:#else}                                   |
+        {6:    printf("%d\n", x);}                  |
+        {6:#endif}                                  |
+        }                                       |
+        ^}                                       |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]] }
+
+    end)
+
     it('buffer is unhighlighted when client is detached', function()
       exec_lua([[
         bufnr = vim.api.nvim_get_current_buf()
@@ -578,16 +623,33 @@ describe('semantic token highlighting', function()
         expected = {
           {
             line = 0,
-            modifiers = {
-              'declaration',
-              'globalScope',
-            },
+            modifiers = { declaration = true, globalScope = true },
             start_col = 6,
             end_col = 9,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
         },
+        expected_screen = function()
+          screen:expect{grid=[[
+            char* {7:foo} = "\n"^;                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'clangd-15 on C++',
@@ -613,67 +675,67 @@ int main()
         expected = {
           { -- main
             line = 1,
-            modifiers = { 'declaration', 'globalScope' },
+            modifiers = { declaration = true, globalScope = true },
             start_col = 4,
             end_col = 8,
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           { --  __cplusplus
             line = 3,
-            modifiers = { 'globalScope' },
+            modifiers = { globalScope = true },
             start_col = 9,
             end_col = 20,
             type = 'macro',
-            extmark_added = true,
+            marked = true,
           },
           { -- x
             line = 4,
-            modifiers = { 'declaration', 'readonly', 'functionScope' },
+            modifiers = { declaration = true, readonly = true, functionScope = true },
             start_col = 12,
             end_col = 13,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           { -- std
             line = 5,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 2,
             end_col = 5,
             type = 'namespace',
-            extmark_added = true,
+            marked = true,
           },
           { -- cout
             line = 5,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 7,
             end_col = 11,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           { -- x
             line = 5,
-            modifiers = { 'readonly', 'functionScope' },
+            modifiers = { readonly = true, functionScope = true },
             start_col = 15,
             end_col = 16,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           { -- std
             line = 5,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 20,
             end_col = 23,
             type = 'namespace',
-            extmark_added = true,
+            marked = true,
           },
           { -- endl
             line = 5,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             start_col = 25,
             end_col = 29,
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           { -- #else comment #endif
             line = 6,
@@ -681,7 +743,7 @@ int main()
             start_col = 0,
             end_col = 7,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 7,
@@ -689,7 +751,7 @@ int main()
             start_col = 0,
             end_col = 11,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 8,
@@ -697,9 +759,29 @@ int main()
             start_col = 0,
             end_col = 8,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
         },
+        expected_screen = function()
+          screen:expect{grid=[[
+            #include <iostream>                     |
+            int {8:main}()                              |
+            {                                       |
+              #ifdef {5:__cplusplus}                    |
+              const int {7:x} = 1;                      |
+              {4:std}::{2:cout} << {2:x} << {4:std}::{3:endl};          |
+            {6:  #else}                                 |
+            {6:    comment}                             |
+            {6:  #endif}                                |
+            ^}                                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'sumneko_lua',
@@ -722,25 +804,45 @@ b = "as"]],
             start_col = 0,
             end_col = 10,
             type = 'comment', -- comment
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 1,
-            modifiers = { 'declaration' }, -- a
+            modifiers = { declaration = true }, -- a
             start_col = 6,
             end_col = 7,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 2,
-            modifiers = { 'static' }, -- b (global)
+            modifiers = { static = true }, -- b (global)
             start_col = 0,
             end_col = 1,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
         },
+        expected_screen = function()
+          screen:expect{grid=[[
+            {6:-- comment}                              |
+            local {7:a} = 1                             |
+            {2:b} = "as^"                                |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'rust-analyzer',
@@ -768,7 +870,7 @@ b = "as"]],
             start_col = 0,
             end_col = 3, -- pub
             type = 'keyword',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
@@ -776,15 +878,15 @@ b = "as"]],
             start_col = 4,
             end_col = 6, -- fn
             type = 'keyword',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
-            modifiers = { 'declaration', 'public' },
+            modifiers = { declaration = true, public = true },
             start_col = 7,
             end_col = 11, -- main
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
@@ -792,7 +894,7 @@ b = "as"]],
             start_col = 11,
             end_col = 12,
             type = 'parenthesis',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
@@ -800,7 +902,7 @@ b = "as"]],
             start_col = 12,
             end_col = 13,
             type = 'parenthesis',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 0,
@@ -808,15 +910,15 @@ b = "as"]],
             start_col = 14,
             end_col = 15,
             type = 'brace',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 1,
-            modifiers = { 'controlFlow' },
+            modifiers = { controlFlow = true },
             start_col = 4,
             end_col = 9, -- break
             type = 'keyword',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 1,
@@ -824,7 +926,7 @@ b = "as"]],
             start_col = 10,
             end_col = 13, -- rust
             type = 'unresolvedReference',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 1,
@@ -832,15 +934,15 @@ b = "as"]],
             start_col = 13,
             end_col = 13,
             type = 'semicolon',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 2,
-            modifiers = { 'documentation' },
+            modifiers = { documentation = true },
             start_col = 4,
             end_col = 11,
             type = 'comment', -- /// what?
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 3,
@@ -848,9 +950,29 @@ b = "as"]],
             start_col = 0,
             end_col = 1,
             type = 'brace',
-            extmark_added = true,
+            marked = true,
           },
         },
+        expected_screen = function()
+          screen:expect{grid=[[
+            pub fn {8:main}() {                         |
+              break rust;                           |
+              //{6:/ what?}                             |
+            }                                       |
+            ^                                        |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
     }) do
       it(test.it, function()
@@ -877,6 +999,8 @@ b = "as"]],
 
         insert(test.text)
 
+        test.expected_screen()
+
         local highlights = exec_lua([[
           local semantic_tokens = vim.lsp.semantic_tokens
           return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
@@ -906,28 +1030,68 @@ b = "as"]],
           {
             line = 0,
             modifiers = {
-              'declaration',
-              'globalScope',
+              declaration = true,
+              globalScope = true,
             },
             start_col = 6,
             end_col = 9,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           }
         },
         expected2 = {
           {
             line = 1,
             modifiers = {
-              'declaration',
-              'globalScope',
+              declaration = true,
+              globalScope = true,
             },
             start_col = 6,
             end_col = 9,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           }
         },
+        expected_screen1 = function()
+        screen:expect{grid=[[
+          char* {7:foo} = "\n"^;                       |
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+          {1:~                                       }|
+                                                  |
+        ]]}
+        end,
+        expected_screen2 = function()
+          screen:expect{grid=[[
+            ^                                        |
+            char* {7:foo} = "\n";                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'response with multiple delta edits',
@@ -976,55 +1140,55 @@ int main()
             line = 2,
             start_col = 4,
             end_col = 8,
-            modifiers = { 'declaration', 'globalScope' },
+            modifiers = { declaration = true, globalScope = true },
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 4,
             start_col = 8,
             end_col = 9,
-            modifiers = { 'declaration', 'functionScope' },
+            modifiers = { declaration = true, functionScope = true },
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 5,
             start_col = 7,
             end_col = 18,
-            modifiers = { 'globalScope' },
+            modifiers = { globalScope = true },
             type = 'macro',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 6,
             start_col = 4,
             end_col = 7,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             type = 'namespace',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 6,
             start_col = 9,
             end_col = 13,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 6,
             start_col = 17,
             end_col = 18,
-            extmark_added = true,
-            modifiers = { 'functionScope' },
+            marked = true,
+            modifiers = { functionScope = true },
             type = 'variable',
           },
           {
             line = 7,
             start_col = 0,
             end_col = 5,
-            extmark_added = true,
+            marked = true,
             modifiers = {},
             type = 'comment',
           },
@@ -1034,7 +1198,7 @@ int main()
             modifiers = {},
             start_col = 0,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 9,
@@ -1042,7 +1206,7 @@ int main()
             end_col = 6,
             modifiers = {},
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           }
         },
         expected2 = {
@@ -1050,63 +1214,63 @@ int main()
             line = 2,
             start_col = 4,
             end_col = 8,
-            modifiers = { 'declaration', 'globalScope' },
+            modifiers = { declaration = true, globalScope = true },
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 4,
             start_col = 8,
             end_col = 9,
-            modifiers = { 'declaration', 'globalScope' },
+            modifiers = { declaration = true, globalScope = true },
             type = 'function',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 5,
             end_col = 12,
             start_col = 11,
-            modifiers = { 'declaration', 'functionScope' },
+            modifiers = { declaration = true, functionScope = true },
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 6,
             start_col = 7,
             end_col = 18,
-            modifiers = { 'globalScope' },
+            modifiers = { globalScope = true },
             type = 'macro',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 7,
             start_col = 4,
             end_col = 7,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             type = 'namespace',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 7,
             start_col = 9,
             end_col = 13,
-            modifiers = { 'defaultLibrary', 'globalScope' },
+            modifiers = { defaultLibrary = true, globalScope = true },
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 7,
             start_col = 17,
             end_col = 18,
-            extmark_added = true,
-            modifiers = { 'globalScope' },
+            marked = true,
+            modifiers = { globalScope = true },
             type = 'function',
           },
           {
             line = 8,
             start_col = 0,
             end_col = 5,
-            extmark_added = true,
+            marked = true,
             modifiers = {},
             type = 'comment',
           },
@@ -1116,7 +1280,7 @@ int main()
             modifiers = {},
             start_col = 0,
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           },
           {
             line = 10,
@@ -1124,9 +1288,49 @@ int main()
             end_col = 6,
             modifiers = {},
             type = 'comment',
-            extmark_added = true,
+            marked = true,
           }
         },
+        expected_screen1 = function()
+          screen:expect{grid=[[
+            #include <iostream>                     |
+                                                    |
+            int {8:main}()                              |
+            {                                       |
+                int {7:x};                              |
+            #ifdef {5:__cplusplus}                      |
+                {4:std}::{2:cout} << {2:x} << "\n";             |
+            {6:#else}                                   |
+            {6:    printf("%d\n", x);}                  |
+            {6:#endif}                                  |
+            ^}                                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
+        expected_screen2 = function()
+          screen:expect{grid=[[
+            #include <iostream>                     |
+                                                    |
+            int {8:main}()                              |
+            {                                       |
+                int {8:x}();                            |
+                double {7:y};                           |
+            #ifdef {5:__cplusplus}                      |
+                {4:std}::{2:cout} << {3:x} << "\n";             |
+            {6:#else}                                   |
+            {6:    printf("%d\n", x);}                  |
+            {6:^#endif}                                  |
+            }                                       |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
       {
         it = 'optional token_edit.data on deletion',
@@ -1146,16 +1350,56 @@ int main()
           {
             line = 0,
             modifiers = {
-              'declaration',
+              declaration = true,
             },
             start_col = 0,
             end_col = 6,
             type = 'variable',
-            extmark_added = true,
+            marked = true,
           }
         },
         expected2 = {
         },
+        expected_screen1 = function()
+          screen:expect{grid=[[
+            {7:string} = "test^"                         |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
+        expected_screen2 = function()
+          screen:expect{grid=[[
+            ^                                        |
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+            {1:~                                       }|
+                                                    |
+          ]]}
+        end,
       },
     }) do
       it(test.it, function()
@@ -1192,6 +1436,8 @@ int main()
 
         insert(test.text1)
 
+        test.expected_screen1()
+
         local highlights = exec_lua([[
           return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
         ]])
@@ -1204,10 +1450,12 @@ int main()
           exec_lua([[
             local text = ...
             vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, vim.fn.split(text, "\n"))
-            vim.wait(15) -- wait fot debounce
+            vim.wait(15) -- wait for debounce
           ]], test.text2)
         end
 
+        test.expected_screen2()
+
         highlights = exec_lua([[
           return semantic_tokens.__STHighlighter.active[bufnr].client_state[client_id].current_result.highlights
         ]])