extmarks/bufhl: reimplement using new marktree data structure

Add new "splice" interface for tracking buffer changes at the byte
level. This will later be reused for byte-resolution buffer updates.
(Implementation has been started, but using undocumented "_on_bytes"
option now as interface hasn't been finalized).

Use this interface to improve many edge cases of extmark adjustment.
Changed tests indicate previously incorrect behavior. Adding tests for
more edge cases will be follow-up work (overlaps on_bytes tests)

Don't consider creation/deletion of marks an undoable event by itself.
This behavior was never documented, and imposes  complexity for little gain.

Add nvim__buf_add_decoration temporary API for direct access to the new
implementation. This should be refactored into a proper API for
decorations, probably involving a huge dict.

fixes #11598

1 files changed, 174 insertions, 24 deletions

diff --git a/test/functional/ui/bufhl_spec.lua b/test/functional/ui/bufhl_spec.lua
index f589bb0e83..3cb592c714 100644
--- a/test/functional/ui/bufhl_spec.lua
+++ b/test/functional/ui/bufhl_spec.lua
@@ -5,6 +5,7 @@ local clear, feed, insert = helpers.clear, helpers.feed, helpers.insert
 local command, neq = helpers.command, helpers.neq
 local meths = helpers.meths
 local curbufmeths, eq = helpers.curbufmeths, helpers.eq
+local pcall_err = helpers.pcall_err
 
 describe('Buffer highlighting', function()
   local screen
@@ -34,6 +35,7 @@ describe('Buffer highlighting', function()
       [17] = {foreground = Screen.colors.Magenta, background = Screen.colors.LightRed},
       [18] = {background = Screen.colors.LightRed},
       [19] = {foreground = Screen.colors.Blue1, background = Screen.colors.LightRed},
+      [20] = {underline = true, bold = true, foreground = Screen.colors.Cyan4},
     })
   end)
 
@@ -205,17 +207,116 @@ describe('Buffer highlighting', function()
                                                 |
       ]])
 
-      command(':3move 4')
-      screen:expect([[
+      -- TODO(bfedl): this behaves a bit weirdly due to the highlight on
+      -- the deleted line wrapping around. we should invalidate
+      -- highlights when they are completely inside deleted text
+      command('3move 4')
+      screen:expect{grid=[[
         a {5:longer} example                        |
                                                 |
+        {8:from different sources}                  |
+        {8:^in }{20:order}{8: to demonstrate}                 |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]]}
+      --screen:expect([[
+      --  a {5:longer} example                        |
+      --                                          |
+      --  {9:from }{8:diff}{7:erent} sources                  |
+      --  ^in {6:order} to {7:de}{5:monstr}{7:ate}                 |
+      --  {1:~                                       }|
+      --  {1:~                                       }|
+      --  {1:~                                       }|
+      --                                          |
+      --]])
+
+      command('undo')
+      screen:expect{grid=[[
+        a {5:longer} example                        |
+        ^                                        |
+        in {6:order} to {7:de}{5:monstr}{7:ate}                 |
         {9:from }{8:diff}{7:erent} sources                  |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+        1 change; before #4  {MATCH:.*}|
+      ]]}
+
+      command('undo')
+      screen:expect{grid=[[
+        ^a {5:longer} example                        |
+        in {6:order} to {7:de}{5:monstr}{7:ate}                 |
+        {9:from }{8:diff}{7:erent} sources                  |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+        1 line less; before #3  {MATCH:.*}|
+      ]]}
+
+      command('undo')
+      screen:expect{grid=[[
+        a {5:longer} example                        |
+        in {6:order} to {7:de}{5:monstr}{7:ate}                 |
+        {7:^combin}{8:ing}{9: hi}ghlights                    |
+        {9:from }{8:diff}{7:erent} sources                  |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+        1 more line; before #2  {MATCH:.*}|
+      ]]}
+    end)
+
+    it('and moving lines around', function()
+      command('2move 3')
+      screen:expect{grid=[[
+        a {5:longer} example                        |
+        {7:combin}{8:ing}{9: hi}ghlights                    |
         ^in {6:order} to {7:de}{5:monstr}{7:ate}                 |
+        {9:from }{8:diff}{7:erent} sources                  |
         {1:~                                       }|
         {1:~                                       }|
         {1:~                                       }|
                                                 |
-      ]])
+      ]]}
+
+      command('1,2move 4')
+      screen:expect{grid=[[
+        in {6:order} to {7:de}{5:monstr}{7:ate}                 |
+        {9:from }{8:diff}{7:erent} sources                  |
+        a {5:longer} example                        |
+        {7:^combin}{8:ing}{9: hi}ghlights                    |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+                                                |
+      ]]}
+
+      command('undo')
+      screen:expect{grid=[[
+        a {5:longer} example                        |
+        {7:combin}{8:ing}{9: hi}ghlights                    |
+        ^in {6:order} to {7:de}{5:monstr}{7:ate}                 |
+        {9:from }{8:diff}{7:erent} sources                  |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+        2 change3; before #3  {MATCH:.*}|
+      ]]}
+
+      command('undo')
+      screen:expect{grid=[[
+        a {5:longer} example                        |
+        ^in {6:order} to {7:de}{5:monstr}{7:ate}                 |
+        {7:combin}{8:ing}{9: hi}ghlights                    |
+        {9:from }{8:diff}{7:erent} sources                  |
+        {1:~                                       }|
+        {1:~                                       }|
+        {1:~                                       }|
+        1 change; before #2  {MATCH:.*}|
+      ]]}
     end)
 
     it('and adjusting columns', function()
@@ -272,7 +373,7 @@ describe('Buffer highlighting', function()
       feed('u')
       screen:expect{grid=[[
         a {5:longer} example                        |
-        in {6:ordAAAAr} to^ demonstrate              |
+        in {6:ordAAAAr} to^ {7:de}{5:monstr}{7:ate}              |
         {7:combin}{8:ing}{9: hi}ghlights                    |
         {9:from }{8:diff}{7:erent} sources                  |
         {1:~                                       }|
@@ -284,7 +385,7 @@ describe('Buffer highlighting', function()
       feed('u')
       screen:expect{grid=[[
         a {5:longer} example                        |
-        in {6:ord^er} to demonstrate                 |
+        in {6:ord^er} to {7:de}{5:monstr}{7:ate}                 |
         {7:combin}{8:ing}{9: hi}ghlights                    |
         {9:from }{8:diff}{7:erent} sources                  |
         {1:~                                       }|
@@ -292,14 +393,14 @@ describe('Buffer highlighting', function()
         {1:~                                       }|
         1 change; before #3  {MATCH:.*}|
       ]]}
-      end)
+    end)
 
     it('and joining lines', function()
       feed('ggJJJ')
       screen:expect{grid=[[
         a {5:longer} example in {6:order} to {7:de}{5:monstr}{7:ate}|
-        {7: combin}{8:ing hi}{7:ghlights^ }{8:from diff}{7:erent sou}|
-        {7:rces}                                    |
+         {7:combin}{8:ing}{9: hi}ghlights^ {9:from }{8:diff}{7:erent} sou|
+        rces                                    |
         {1:~                                       }|
         {1:~                                       }|
         {1:~                                       }|
@@ -307,13 +408,12 @@ describe('Buffer highlighting', function()
                                                 |
       ]]}
 
-      -- TODO(bfredl): perhaps better undo
       feed('uuu')
       screen:expect{grid=[[
-        ^a longer example                        |
-        in order to demonstrate                 |
-        combining highlights                    |
-        from different sources                  |
+        ^a {5:longer} example                        |
+        in {6:order} to {7:de}{5:monstr}{7:ate}                 |
+        {7:combin}{8:ing}{9: hi}ghlights                    |
+        {9:from }{8:diff}{7:erent} sources                  |
         {1:~                                       }|
         {1:~                                       }|
         {1:~                                       }|
@@ -334,25 +434,23 @@ describe('Buffer highlighting', function()
         {7:-- INSERT --}                            |
       ]]}
 
-      -- TODO(bfredl): keep both "parts" after split, requires proper extmark ranges
       feed('<esc>tsi<cr>')
       screen:expect{grid=[[
         a {5:longer} example                        |
         in {6:order}                                |
          to {7:de}{5:mo}                                |
-        ^nstrate                                 |
+        {5:^nstr}{7:ate}                                 |
         {7:combin}{8:ing}{9: hi}ghlights                    |
         {9:from }{8:diff}{7:erent} sources                  |
         {1:~                                       }|
         {7:-- INSERT --}                            |
       ]]}
 
-      -- TODO(bfredl): perhaps better undo
       feed('<esc>u')
       screen:expect{grid=[[
         a {5:longer} example                        |
         in {6:order}                                |
-         to demo{7:^nstrat}{8:e}                         |
+         to {7:de}{5:mo^nstr}{7:ate}                         |
         {7:combin}{8:ing}{9: hi}ghlights                    |
         {9:from }{8:diff}{7:erent} sources                  |
         {1:~                                       }|
@@ -363,7 +461,7 @@ describe('Buffer highlighting', function()
       feed('<esc>u')
       screen:expect{grid=[[
         a {5:longer} example                        |
-        in order^ to demonstrate                 |
+        in {6:order}^ to {7:de}{5:monstr}{7:ate}                 |
         {7:combin}{8:ing}{9: hi}ghlights                    |
         {9:from }{8:diff}{7:erent} sources                  |
         {1:~                                       }|
@@ -374,7 +472,7 @@ describe('Buffer highlighting', function()
     end)
   end)
 
-  it('prioritizes latest added highlight', function()
+  pending('prioritizes latest added highlight', function()
     insert([[
       three overlapping colors]])
     add_highlight(0, "Identifier", 0, 6, 17)
@@ -405,6 +503,37 @@ describe('Buffer highlighting', function()
     ]])
   end)
 
+  it('prioritizes earlier highlight groups (TEMP)', function()
+    insert([[
+      three overlapping colors]])
+    add_highlight(0, "Identifier", 0, 6, 17)
+    add_highlight(0, "String", 0, 14, 23)
+    local id = add_highlight(0, "Special", 0, 0, 9)
+
+    screen:expect{grid=[[
+      {4:three }{6:overlapp}{2:ing color}^s                |
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+                                              |
+    ]]}
+
+    clear_namespace(id, 0, 1)
+    screen:expect{grid=[[
+      three {6:overlapp}{2:ing color}^s                |
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+                                              |
+    ]]}
+  end)
+
   it('works with multibyte text', function()
     insert([[
       Ta båten över sjön!]])
@@ -451,7 +580,7 @@ describe('Buffer highlighting', function()
     ]])
   end)
 
-  describe('virtual text annotations', function()
+  describe('virtual text decorations', function()
     local set_virtual_text = curbufmeths.set_virtual_text
     local id1, id2
     before_each(function()
@@ -529,16 +658,35 @@ describe('Buffer highlighting', function()
       ]])
 
       feed("2Gdd")
-      screen:expect([[
+      -- TODO(bfredl): currently decorations get moved from a deleted line
+      -- to the next one. We might want to add "invalidation" when deleting
+      -- over a decoration.
+      screen:expect{grid=[[
         1 + 2                                   |
         ^5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5|
-        , 5, 5, 5, 5, 5, 5,  Lorem ipsum dolor s|
+        , 5, 5, 5, 5, 5, 5,  {12:暗x事zz速野谷質結育}|
         x = 4                                   |
         {1:~                                       }|
         {1:~                                       }|
         {1:~                                       }|
                                                 |
-      ]])
+      ]]}
+      --screen:expect([[
+      --  1 + 2                                   |
+      --  ^5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5|
+      --  , 5, 5, 5, 5, 5, 5,  Lorem ipsum dolor s|
+      --  x = 4                                   |
+      --  {1:~                                       }|
+      --  {1:~                                       }|
+      --  {1:~                                       }|
+      --                                          |
+      --]])
+    end)
+
+    it('validates contents', function()
+      -- this used to leak memory
+      eq('Chunk is not an array', pcall_err(set_virtual_text, id1, 0, {"texty"}, {}))
+      eq('Chunk is not an array', pcall_err(set_virtual_text, id1, 0, {{"very"}, "texty"}, {}))
     end)
 
     it('can be retrieved', function()
@@ -548,7 +696,9 @@ describe('Buffer highlighting', function()
       local s1 = {{'Köttbullar', 'Comment'}, {'Kräuterbutter'}}
       local s2 = {{'こんにちは', 'Comment'}}
 
-      set_virtual_text(-1, 0, s1, {})
+      -- TODO: only a virtual text from the same ns curretly overrides
+      -- an existing virtual text. We might add a prioritation system.
+      set_virtual_text(id1, 0, s1, {})
       eq(s1, get_virtual_text(0))
 
       set_virtual_text(-1, line_count(), s2, {})