ui: use line-based rather than char-based updates in screen.c

Add ext_newgrid and ext_hlstate extensions. These use predefined
highlights and line-segment based updates, for efficiency and
simplicity.. The ext_hlstate extension in addition allows semantic
identification of builtin and syntax highlights.

Reimplement the old char-based updates in the remote UI layer, for
compatibility. For the moment, this is still the default. The bulitin
TUI uses the new line-based protocol.

cmdline uses curwin cursor position when ext_cmdline is active.

4 files changed, 107 insertions, 19 deletions

diff --git a/test/functional/ui/cmdline_spec.lua b/test/functional/ui/cmdline_spec.lua
index 5ce49822e5..b2fc008dba 100644
--- a/test/functional/ui/cmdline_spec.lua
+++ b/test/functional/ui/cmdline_spec.lua
@@ -29,6 +29,9 @@ describe('external cmdline', function()
       if name == "cmdline_show" then
         local content, pos, firstc, prompt, indent, level = unpack(data)
         ok(level > 0)
+        for _,item in ipairs(content) do
+          item[1] = screen:get_hl(item[1])
+        end
         cmdline[level] = {content=content, pos=pos, firstc=firstc,
                           prompt=prompt, indent=indent}
         last_level = level
@@ -87,6 +90,7 @@ describe('external cmdline', function()
                                |
     ]], nil, nil, function()
       eq(1, last_level)
+      --print(require('inspect')(cmdline))
       eq({{
         content = { { {}, "" } },
         firstc = ":",
@@ -168,10 +172,10 @@ describe('external cmdline', function()
     it('from normal mode', function()
       feed(':')
       screen:expect([[
-                                 |
+        ^                         |
         {1:~                        }|
         {1:~                        }|
-        {3:c^                        }|
+        {3:c                        }|
                                  |
       ]], nil, nil, function()
         eq({{
@@ -351,11 +355,11 @@ describe('external cmdline', function()
     -- redraw! forgets cursor position. Be OK with that, as UI should indicate
     -- focus is at external cmdline anyway.
     screen:expect([[
-                               |
+      ^                         |
       {1:~                        }|
       {1:~                        }|
       {1:~                        }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq(expectation, cmdline)
     end)
@@ -363,11 +367,11 @@ describe('external cmdline', function()
 
     feed('<cr>')
     screen:expect([[
-                               |
+      ^                         |
       {1:~                        }|
       {1:~                        }|
       {1:~                        }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq({{
         content = { { {}, "xx3" } },
@@ -424,11 +428,11 @@ describe('external cmdline', function()
     block = {}
     command("redraw!")
     screen:expect([[
-                               |
+      ^                         |
       {1:~                        }|
       {1:~                        }|
       {1:~                        }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq({ { { {}, 'function Foo()'} },
            { { {}, '  line1'} } }, block)
@@ -528,9 +532,9 @@ describe('external cmdline', function()
     screen:expect([[
                                |
       {2:[No Name]                }|
-      {1::}make                    |
+      {1::}make^                    |
       {3:[Command Line]           }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq({nil, {
         content = { { {}, "yank" } },
@@ -572,11 +576,11 @@ describe('external cmdline', function()
     cmdline = {}
     command("redraw!")
     screen:expect([[
-                               |
+      ^                         |
       {1:~                        }|
       {1:~                        }|
       {1:~                        }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq({{
         content = { { {}, "make" } },
diff --git a/test/functional/ui/options_spec.lua b/test/functional/ui/options_spec.lua
index 62b08c0967..322a94763f 100644
--- a/test/functional/ui/options_spec.lua
+++ b/test/functional/ui/options_spec.lua
@@ -30,10 +30,15 @@ describe('ui receives option updates', function()
     ext_popupmenu=false,
     ext_tabline=false,
     ext_wildmenu=false,
+    ext_newgrid=false,
+    ext_hlstate=false,
   }
 
   it("for defaults", function()
     screen:attach()
+    -- NB: UI test suite can be run in both "newgrid" and legacy grid mode.
+    -- In both cases check that the received value is the one requested.
+    defaults.ext_newgrid = screen._options.ext_newgrid or false
     screen:expect(function()
       eq(defaults, screen.options)
     end)
@@ -41,6 +46,7 @@ describe('ui receives option updates', function()
 
   it("when setting options", function()
     screen:attach()
+    defaults.ext_newgrid = screen._options.ext_newgrid or false
     local changed = {}
     for k,v in pairs(defaults) do
       changed[k] = v
@@ -89,6 +95,7 @@ describe('ui receives option updates', function()
     end
 
     screen:attach({ext_cmdline=true, ext_wildmenu=true})
+    defaults.ext_newgrid = screen._options.ext_newgrid or false
     changed.ext_cmdline = true
     changed.ext_wildmenu = true
     screen:expect(function()
diff --git a/test/functional/ui/screen.lua b/test/functional/ui/screen.lua
index 7607131e9b..d96aae1aa7 100644
--- a/test/functional/ui/screen.lua
+++ b/test/functional/ui/screen.lua
@@ -142,6 +142,8 @@ function Screen.new(width, height)
     _default_attr_ignore = nil,
     _mouse_enabled = true,
     _attrs = {},
+    _attr_table = {[0]={{},{}}},
+    _clear_attrs = {},
     _cursor = {
       row = 1, col = 1
     },
@@ -163,6 +165,11 @@ function Screen:attach(options)
   if options == nil then
     options = {rgb=true}
   end
+  if options.ext_newgrid == nil then
+    options.ext_newgrid = true
+  end
+  self._options = options
+  self._clear_attrs = (options.ext_newgrid and {{},{}}) or {}
   uimeths.attach(self._width, self._height, options)
 end
 
@@ -176,6 +183,7 @@ end
 
 function Screen:set_option(option, value)
   uimeths.set_option(option, value)
+  self._options[option] = value
 end
 
 -- Asserts that `expected` eventually matches the screen state.
@@ -339,7 +347,7 @@ function Screen:_handle_resize(width, height)
   for _ = 1, height do
     local cols = {}
     for _ = 1, width do
-      table.insert(cols, {text = ' ', attrs = {}})
+      table.insert(cols, {text = ' ', attrs = self._clear_attrs})
     end
     table.insert(rows, cols)
   end
@@ -353,14 +361,24 @@ function Screen:_handle_resize(width, height)
   }
 end
 
+function Screen:_handle_grid_resize(grid, width, height)
+  assert(grid == 1)
+  self:_handle_resize(width, height)
+end
+
+
 function Screen:_handle_mode_info_set(cursor_style_enabled, mode_info)
   self._cursor_style_enabled = cursor_style_enabled
   self._mode_info = mode_info
 end
 
 function Screen:_handle_clear()
-  self:_clear_block(self._scroll_region.top, self._scroll_region.bot,
-                    self._scroll_region.left, self._scroll_region.right)
+  self:_clear_block(1, self._height, 1, self._width)
+end
+
+function Screen:_handle_grid_clear(grid)
+  assert(grid == 1)
+  self:_handle_clear()
 end
 
 function Screen:_handle_eol_clear()
@@ -373,6 +391,12 @@ function Screen:_handle_cursor_goto(row, col)
   self._cursor.col = col + 1
 end
 
+function Screen:_handle_grid_cursor_goto(grid, row, col)
+  assert(grid == 1)
+  self._cursor.row = row + 1
+  self._cursor.col = col + 1
+end
+
 function Screen:_handle_busy_start()
   self._busy = true
 end
@@ -434,6 +458,27 @@ function Screen:_handle_scroll(count)
   end
 end
 
+function Screen:_handle_grid_scroll(grid, top, bot, left, right, rows, cols)
+  assert(grid == 1)
+  assert(cols == 0)
+  -- TODO: if we truly believe we should translate the other way
+  self:_handle_set_scroll_region(top,bot-1,left,right-1)
+  self:_handle_scroll(rows)
+end
+
+function Screen:_handle_hl_attr_define(id, rgb_attrs, cterm_attrs, info)
+  self._attr_table[id] = {rgb_attrs, cterm_attrs}
+  self._new_attrs = true
+end
+
+function Screen:get_hl(val)
+  if self._options.ext_newgrid then
+    return self._attr_table[val][1]
+  else
+    return val
+  end
+end
+
 function Screen:_handle_highlight_set(attrs)
   self._attrs = attrs
 end
@@ -445,6 +490,25 @@ function Screen:_handle_put(str)
   self._cursor.col = self._cursor.col + 1
 end
 
+function Screen:_handle_grid_line(grid, row, col, items)
+  assert(grid == 1)
+  local line = self._rows[row+1]
+  local colpos = col+1
+  local hl = self._clear_attrs
+  for _,item in ipairs(items) do
+    local text, hlid, count = unpack(item)
+    if hlid ~= nil then
+      hl = self._attr_table[hlid]
+    end
+    for _ = 1, (count or 1) do
+      local cell = line[colpos]
+      cell.text = text
+      cell.attrs = hl
+      colpos = colpos+1
+    end
+  end
+end
+
 function Screen:_handle_bell()
   self.bell = true
 end
@@ -498,7 +562,7 @@ function Screen:_clear_row_section(rownum, startcol, stopcol)
   local row = self._rows[rownum]
   for i = startcol, stopcol do
     row[i].text = ' '
-    row[i].attrs = {}
+    row[i].attrs = self._clear_attrs
   end
 end
 
@@ -506,7 +570,11 @@ function Screen:_row_repr(row, attr_ids, attr_ignore)
   local rv = {}
   local current_attr_id
   for i = 1, self._width do
-    local attr_id = self:_get_attr_id(attr_ids, attr_ignore, row[i].attrs)
+    local attrs = row[i].attrs
+    if self._options.ext_newgrid then
+      attrs = attrs[(self._options.rgb and 1) or 2]
+    end
+    local attr_id = self:_get_attr_id(attr_ids, attr_ignore, attrs, row[i].hl_id)
     if current_attr_id and attr_id ~= current_attr_id then
       -- close current attribute bracket, add it before any whitespace
       -- up to the current cell
@@ -647,6 +715,7 @@ function Screen:_get_attr_id(attr_ids, ignore, attrs)
   if not attr_ids then
     return
   end
+
   for id, a in pairs(attr_ids) do
     if self:_equal_attrs(a, attrs) then
        return id
diff --git a/test/functional/ui/screen_basic_spec.lua b/test/functional/ui/screen_basic_spec.lua
index 6f04cde4d4..75a2d4978d 100644
--- a/test/functional/ui/screen_basic_spec.lua
+++ b/test/functional/ui/screen_basic_spec.lua
@@ -48,13 +48,13 @@ describe('screen', function()
   end)
 end)
 
-describe('Screen', function()
+local function screen_tests(newgrid)
   local screen
 
   before_each(function()
     clear()
     screen = Screen.new()
-    screen:attach()
+    screen:attach({rgb=true,ext_newgrid=newgrid})
     screen:set_default_attr_ids( {
       [0] = {bold=true, foreground=255},
       [1] = {bold=true, reverse=true},
@@ -741,4 +741,12 @@ describe('Screen', function()
                                                            |
     ]])
   end)
+end
+
+describe("Screen (char-based)", function()
+  screen_tests(false)
+end)
+
+describe("Screen (line-based)", function()
+  screen_tests(true)
 end)