Merge pull request #8221 from bfredl/hlstate

UI grid protocol revision: line based updates and semantic highlights

9 files changed, 613 insertions, 75 deletions

diff --git a/test/functional/api/version_spec.lua b/test/functional/api/version_spec.lua
index 7bf54c0d1e..e0472977cc 100644
--- a/test/functional/api/version_spec.lua
+++ b/test/functional/api/version_spec.lua
@@ -156,6 +156,6 @@ describe("ui_options in metadata", function()
     local api = helpers.call('api_info')
     local options = api.ui_options
     eq({'rgb', 'ext_cmdline', 'ext_popupmenu',
-        'ext_tabline', 'ext_wildmenu'}, options)
+        'ext_tabline', 'ext_wildmenu', 'ext_newgrid', 'ext_hlstate'}, options)
   end)
 end)
diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua
index e4b343c123..ae8a8488d4 100644
--- a/test/functional/api/vim_spec.lua
+++ b/test/functional/api/vim_spec.lua
@@ -1242,6 +1242,8 @@ describe('API', function()
           ext_popupmenu = false,
           ext_tabline = false,
           ext_wildmenu = false,
+          ext_newgrid = screen._options.ext_newgrid or false,
+          ext_hlstate=false,
           height = 4,
           rgb = true,
           width = 20,
@@ -1252,18 +1254,9 @@ describe('API', function()
       screen:detach()
       screen = Screen.new(44, 99)
       screen:attach({ rgb = false })
-      expected = {
-        {
-          chan = 1,
-          ext_cmdline = false,
-          ext_popupmenu = false,
-          ext_tabline = false,
-          ext_wildmenu = false,
-          height = 99,
-          rgb = false,
-          width = 44,
-        }
-      }
+      expected[1].rgb = false
+      expected[1].width = 44
+      expected[1].height = 99
       eq(expected, nvim("list_uis"))
     end)
   end)
diff --git a/test/functional/terminal/tui_spec.lua b/test/functional/terminal/tui_spec.lua
index 5603224975..c0404ff463 100644
--- a/test/functional/terminal/tui_spec.lua
+++ b/test/functional/terminal/tui_spec.lua
@@ -207,7 +207,7 @@ describe('tui', function()
     screen:set_default_attr_ids({
       [1] = {reverse = true},
       [2] = {foreground = 13, special = Screen.colors.Grey0},
-      [3] = {special = Screen.colors.Grey0, bold = true, reverse = true},
+      [3] = {bold = true, reverse = true, special = Screen.colors.Grey0},
       [4] = {bold = true},
       [5] = {special = Screen.colors.Grey0, reverse = true, foreground = 4},
       [6] = {foreground = 4, special = Screen.colors.Grey0},
@@ -257,11 +257,11 @@ describe('tui', function()
   it('shows up in nvim_list_uis', function()
     feed_data(':echo map(nvim_list_uis(), {k,v -> sort(items(v))})\013')
     screen:expect([=[
-      {5:                                                  }|
-      [[['ext_cmdline', v:false], ['ext_popupmenu', v:fa|
-      lse], ['ext_tabline', v:false], ['ext_wildmenu', v|
-      :false], ['height', 6], ['rgb', v:false], ['width'|
-      , 50]]]                                           |
+      [[['ext_cmdline', v:false], ['ext_hlstate', v:fals|
+      e], ['ext_newgrid', v:true], ['ext_popupmenu', v:f|
+      alse], ['ext_tabline', v:false], ['ext_wildmenu', |
+      v:false], ['height', 6], ['rgb', v:false], ['width|
+      ', 50]]]                                          |
       {10:Press ENTER or type command to continue}{1: }          |
       {3:-- TERMINAL --}                                    |
     ]=])
diff --git a/test/functional/ui/cmdline_spec.lua b/test/functional/ui/cmdline_spec.lua
index 5ce49822e5..b2fc008dba 100644
--- a/test/functional/ui/cmdline_spec.lua
+++ b/test/functional/ui/cmdline_spec.lua
@@ -29,6 +29,9 @@ describe('external cmdline', function()
       if name == "cmdline_show" then
         local content, pos, firstc, prompt, indent, level = unpack(data)
         ok(level > 0)
+        for _,item in ipairs(content) do
+          item[1] = screen:get_hl(item[1])
+        end
         cmdline[level] = {content=content, pos=pos, firstc=firstc,
                           prompt=prompt, indent=indent}
         last_level = level
@@ -87,6 +90,7 @@ describe('external cmdline', function()
                                |
     ]], nil, nil, function()
       eq(1, last_level)
+      --print(require('inspect')(cmdline))
       eq({{
         content = { { {}, "" } },
         firstc = ":",
@@ -168,10 +172,10 @@ describe('external cmdline', function()
     it('from normal mode', function()
       feed(':')
       screen:expect([[
-                                 |
+        ^                         |
         {1:~                        }|
         {1:~                        }|
-        {3:c^                        }|
+        {3:c                        }|
                                  |
       ]], nil, nil, function()
         eq({{
@@ -351,11 +355,11 @@ describe('external cmdline', function()
     -- redraw! forgets cursor position. Be OK with that, as UI should indicate
     -- focus is at external cmdline anyway.
     screen:expect([[
-                               |
+      ^                         |
       {1:~                        }|
       {1:~                        }|
       {1:~                        }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq(expectation, cmdline)
     end)
@@ -363,11 +367,11 @@ describe('external cmdline', function()
 
     feed('<cr>')
     screen:expect([[
-                               |
+      ^                         |
       {1:~                        }|
       {1:~                        }|
       {1:~                        }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq({{
         content = { { {}, "xx3" } },
@@ -424,11 +428,11 @@ describe('external cmdline', function()
     block = {}
     command("redraw!")
     screen:expect([[
-                               |
+      ^                         |
       {1:~                        }|
       {1:~                        }|
       {1:~                        }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq({ { { {}, 'function Foo()'} },
            { { {}, '  line1'} } }, block)
@@ -528,9 +532,9 @@ describe('external cmdline', function()
     screen:expect([[
                                |
       {2:[No Name]                }|
-      {1::}make                    |
+      {1::}make^                    |
       {3:[Command Line]           }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq({nil, {
         content = { { {}, "yank" } },
@@ -572,11 +576,11 @@ describe('external cmdline', function()
     cmdline = {}
     command("redraw!")
     screen:expect([[
-                               |
+      ^                         |
       {1:~                        }|
       {1:~                        }|
       {1:~                        }|
-      ^                         |
+                               |
     ]], nil, nil, function()
       eq({{
         content = { { {}, "make" } },
diff --git a/test/functional/ui/highlight_spec.lua b/test/functional/ui/highlight_spec.lua
index b46a6c1e46..4b6fbc0d74 100644
--- a/test/functional/ui/highlight_spec.lua
+++ b/test/functional/ui/highlight_spec.lua
@@ -834,7 +834,7 @@ describe("'winhighlight' highlight", function()
       {1:a^a                  }|
       {2:~                   }|
       {2:~                   }|
-      {11:[No Name] [+]       }|
+      {3:[No Name] [+]       }|
       aa                  |
       {0:~                   }|
       {4:[No Name] [+]       }|
@@ -846,7 +846,7 @@ describe("'winhighlight' highlight", function()
       {1:^                    }|
       {2:~                   }|
       {2:~                   }|
-      {11:[No Name]           }|
+      {3:[No Name]           }|
       aa                  |
       {0:~                   }|
       {4:[No Name] [+]       }|
@@ -891,7 +891,7 @@ describe("'winhighlight' highlight", function()
       {1:a^a                  }|
       {2:~                   }|
       {2:~                   }|
-      {11:[No Name] [+]       }|
+      {3:[No Name] [+]       }|
       aa                  |
       {0:~                   }|
       {4:[No Name] [+]       }|
@@ -915,7 +915,7 @@ describe("'winhighlight' highlight", function()
       {1:^aa                  }|
       {2:~                   }|
       {2:~                   }|
-      {11:[No Name] [+]       }|
+      {3:[No Name] [+]       }|
       aa                  |
       {0:~                   }|
       {4:[No Name] [+]       }|
@@ -931,10 +931,10 @@ describe("'winhighlight' highlight", function()
       {1:^                    }|
       {2:~                   }|
       {2:~                   }|
-      {11:[No Name]           }|
+      {3:[No Name]           }|
       {5:                    }|
       {6:~                   }|
-      {12:[No Name]           }|
+      {4:[No Name]           }|
                           |
     ]])
 
@@ -943,10 +943,10 @@ describe("'winhighlight' highlight", function()
       {5:                    }|
       {6:~                   }|
       {6:~                   }|
-      {12:[No Name]           }|
+      {4:[No Name]           }|
       {1:^                    }|
       {2:~                   }|
-      {11:[No Name]           }|
+      {3:[No Name]           }|
                           |
     ]])
 
@@ -955,10 +955,10 @@ describe("'winhighlight' highlight", function()
       {1:^                    }|
       {2:~                   }|
       {2:~                   }|
-      {11:[No Name]           }|
+      {3:[No Name]           }|
       {5:                    }|
       {6:~                   }|
-      {12:[No Name]           }|
+      {4:[No Name]           }|
                           |
     ]])
   end)
@@ -974,7 +974,7 @@ describe("'winhighlight' highlight", function()
       {3:[No Name]           }|
       {7:                    }|
       {8:~                   }|
-      {13:[No Name]           }|
+      {4:[No Name]           }|
                           |
     ]])
 
@@ -983,7 +983,7 @@ describe("'winhighlight' highlight", function()
       {7:                    }|
       {8:~                   }|
       {8:~                   }|
-      {13:[No Name]           }|
+      {4:[No Name]           }|
       ^                    |
       {0:~                   }|
       {3:[No Name]           }|
@@ -997,10 +997,10 @@ describe("'winhighlight' highlight", function()
       {7:                    }|
       {8:~                   }|
       {8:~                   }|
-      {13:[No Name]           }|
+      {4:[No Name]           }|
       {1:^                    }|
       {2:~                   }|
-      {11:[No Name]           }|
+      {3:[No Name]           }|
                           |
     ]])
 
@@ -1012,7 +1012,7 @@ describe("'winhighlight' highlight", function()
       {3:[No Name]           }|
       {1:                    }|
       {2:~                   }|
-      {14:[No Name]           }|
+      {4:[No Name]           }|
                           |
     ]])
 
@@ -1022,10 +1022,10 @@ describe("'winhighlight' highlight", function()
       {7:                    }|
       {8:~                   }|
       {8:~                   }|
-      {13:[No Name]           }|
+      {4:[No Name]           }|
       {1:^                    }|
       {2:~                   }|
-      {11:[No Name]           }|
+      {3:[No Name]           }|
                           |
     ]])
 
@@ -1037,7 +1037,7 @@ describe("'winhighlight' highlight", function()
       {3:[No Name]           }|
       {5:                    }|
       {6:~                   }|
-      {12:[No Name]           }|
+      {4:[No Name]           }|
                           |
     ]])
   end)
diff --git a/test/functional/ui/hlstate_spec.lua b/test/functional/ui/hlstate_spec.lua
new file mode 100644
index 0000000000..672af5fb22
--- /dev/null
+++ b/test/functional/ui/hlstate_spec.lua
@@ -0,0 +1,287 @@
+local helpers = require('test.functional.helpers')(after_each)
+local Screen = require('test.functional.ui.screen')
+
+local clear, insert = helpers.clear, helpers.insert
+local command = helpers.command
+local meths = helpers.meths
+local iswin = helpers.iswin
+local nvim_dir = helpers.nvim_dir
+local thelpers = require('test.functional.terminal.helpers')
+
+describe('ext_hlstate detailed highlights', function()
+  local screen
+
+  before_each(function()
+    clear()
+    command('syntax on')
+    screen = Screen.new(40, 8)
+    screen:attach({ext_hlstate=true})
+  end)
+
+  after_each(function()
+    screen:detach()
+  end)
+
+
+  it('work with combined UI and syntax highlights', function()
+    insert([[
+      these are some lines
+      with colorful text]])
+    meths.buf_add_highlight(0, -1, "String", 0 , 10, 14)
+    meths.buf_add_highlight(0, -1, "Statement", 1 , 5, -1)
+    command("/th co")
+
+    screen:expect([[
+      these are {1:some} lines                    |
+      ^wi{2:th }{4:co}{3:lorful text}                      |
+      {5:~                                       }|
+      {5:~                                       }|
+      {5:~                                       }|
+      {5:~                                       }|
+      {5:~                                       }|
+      {6:search hit BOTTOM, continuing at TOP}    |
+    ]], {
+    [1] = {{foreground = Screen.colors.Magenta},
+           {{hi_name = "Constant", kind = "syntax"}}},
+    [2] = {{background = Screen.colors.Yellow},
+           {{hi_name = "Search", ui_name = "Search", kind = "ui"}}},
+    [3] = {{bold = true, foreground = Screen.colors.Brown},
+           {{hi_name = "Statement", kind = "syntax"}}},
+    [4] = {{bold = true, background = Screen.colors.Yellow, foreground = Screen.colors.Brown}, {3, 2}},
+    [5] = {{bold = true, foreground = Screen.colors.Blue1},
+           {{hi_name = "NonText", ui_name = "EndOfBuffer", kind = "ui"}}},
+    [6] = {{foreground = Screen.colors.Red},
+           {{hi_name = "WarningMsg", ui_name = "WarningMsg", kind = "ui"}}},
+    })
+  end)
+
+  it('work with cleared UI highlights', function()
+    screen:set_default_attr_ids({
+      [1] = {{}, {{hi_name = "VertSplit", ui_name = "VertSplit", kind = "ui"}}},
+      [2] = {{bold = true, foreground = Screen.colors.Blue1},
+             {{hi_name = "NonText", ui_name = "EndOfBuffer", kind = "ui"}}},
+      [3] = {{bold = true, reverse = true},
+            {{hi_name = "StatusLine", ui_name = "StatusLine", kind = "ui"}}} ,
+      [4] = {{reverse = true},
+            {{hi_name = "StatusLineNC", ui_name = "StatusLineNC" , kind = "ui"}}},
+      [5] = {{}, {{hi_name = "StatusLine", ui_name = "StatusLine", kind = "ui"}}},
+      [6] = {{}, {{hi_name = "StatusLineNC", ui_name = "StatusLineNC", kind = "ui"}}},
+    })
+    command("hi clear VertSplit")
+    command("vsplit")
+
+    screen:expect([[
+      ^                    {1:│}                   |
+      {2:~                   }{1:│}{2:~                  }|
+      {2:~                   }{1:│}{2:~                  }|
+      {2:~                   }{1:│}{2:~                  }|
+      {2:~                   }{1:│}{2:~                  }|
+      {2:~                   }{1:│}{2:~                  }|
+      {3:[No Name]            }{4:[No Name]          }|
+                                              |
+    ]])
+
+    command("hi clear StatusLine | hi clear StatuslineNC")
+    screen:expect([[
+      ^                    {1:│}                   |
+      {2:~                   }{1:│}{2:~                  }|
+      {2:~                   }{1:│}{2:~                  }|
+      {2:~                   }{1:│}{2:~                  }|
+      {2:~                   }{1:│}{2:~                  }|
+      {2:~                   }{1:│}{2:~                  }|
+      {5:[No Name]            }{6:[No Name]          }|
+                                              |
+    ]])
+
+    -- redrawing is done even if visible highlights didn't change
+    command("wincmd w")
+    screen:expect([[
+                         {1:│}^                    |
+      {2:~                  }{1:│}{2:~                   }|
+      {2:~                  }{1:│}{2:~                   }|
+      {2:~                  }{1:│}{2:~                   }|
+      {2:~                  }{1:│}{2:~                   }|
+      {2:~                  }{1:│}{2:~                   }|
+      {6:[No Name]           }{5:[No Name]           }|
+                                              |
+    ]])
+
+  end)
+
+  it("work with window-local highlights", function()
+    screen:set_default_attr_ids({
+        [1] = {{foreground = Screen.colors.Brown}, {{hi_name = "LineNr", ui_name = "LineNr", kind = "ui"}}},
+        [2] = {{bold = true, foreground = Screen.colors.Blue1}, {{hi_name = "NonText", ui_name = "EndOfBuffer", kind = "ui"}}},
+        [3] = {{bold = true, reverse = true}, {{hi_name = "StatusLine", ui_name = "StatusLine", kind = "ui"}}},
+        [4] = {{reverse = true}, {{hi_name = "StatusLineNC", ui_name = "StatusLineNC", kind = "ui"}}},
+        [5] = {{background = Screen.colors.Red, foreground = Screen.colors.Grey100}, {{hi_name = "ErrorMsg", ui_name = "LineNr", kind = "ui"}}},
+        [6] = {{bold = true, reverse = true}, {{hi_name = "MsgSeparator", ui_name = "Normal", kind = "ui"}}},
+        [7] = {{foreground = Screen.colors.Brown, bold = true, reverse = true}, {6, 1}},
+        [8] = {{foreground = Screen.colors.Blue1, bold = true, reverse = true}, {6, 2}},
+        [9] = {{bold = true, foreground = Screen.colors.Brown}, {{hi_name = "Statement", ui_name = "NormalNC", kind = "ui"}}},
+        [10] = {{bold = true, foreground = Screen.colors.Brown}, {9, 1}},
+        [11] = {{bold = true, foreground = Screen.colors.Blue1}, {9, 2}}
+    })
+
+    command("set number")
+    command("split")
+    -- NormalNC is not applied if not set, to avoid spurious redraws
+    screen:expect([[
+      {1:  1 }^                                    |
+      {2:~                                       }|
+      {2:~                                       }|
+      {3:[No Name]                               }|
+      {1:  1 }                                    |
+      {2:~                                       }|
+      {4:[No Name]                               }|
+                                              |
+    ]])
+
+    command("set winhl=LineNr:ErrorMsg")
+    screen:expect([[
+      {5:  1 }^                                    |
+      {2:~                                       }|
+      {2:~                                       }|
+      {3:[No Name]                               }|
+      {1:  1 }                                    |
+      {2:~                                       }|
+      {4:[No Name]                               }|
+                                              |
+    ]])
+
+    command("set winhl=Normal:MsgSeparator,NormalNC:Statement")
+    screen:expect([[
+      {7:  1 }{6:^                                    }|
+      {8:~                                       }|
+      {8:~                                       }|
+      {3:[No Name]                               }|
+      {1:  1 }                                    |
+      {2:~                                       }|
+      {4:[No Name]                               }|
+                                              |
+    ]])
+
+    command("wincmd w")
+    screen:expect([[
+      {10:  1 }{9:                                    }|
+      {11:~                                       }|
+      {11:~                                       }|
+      {4:[No Name]                               }|
+      {1:  1 }^                                    |
+      {2:~                                       }|
+      {3:[No Name]                               }|
+                                              |
+    ]])
+  end)
+
+  it("work with :terminal", function()
+    screen:set_default_attr_ids({
+      [1] = {{}, {{hi_name = "TermCursorNC", ui_name = "TermCursorNC", kind = "ui"}}},
+      [2] = {{special = Screen.colors.Grey0, foreground = 52479}, {{kind = "term"}}},
+      [3] = {{special = Screen.colors.Grey0, bold = true, foreground = 52479}, {{kind = "term"}}},
+      [4] = {{special = Screen.colors.Grey0, foreground = 52479}, {2, 1}},
+      [5] = {{special = Screen.colors.Grey0, foreground = 4259839}, {{kind = "term"}}},
+      [6] = {{special = Screen.colors.Grey0, foreground = 4259839}, {5, 1}},
+    })
+    command('enew | call termopen(["'..nvim_dir..'/tty-test"])')
+    screen:expect([[
+      ^tty ready                               |
+      {1: }                                       |
+                                              |
+                                              |
+                                              |
+                                              |
+                                              |
+                                              |
+    ]])
+
+    thelpers.feed_data('x ')
+    thelpers.set_fg(45)
+    thelpers.feed_data('y ')
+    thelpers.set_bold()
+    thelpers.feed_data('z\n')
+    -- TODO(bfredl): check if this distinction makes sense
+    if iswin() then
+      screen:expect([[
+        ^tty ready                               |
+        x {5:y z}                                   |
+        {1: }                                       |
+                                                |
+                                                |
+                                                |
+                                                |
+                                                |
+      ]])
+    else
+      screen:expect([[
+        ^tty ready                               |
+        x {2:y }{3:z}                                   |
+        {1: }                                       |
+                                                |
+                                                |
+                                                |
+                                                |
+                                                |
+      ]])
+    end
+
+    thelpers.feed_termcode("[A")
+    thelpers.feed_termcode("[2C")
+    if iswin() then
+      screen:expect([[
+        ^tty ready                               |
+        x {6:y}{5: z}                                   |
+                                                |
+                                                |
+                                                |
+                                                |
+                                                |
+                                                |
+      ]])
+    else
+      screen:expect([[
+        ^tty ready                               |
+        x {4:y}{2: }{3:z}                                   |
+                                                |
+                                                |
+                                                |
+                                                |
+                                                |
+                                                |
+      ]])
+    end
+  end)
+
+  it("can use independent cterm and rgb colors", function()
+    -- tell test module to save all attributes (doesn't change nvim options)
+    screen:set_hlstate_cterm(true)
+
+    screen:set_default_attr_ids({
+        [1] = {{bold = true, foreground = Screen.colors.Blue1}, {foreground = 12}, {{hi_name = "NonText", ui_name = "EndOfBuffer", kind = "ui"}}},
+        [2] = {{reverse = true, foreground = Screen.colors.Red}, {foreground = 10, italic=true}, {{hi_name = "NonText", ui_name = "EndOfBuffer", kind = "ui"}}},
+    })
+    screen:expect([[
+      ^                                        |
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+      {1:~                                       }|
+                                              |
+    ]])
+
+    command("hi NonText guifg=Red gui=reverse ctermfg=Green cterm=italic")
+    screen:expect([[
+      ^                                        |
+      {2:~                                       }|
+      {2:~                                       }|
+      {2:~                                       }|
+      {2:~                                       }|
+      {2:~                                       }|
+      {2:~                                       }|
+                                              |
+    ]])
+
+  end)
+end)
diff --git a/test/functional/ui/options_spec.lua b/test/functional/ui/options_spec.lua
index 62b08c0967..322a94763f 100644
--- a/test/functional/ui/options_spec.lua
+++ b/test/functional/ui/options_spec.lua
@@ -30,10 +30,15 @@ describe('ui receives option updates', function()
     ext_popupmenu=false,
     ext_tabline=false,
     ext_wildmenu=false,
+    ext_newgrid=false,
+    ext_hlstate=false,
   }
 
   it("for defaults", function()
     screen:attach()
+    -- NB: UI test suite can be run in both "newgrid" and legacy grid mode.
+    -- In both cases check that the received value is the one requested.
+    defaults.ext_newgrid = screen._options.ext_newgrid or false
     screen:expect(function()
       eq(defaults, screen.options)
     end)
@@ -41,6 +46,7 @@ describe('ui receives option updates', function()
 
   it("when setting options", function()
     screen:attach()
+    defaults.ext_newgrid = screen._options.ext_newgrid or false
     local changed = {}
     for k,v in pairs(defaults) do
       changed[k] = v
@@ -89,6 +95,7 @@ describe('ui receives option updates', function()
     end
 
     screen:attach({ext_cmdline=true, ext_wildmenu=true})
+    defaults.ext_newgrid = screen._options.ext_newgrid or false
     changed.ext_cmdline = true
     changed.ext_wildmenu = true
     screen:expect(function()
diff --git a/test/functional/ui/screen.lua b/test/functional/ui/screen.lua
index 7607131e9b..d71d8cf3a8 100644
--- a/test/functional/ui/screen.lua
+++ b/test/functional/ui/screen.lua
@@ -142,6 +142,10 @@ function Screen.new(width, height)
     _default_attr_ignore = nil,
     _mouse_enabled = true,
     _attrs = {},
+    _hl_info = {},
+    _attr_table = {[0]={{},{}}},
+    _clear_attrs = {},
+    _new_attrs = false,
     _cursor = {
       row = 1, col = 1
     },
@@ -159,10 +163,19 @@ function Screen:set_default_attr_ignore(attr_ignore)
   self._default_attr_ignore = attr_ignore
 end
 
+function Screen:set_hlstate_cterm(val)
+  self._hlstate_cterm = val
+end
+
 function Screen:attach(options)
   if options == nil then
     options = {rgb=true}
   end
+  if options.ext_newgrid == nil then
+    options.ext_newgrid = true
+  end
+  self._options = options
+  self._clear_attrs = (options.ext_newgrid and {{},{}}) or {}
   uimeths.attach(self._width, self._height, options)
 end
 
@@ -176,6 +189,7 @@ end
 
 function Screen:set_option(option, value)
   uimeths.set_option(option, value)
+  self._options[option] = value
 end
 
 -- Asserts that `expected` eventually matches the screen state.
@@ -210,6 +224,11 @@ function Screen:expect(expected, attr_ids, attr_ignore, condition, any)
   end
   local ids = attr_ids or self._default_attr_ids
   local ignore = attr_ignore or self._default_attr_ignore
+  local id_to_index
+  if self._options.ext_hlstate then
+    id_to_index = self:hlstate_check_attrs(ids or {})
+  end
+  self._new_attrs = false
   self:wait(function()
     if condition ~= nil then
       local status, res = pcall(condition)
@@ -223,9 +242,14 @@ function Screen:expect(expected, attr_ids, attr_ignore, condition, any)
               .. ') differs from configured height(' .. self._height .. ') of Screen.')
     end
 
+    if self._options.ext_hlstate and self._new_attrs then
+      id_to_index = self:hlstate_check_attrs(ids or {})
+    end
+
+    local info = self._options.ext_hlstate and id_to_index or ids
     local actual_rows = {}
     for i = 1, self._height do
-      actual_rows[i] = self:_row_repr(self._rows[i], ids, ignore)
+      actual_rows[i] = self:_row_repr(self._rows[i], info, ignore)
     end
 
     if expected == nil then
@@ -339,7 +363,7 @@ function Screen:_handle_resize(width, height)
   for _ = 1, height do
     local cols = {}
     for _ = 1, width do
-      table.insert(cols, {text = ' ', attrs = {}})
+      table.insert(cols, {text = ' ', attrs = self._clear_attrs, hl_id = 0})
     end
     table.insert(rows, cols)
   end
@@ -353,14 +377,24 @@ function Screen:_handle_resize(width, height)
   }
 end
 
+function Screen:_handle_grid_resize(grid, width, height)
+  assert(grid == 1)
+  self:_handle_resize(width, height)
+end
+
+
 function Screen:_handle_mode_info_set(cursor_style_enabled, mode_info)
   self._cursor_style_enabled = cursor_style_enabled
   self._mode_info = mode_info
 end
 
 function Screen:_handle_clear()
-  self:_clear_block(self._scroll_region.top, self._scroll_region.bot,
-                    self._scroll_region.left, self._scroll_region.right)
+  self:_clear_block(1, self._height, 1, self._width)
+end
+
+function Screen:_handle_grid_clear(grid)
+  assert(grid == 1)
+  self:_handle_clear()
 end
 
 function Screen:_handle_eol_clear()
@@ -373,6 +407,12 @@ function Screen:_handle_cursor_goto(row, col)
   self._cursor.col = col + 1
 end
 
+function Screen:_handle_grid_cursor_goto(grid, row, col)
+  assert(grid == 1)
+  self._cursor.row = row + 1
+  self._cursor.col = col + 1
+end
+
 function Screen:_handle_busy_start()
   self._busy = true
 end
@@ -425,6 +465,7 @@ function Screen:_handle_scroll(count)
     for j = left, right do
       target[j].text = source[j].text
       target[j].attrs = source[j].attrs
+      target[j].hl_id = source[j].hl_id
     end
   end
 
@@ -434,6 +475,28 @@ function Screen:_handle_scroll(count)
   end
 end
 
+function Screen:_handle_grid_scroll(grid, top, bot, left, right, rows, cols)
+  assert(grid == 1)
+  assert(cols == 0)
+  -- TODO: if we truly believe we should translate the other way
+  self:_handle_set_scroll_region(top,bot-1,left,right-1)
+  self:_handle_scroll(rows)
+end
+
+function Screen:_handle_hl_attr_define(id, rgb_attrs, cterm_attrs, info)
+  self._attr_table[id] = {rgb_attrs, cterm_attrs}
+  self._hl_info[id] = info
+  self._new_attrs = true
+end
+
+function Screen:get_hl(val)
+  if self._options.ext_newgrid then
+    return self._attr_table[val][1]
+  else
+    return val
+  end
+end
+
 function Screen:_handle_highlight_set(attrs)
   self._attrs = attrs
 end
@@ -442,9 +505,32 @@ function Screen:_handle_put(str)
   local cell = self._rows[self._cursor.row][self._cursor.col]
   cell.text = str
   cell.attrs = self._attrs
+  cell.hl_id = -1
   self._cursor.col = self._cursor.col + 1
 end
 
+function Screen:_handle_grid_line(grid, row, col, items)
+  assert(grid == 1)
+  local line = self._rows[row+1]
+  local colpos = col+1
+  local hl = self._clear_attrs
+  local hl_id = 0
+  for _,item in ipairs(items) do
+    local text, hl_id_cell, count = unpack(item)
+    if hl_id_cell ~= nil then
+      hl_id = hl_id_cell
+      hl = self._attr_table[hl_id]
+    end
+    for _ = 1, (count or 1) do
+      local cell = line[colpos]
+      cell.text = text
+      cell.hl_id = hl_id
+      cell.attrs = hl
+      colpos = colpos+1
+    end
+  end
+end
+
 function Screen:_handle_bell()
   self.bell = true
 end
@@ -498,7 +584,7 @@ function Screen:_clear_row_section(rownum, startcol, stopcol)
   local row = self._rows[rownum]
   for i = startcol, stopcol do
     row[i].text = ' '
-    row[i].attrs = {}
+    row[i].attrs = self._clear_attrs
   end
 end
 
@@ -506,7 +592,11 @@ function Screen:_row_repr(row, attr_ids, attr_ignore)
   local rv = {}
   local current_attr_id
   for i = 1, self._width do
-    local attr_id = self:_get_attr_id(attr_ids, attr_ignore, row[i].attrs)
+    local attrs = row[i].attrs
+    if self._options.ext_newgrid then
+      attrs = attrs[(self._options.rgb and 1) or 2]
+    end
+    local attr_id = self:_get_attr_id(attr_ids, attr_ignore, attrs, row[i].hl_id)
     if current_attr_id and attr_id ~= current_attr_id then
       -- close current attribute bracket, add it before any whitespace
       -- up to the current cell
@@ -573,22 +663,33 @@ function Screen:print_snapshot(attrs, ignore)
   if ignore == nil then
     ignore = self._default_attr_ignore
   end
+  local id_to_index = {}
   if attrs == nil then
     attrs = {}
     if self._default_attr_ids ~= nil then
       for i, a in pairs(self._default_attr_ids) do
         attrs[i] = a
       end
+      if self._options.ext_hlstate then
+        id_to_index = self:hlstate_check_attrs(attrs)
+      end
     end
 
     if ignore ~= true then
       for i = 1, self._height do
         local row = self._rows[i]
         for j = 1, self._width do
-          local attr = row[j].attrs
-          if self:_attr_index(attrs, attr) == nil and self:_attr_index(ignore, attr) == nil then
-            if not self:_equal_attrs(attr, {}) then
-              table.insert(attrs, attr)
+          if self._options.ext_hlstate then
+            local hl_id = row[j].hl_id
+            if hl_id ~= 0 then
+              self:_insert_hl_id(attrs, id_to_index, hl_id)
+            end
+          else
+            local attr = row[j].attrs
+            if self:_attr_index(attrs, attr) == nil and self:_attr_index(ignore, attr) == nil then
+              if not self:_equal_attrs(attr, {}) then
+                table.insert(attrs, attr)
+              end
             end
           end
         end
@@ -597,8 +698,9 @@ function Screen:print_snapshot(attrs, ignore)
   end
 
   local rv = {}
+  local info = self._options.ext_hlstate and id_to_index or attrs
   for i = 1, self._height do
-    table.insert(rv, "  "..self:_row_repr(self._rows[i],attrs, ignore).."|")
+    table.insert(rv, "  "..self:_row_repr(self._rows[i], info, ignore).."|")
   end
   local attrstrs = {}
   local alldefault = true
@@ -606,7 +708,12 @@ function Screen:print_snapshot(attrs, ignore)
     if self._default_attr_ids == nil or self._default_attr_ids[i] ~= a then
       alldefault = false
     end
-    local dict = "{"..self:_pprint_attrs(a).."}"
+    local dict
+    if self._options.ext_hlstate then
+      dict = self:_pprint_hlstate(a)
+    else
+      dict = "{"..self:_pprint_attrs(a).."}"
+    end
     table.insert(attrstrs, "["..tostring(i).."] = "..dict)
   end
   local attrstr = "{"..table.concat(attrstrs, ", ").."}"
@@ -620,6 +727,117 @@ function Screen:print_snapshot(attrs, ignore)
   io.stdout:flush()
 end
 
+function Screen:_insert_hl_id(attrs, id_to_index, hl_id)
+  if id_to_index[hl_id] ~= nil then
+    return id_to_index[hl_id]
+  end
+  local raw_info = self._hl_info[hl_id]
+  local info = {}
+  if #raw_info > 1 then
+    for i, item in ipairs(raw_info) do
+      info[i] = self:_insert_hl_id(attrs, id_to_index, item.id)
+    end
+  else
+    info[1] = {}
+    for k, v in pairs(raw_info[1]) do
+      if k ~= "id" then
+        info[1][k] = v
+      end
+    end
+  end
+
+  local entry = self._attr_table[hl_id]
+  local attrval
+  if self._hlstate_cterm then
+    attrval = {entry[1], entry[2], info} -- unpack() doesn't work
+  else
+    attrval = {entry[1], info}
+  end
+
+
+  table.insert(attrs, attrval)
+  id_to_index[hl_id] = #attrs
+  return #attrs
+end
+
+function Screen:hlstate_check_attrs(attrs)
+  local id_to_index = {}
+  for i = 1,#self._attr_table do
+    local iinfo = self._hl_info[i]
+    local matchinfo = {}
+    if #iinfo > 1 then
+      for k,item in ipairs(iinfo) do
+        matchinfo[k] = id_to_index[item.id]
+      end
+    else
+      matchinfo = iinfo
+    end
+    for k,v in pairs(attrs) do
+      local attr, info, attr_rgb, attr_cterm
+      if self._hlstate_cterm then
+        attr_rgb, attr_cterm, info = unpack(v)
+        attr = {attr_rgb, attr_cterm}
+      else
+        attr, info = unpack(v)
+      end
+      if self:_equal_attr_def(attr, self._attr_table[i]) then
+        if #info == #matchinfo then
+          local match = false
+          if #info == 1 then
+            if self:_equal_info(info[1],matchinfo[1]) then
+              match = true
+            end
+          else
+            match = true
+            for j = 1,#info do
+              if info[j] ~= matchinfo[j] then
+                match = false
+              end
+            end
+          end
+          if match then
+            id_to_index[i] = k
+          end
+        end
+      end
+    end
+  end
+  return id_to_index
+end
+
+
+function Screen:_pprint_hlstate(item)
+    --print(require('inspect')(item))
+    local attrdict = "{"..self:_pprint_attrs(item[1]).."}, "
+    local attrdict2, hlinfo
+    if self._hlstate_cterm then
+      attrdict2 = "{"..self:_pprint_attrs(item[2]).."}, "
+      hlinfo = item[3]
+    else
+      attrdict2 = ""
+      hlinfo = item[2]
+    end
+    local descdict = "{"..self:_pprint_hlinfo(hlinfo).."}"
+    return "{"..attrdict..attrdict2..descdict.."}"
+end
+
+function Screen:_pprint_hlinfo(states)
+  if #states == 1 then
+    local items = {}
+    for f, v in pairs(states[1]) do
+      local desc = tostring(v)
+      if type(v) == type("") then
+        desc = '"'..desc..'"'
+      end
+      table.insert(items, f.." = "..desc)
+    end
+    return "{"..table.concat(items, ", ").."}"
+  else
+    return table.concat(states, ", ")
+  end
+end
+
+
 function Screen:_pprint_attrs(attrs)
     local items = {}
     for f, v in pairs(attrs) do
@@ -643,32 +861,53 @@ local function backward_find_meaningful(tbl, from)  -- luacheck: no unused
   return from
 end
 
-function Screen:_get_attr_id(attr_ids, ignore, attrs)
+function Screen:_get_attr_id(attr_ids, ignore, attrs, hl_id)
   if not attr_ids then
     return
   end
-  for id, a in pairs(attr_ids) do
-    if self:_equal_attrs(a, attrs) then
-       return id
-     end
+
+  if self._options.ext_hlstate then
+    local id = attr_ids[hl_id]
+    if id ~= nil or hl_id == 0 then
+      return id
+    end
+    return "UNEXPECTED "..self:_pprint_attrs(self._attr_table[hl_id][1])
+  else
+    for id, a in pairs(attr_ids) do
+      if self:_equal_attrs(a, attrs) then
+         return id
+       end
+    end
+    if self:_equal_attrs(attrs, {}) or
+        ignore == true or self:_attr_index(ignore, attrs) ~= nil then
+      -- ignore this attrs
+      return nil
+    end
+    return "UNEXPECTED "..self:_pprint_attrs(attrs)
   end
-  if self:_equal_attrs(attrs, {}) or
-      ignore == true or self:_attr_index(ignore, attrs) ~= nil then
-    -- ignore this attrs
-    return nil
+end
+
+function Screen:_equal_attr_def(a, b)
+  if self._hlstate_cterm then
+    return self:_equal_attrs(a[1],b[1]) and self:_equal_attrs(a[2],b[2])
+  else
+    return self:_equal_attrs(a,b[1])
   end
-  return "UNEXPECTED "..self:_pprint_attrs(attrs)
 end
 
 function Screen:_equal_attrs(a, b)
     return a.bold == b.bold and a.standout == b.standout and
        a.underline == b.underline and a.undercurl == b.undercurl and
        a.italic == b.italic and a.reverse == b.reverse and
-       a.foreground == b.foreground and
-       a.background == b.background and
+       a.foreground == b.foreground and a.background == b.background and
        a.special == b.special
 end
 
+function Screen:_equal_info(a, b)
+    return a.kind == b.kind and a.hi_name == b.hi_name and
+       a.ui_name == b.ui_name
+end
+
 function Screen:_attr_index(attrs, attr)
   if not attrs then
     return nil
diff --git a/test/functional/ui/screen_basic_spec.lua b/test/functional/ui/screen_basic_spec.lua
index 6f04cde4d4..75a2d4978d 100644
--- a/test/functional/ui/screen_basic_spec.lua
+++ b/test/functional/ui/screen_basic_spec.lua
@@ -48,13 +48,13 @@ describe('screen', function()
   end)
 end)
 
-describe('Screen', function()
+local function screen_tests(newgrid)
   local screen
 
   before_each(function()
     clear()
     screen = Screen.new()
-    screen:attach()
+    screen:attach({rgb=true,ext_newgrid=newgrid})
     screen:set_default_attr_ids( {
       [0] = {bold=true, foreground=255},
       [1] = {bold=true, reverse=true},
@@ -741,4 +741,12 @@ describe('Screen', function()
                                                            |
     ]])
   end)
+end
+
+describe("Screen (char-based)", function()
+  screen_tests(false)
+end)
+
+describe("Screen (line-based)", function()
+  screen_tests(true)
 end)