ui: use line-based rather than char-based updates in screen.c

Add ext_newgrid and ext_hlstate extensions. These use predefined
highlights and line-segment based updates, for efficiency and
simplicity.. The ext_hlstate extension in addition allows semantic
identification of builtin and syntax highlights.

Reimplement the old char-based updates in the remote UI layer, for
compatibility. For the moment, this is still the default. The bulitin
TUI uses the new line-based protocol.

cmdline uses curwin cursor position when ext_cmdline is active.

21 files changed, 848 insertions, 526 deletions

diff --git a/src/nvim/api/ui.c b/src/nvim/api/ui.c
index 01e6e0ca03..63c2c4a1b9 100644
--- a/src/nvim/api/ui.c
+++ b/src/nvim/api/ui.c
@@ -26,6 +26,12 @@
 typedef struct {
   uint64_t channel_id;
   Array buffer;
+
+  int hl_id;  // current higlight for legacy put event
+  Integer cursor_row, cursor_col;  // Intended visibule cursor position
+
+  // Position of legacy cursor, used both for drawing and visible user cursor.
+  Integer client_row, client_col;
 } UIData;
 
 static PMap(uint64_t) *connected_uis = NULL;
@@ -71,10 +77,9 @@ void nvim_ui_attach(uint64_t channel_id, Integer width, Integer height,
   ui->width = (int)width;
   ui->height = (int)height;
   ui->rgb = true;
-  ui->resize = remote_ui_resize;
-  ui->clear = remote_ui_clear;
-  ui->eol_clear = remote_ui_eol_clear;
-  ui->cursor_goto = remote_ui_cursor_goto;
+  ui->grid_resize = remote_ui_grid_resize;
+  ui->grid_clear = remote_ui_grid_clear;
+  ui->grid_cursor_goto = remote_ui_grid_cursor_goto;
   ui->mode_info_set = remote_ui_mode_info_set;
   ui->update_menu = remote_ui_update_menu;
   ui->busy_start = remote_ui_busy_start;
@@ -82,16 +87,12 @@ void nvim_ui_attach(uint64_t channel_id, Integer width, Integer height,
   ui->mouse_on = remote_ui_mouse_on;
   ui->mouse_off = remote_ui_mouse_off;
   ui->mode_change = remote_ui_mode_change;
-  ui->set_scroll_region = remote_ui_set_scroll_region;
-  ui->scroll = remote_ui_scroll;
-  ui->highlight_set = remote_ui_highlight_set;
-  ui->put = remote_ui_put;
+  ui->grid_scroll = remote_ui_grid_scroll;
+  ui->hl_attr_define = remote_ui_hl_attr_define;
+  ui->raw_line = remote_ui_raw_line;
   ui->bell = remote_ui_bell;
   ui->visual_bell = remote_ui_visual_bell;
   ui->default_colors_set = remote_ui_default_colors_set;
-  ui->update_fg = remote_ui_update_fg;
-  ui->update_bg = remote_ui_update_bg;
-  ui->update_sp = remote_ui_update_sp;
   ui->flush = remote_ui_flush;
   ui->suspend = remote_ui_suspend;
   ui->set_title = remote_ui_set_title;
@@ -103,16 +104,22 @@ void nvim_ui_attach(uint64_t channel_id, Integer width, Integer height,
   memset(ui->ui_ext, 0, sizeof(ui->ui_ext));
 
   for (size_t i = 0; i < options.size; i++) {
-    ui_set_option(ui, options.items[i].key, options.items[i].value, err);
+    ui_set_option(ui, true, options.items[i].key, options.items[i].value, err);
     if (ERROR_SET(err)) {
       xfree(ui);
       return;
     }
   }
 
+  if (ui->ui_ext[kUIHlState]) {
+    ui->ui_ext[kUINewgrid] = true;
+  }
+
   UIData *data = xmalloc(sizeof(UIData));
   data->channel_id = channel_id;
   data->buffer = (Array)ARRAY_DICT_INIT;
+  data->hl_id = 0;
+  data->client_col = -1;
   ui->data = data;
 
   pmap_put(uint64_t)(connected_uis, channel_id, ui);
@@ -174,13 +181,11 @@ void nvim_ui_set_option(uint64_t channel_id, String name,
   }
   UI *ui = pmap_get(uint64_t)(connected_uis, channel_id);
 
-  ui_set_option(ui, name, value, error);
-  if (!ERROR_SET(error)) {
-    ui_refresh();
-  }
+  ui_set_option(ui, false, name, value, error);
 }
 
-static void ui_set_option(UI *ui, String name, Object value, Error *error)
+static void ui_set_option(UI *ui, bool init, String name, Object value,
+                          Error *error)
 {
   if (strequal(name.data, "rgb")) {
     if (value.type != kObjectTypeBoolean) {
@@ -188,40 +193,46 @@ static void ui_set_option(UI *ui, String name, Object value, Error *error)
       return;
     }
     ui->rgb = value.data.boolean;
+    // A little drastic, but only legacy uis need to use this option
+    if (!init) {
+      ui_refresh();
+    }
     return;
   }
 
+  // LEGACY: Deprecated option, use `ext_cmdline` instead.
+  bool is_popupmenu = strequal(name.data, "popupmenu_external");
+
   for (UIExtension i = 0; i < kUIExtCount; i++) {
-    if (strequal(name.data, ui_ext_names[i])) {
+    if (strequal(name.data, ui_ext_names[i])
+        || (i == kUIPopupmenu && is_popupmenu)) {
       if (value.type != kObjectTypeBoolean) {
         snprintf((char *)IObuff, IOSIZE, "%s must be a Boolean",
-                 ui_ext_names[i]);
+                 name.data);
         api_set_error(error, kErrorTypeValidation, (char *)IObuff);
         return;
       }
-      ui->ui_ext[i] = value.data.boolean;
-      return;
-    }
-  }
-
-  if (strequal(name.data, "popupmenu_external")) {
-    // LEGACY: Deprecated option, use `ext_cmdline` instead.
-    if (value.type != kObjectTypeBoolean) {
-      api_set_error(error, kErrorTypeValidation,
-                    "popupmenu_external must be a Boolean");
+      bool boolval = value.data.boolean;
+      if (!init && i == kUINewgrid && boolval != ui->ui_ext[i]) {
+        // There shouldn't be a reason for an UI to do this ever
+        // so explicitly don't support this.
+        api_set_error(error, kErrorTypeValidation,
+                      "ext_newgrid option cannot be changed");
+      }
+      ui->ui_ext[i] = boolval;
+      if (!init) {
+        ui_set_ext_option(ui, i, boolval);
+      }
       return;
     }
-    ui->ui_ext[kUIPopupmenu] = value.data.boolean;
-    return;
   }
 
   api_set_error(error, kErrorTypeValidation, "No such UI option: %s",
                 name.data);
-#undef UI_EXT_OPTION
 }
 
 /// Pushes data into UI.UIData, to be consumed later by remote_ui_flush().
-static void push_call(UI *ui, char *name, Array args)
+static void push_call(UI *ui, const char *name, Array args)
 {
   Array call = ARRAY_DICT_INIT;
   UIData *data = ui->data;
@@ -243,27 +254,293 @@ static void push_call(UI *ui, char *name, Array args)
   kv_A(data->buffer, kv_size(data->buffer) - 1).data.array = call;
 }
 
+static void remote_ui_grid_clear(UI *ui, Integer grid)
+{
+  Array args = ARRAY_DICT_INIT;
+  if (ui->ui_ext[kUINewgrid]) {
+    ADD(args, INTEGER_OBJ(grid));
+  }
+  const char *name = ui->ui_ext[kUINewgrid] ? "grid_clear" : "clear";
+  push_call(ui, name, args);
+}
 
-static void remote_ui_highlight_set(UI *ui, HlAttrs attrs)
+static void remote_ui_grid_resize(UI *ui, Integer grid,
+                                  Integer width, Integer height)
 {
   Array args = ARRAY_DICT_INIT;
+  if (ui->ui_ext[kUINewgrid]) {
+    ADD(args, INTEGER_OBJ(grid));
+  }
+  ADD(args, INTEGER_OBJ(width));
+  ADD(args, INTEGER_OBJ(height));
+  const char *name = ui->ui_ext[kUINewgrid] ? "grid_resize" : "resize";
+  push_call(ui, name, args);
+}
+
+static void remote_ui_grid_scroll(UI *ui, Integer grid, Integer top,
+                                  Integer bot, Integer left, Integer right,
+                                  Integer rows, Integer cols)
+{
+  if (ui->ui_ext[kUINewgrid]) {
+    Array args = ARRAY_DICT_INIT;
+    ADD(args, INTEGER_OBJ(grid));
+    ADD(args, INTEGER_OBJ(top));
+    ADD(args, INTEGER_OBJ(bot));
+    ADD(args, INTEGER_OBJ(left));
+    ADD(args, INTEGER_OBJ(right));
+    ADD(args, INTEGER_OBJ(rows));
+    ADD(args, INTEGER_OBJ(cols));
+    push_call(ui, "grid_scroll", args);
+  } else {
+    Array args = ARRAY_DICT_INIT;
+    ADD(args, INTEGER_OBJ(top));
+    ADD(args, INTEGER_OBJ(bot-1));
+    ADD(args, INTEGER_OBJ(left));
+    ADD(args, INTEGER_OBJ(right-1));
+    push_call(ui, "set_scroll_region", args);
+
+    args = (Array)ARRAY_DICT_INIT;
+    ADD(args, INTEGER_OBJ(rows));
+    push_call(ui, "scroll", args);
+  }
+}
+
+static void remote_ui_default_colors_set(UI *ui, Integer rgb_fg,
+                                         Integer rgb_bg, Integer rgb_sp,
+                                         Integer cterm_fg, Integer cterm_bg)
+{
+  Array args = ARRAY_DICT_INIT;
+  ADD(args, INTEGER_OBJ(rgb_fg));
+  ADD(args, INTEGER_OBJ(rgb_bg));
+  ADD(args, INTEGER_OBJ(rgb_sp));
+  ADD(args, INTEGER_OBJ(cterm_fg));
+  ADD(args, INTEGER_OBJ(cterm_bg));
+  push_call(ui, "default_colors_set", args);
+
+  // Deprecated
+  if (!ui->ui_ext[kUINewgrid]) {
+    args = (Array)ARRAY_DICT_INIT;
+    ADD(args, INTEGER_OBJ(ui->rgb ? rgb_fg : cterm_fg - 1));
+    push_call(ui, "update_fg", args);
+
+    args = (Array)ARRAY_DICT_INIT;
+    ADD(args, INTEGER_OBJ(ui->rgb ? rgb_bg : cterm_bg - 1));
+    push_call(ui, "update_bg", args);
+
+    args = (Array)ARRAY_DICT_INIT;
+    ADD(args, INTEGER_OBJ(ui->rgb ? rgb_sp : -1));
+    push_call(ui, "update_sp", args);
+  }
+}
+
+static void remote_ui_hl_attr_define(UI *ui, Integer id, HlAttrs rgb_attrs,
+                                     HlAttrs cterm_attrs, Array info)
+{
+  if (!ui->ui_ext[kUINewgrid]) {
+    return;
+  }
+  Array args = ARRAY_DICT_INIT;
+
+  ADD(args, INTEGER_OBJ(id));
+
+  Dictionary rgb_hl = hlattrs2dict(&rgb_attrs, true);
+  ADD(args, DICTIONARY_OBJ(rgb_hl));
+
+  Dictionary cterm_hl = hlattrs2dict(&cterm_attrs, false);
+  ADD(args, DICTIONARY_OBJ(cterm_hl));
+
+  if (ui->ui_ext[kUIHlState]) {
+    ADD(args, ARRAY_OBJ(copy_array(info)));
+  } else {
+    ADD(args, ARRAY_OBJ((Array)ARRAY_DICT_INIT));
+  }
+
+  push_call(ui, "hl_attr_define", args);
+}
+
+static void remote_ui_highlight_set(UI *ui, int id)
+{
+  Array args = ARRAY_DICT_INIT;
+  UIData *data = ui->data;
+
+  HlAttrs attrs = HLATTRS_INIT;
+
+  if (data->hl_id == id) {
+    return;
+  }
+  data->hl_id = id;
+
+  if (id != 0) {
+    HlAttrs *aep = syn_attr2entry(id);
+    if (aep) {
+      attrs = *aep;
+    }
+  }
+
   Dictionary hl = hlattrs2dict(&attrs, ui->rgb);
 
   ADD(args, DICTIONARY_OBJ(hl));
   push_call(ui, "highlight_set", args);
 }
 
+/// "true" cursor used only for input focus
+static void remote_ui_grid_cursor_goto(UI *ui, Integer grid, Integer row,
+                                       Integer col)
+{
+  if (ui->ui_ext[kUINewgrid]) {
+    Array args = ARRAY_DICT_INIT;
+    ADD(args, INTEGER_OBJ(grid));
+    ADD(args, INTEGER_OBJ(row));
+    ADD(args, INTEGER_OBJ(col));
+    push_call(ui, "grid_cursor_goto", args);
+  } else {
+    UIData *data = ui->data;
+    data->cursor_row = row;
+    data->cursor_col = col;
+    remote_ui_cursor_goto(ui, row, col);
+  }
+}
+
+/// emulated cursor used both for drawing and for input focus
+static void remote_ui_cursor_goto(UI *ui, Integer row, Integer col)
+{
+  UIData *data = ui->data;
+  if (data->client_row == row && data->client_col == col) {
+    return;
+  }
+  data->client_row = row;
+  data->client_col = col;
+  Array args = ARRAY_DICT_INIT;
+  ADD(args, INTEGER_OBJ(row));
+  ADD(args, INTEGER_OBJ(col));
+  push_call(ui, "cursor_goto", args);
+}
+
+static void remote_ui_put(UI *ui, const char *cell)
+{
+  UIData *data = ui->data;
+  data->client_col++;
+  Array args = ARRAY_DICT_INIT;
+  ADD(args, STRING_OBJ(cstr_to_string(cell)));
+  push_call(ui, "put", args);
+}
+
+static void remote_ui_raw_line(UI *ui, Integer grid, Integer row,
+                               Integer startcol, Integer endcol,
+                               Integer clearcol, Integer clearattr,
+                               const schar_T *chunk, const sattr_T *attrs)
+{
+  UIData *data = ui->data;
+  if (ui->ui_ext[kUINewgrid]) {
+    Array args = ARRAY_DICT_INIT;
+    ADD(args, INTEGER_OBJ(grid));
+    ADD(args, INTEGER_OBJ(row));
+    ADD(args, INTEGER_OBJ(startcol));
+    Array cells = ARRAY_DICT_INIT;
+    int repeat = 0;
+    size_t ncells = (size_t)(endcol-startcol);
+    int last_hl = -1;
+    for (size_t i = 0; i < ncells; i++) {
+      repeat++;
+      if (i == ncells-1 || attrs[i] != attrs[i+1]
+          || STRCMP(chunk[i], chunk[i+1])) {
+        Array cell = ARRAY_DICT_INIT;
+        ADD(cell, STRING_OBJ(cstr_to_string((const char *)chunk[i])));
+        if (attrs[i] != last_hl || repeat > 1) {
+          ADD(cell, INTEGER_OBJ(attrs[i]));
+          last_hl = attrs[i];
+        }
+        if (repeat > 1) {
+          ADD(cell, INTEGER_OBJ(repeat));
+        }
+        ADD(cells, ARRAY_OBJ(cell));
+        repeat = 0;
+      }
+    }
+    if (endcol < clearcol) {
+      Array cell = ARRAY_DICT_INIT;
+      ADD(cell, STRING_OBJ(cstr_to_string(" ")));
+      ADD(cell, INTEGER_OBJ(clearattr));
+      ADD(cell, INTEGER_OBJ(clearcol-endcol));
+      ADD(cells, ARRAY_OBJ(cell));
+    }
+    ADD(args, ARRAY_OBJ(cells));
+
+    push_call(ui, "grid_line", args);
+  } else {
+    for (int i = 0; i < endcol-startcol; i++) {
+      remote_ui_cursor_goto(ui, row, startcol+i);
+      remote_ui_highlight_set(ui, attrs[i]);
+      remote_ui_put(ui, (const char *)chunk[i]);
+      if (utf_ambiguous_width(utf_ptr2char(chunk[i]))) {
+        data->client_col = -1;  // force cursor update
+      }
+    }
+    if (endcol < clearcol) {
+      remote_ui_cursor_goto(ui, row, endcol);
+      remote_ui_highlight_set(ui, (int)clearattr);
+      // legacy eol_clear was only ever used with cleared attributes
+      // so be on the safe side
+      if (clearattr == 0 && clearcol == Columns) {
+        Array args = ARRAY_DICT_INIT;
+        push_call(ui, "eol_clear", args);
+      } else {
+        for (Integer c = endcol; c < clearcol; c++) {
+          remote_ui_put(ui, " ");
+        }
+      }
+    }
+  }
+}
+
 static void remote_ui_flush(UI *ui)
 {
   UIData *data = ui->data;
   if (data->buffer.size > 0) {
+    if (!ui->ui_ext[kUINewgrid]) {
+      remote_ui_cursor_goto(ui, data->cursor_row, data->cursor_col);
+    }
     rpc_send_event(data->channel_id, "redraw", data->buffer);
     data->buffer = (Array)ARRAY_DICT_INIT;
   }
 }
 
+static void remote_ui_cmdline_show(UI *ui, Array args)
+{
+  Array new_args = ARRAY_DICT_INIT;
+  Array contents = args.items[0].data.array;
+  Array new_contents = ARRAY_DICT_INIT;
+  for (size_t i = 0; i < contents.size; i++) {
+    Array item = contents.items[i].data.array;
+    Array new_item = ARRAY_DICT_INIT;
+    int attr = (int)item.items[0].data.integer;
+    if (attr) {
+      HlAttrs *aep = syn_attr2entry(attr);
+      Dictionary rgb_attrs = hlattrs2dict(aep, ui->rgb ? kTrue : kFalse);
+      ADD(new_item, DICTIONARY_OBJ(rgb_attrs));
+    } else {
+      ADD(new_item, DICTIONARY_OBJ((Dictionary)ARRAY_DICT_INIT));
+    }
+    ADD(new_item, copy_object(item.items[1]));
+    ADD(new_contents, ARRAY_OBJ(new_item));
+  }
+  ADD(new_args, ARRAY_OBJ(new_contents));
+  for (size_t i = 1; i < args.size; i++) {
+    ADD(new_args, copy_object(args.items[i]));
+  }
+  push_call(ui, "cmdline_show", new_args);
+}
+
 static void remote_ui_event(UI *ui, char *name, Array args, bool *args_consumed)
 {
+  if (!ui->ui_ext[kUINewgrid]) {
+    // the representation of cmdline_show changed, translate back
+    if (strequal(name, "cmdline_show")) {
+      remote_ui_cmdline_show(ui, args);
+      // never consumes args
+      return;
+    }
+  }
   Array my_args = ARRAY_DICT_INIT;
   // Objects are currently single-reference
   // make a copy, but only if necessary
diff --git a/src/nvim/api/ui_events.in.h b/src/nvim/api/ui_events.in.h
index 3ef16a7ac3..456ad0c8cc 100644
--- a/src/nvim/api/ui_events.in.h
+++ b/src/nvim/api/ui_events.in.h
@@ -10,14 +10,6 @@
 #include "nvim/func_attr.h"
 #include "nvim/ui.h"
 
-void resize(Integer width, Integer height)
-  FUNC_API_SINCE(3);
-void clear(void)
-  FUNC_API_SINCE(3);
-void eol_clear(void)
-  FUNC_API_SINCE(3);
-void cursor_goto(Integer row, Integer col)
-  FUNC_API_SINCE(3);
 void mode_info_set(Boolean enabled, Array cursor_styles)
   FUNC_API_SINCE(3);
 void update_menu(void)
@@ -32,29 +24,12 @@ void mouse_off(void)
   FUNC_API_SINCE(3);
 void mode_change(String mode, Integer mode_idx)
   FUNC_API_SINCE(3);
-void set_scroll_region(Integer top, Integer bot, Integer left, Integer right)
-  FUNC_API_SINCE(3);
-void scroll(Integer count)
-  FUNC_API_SINCE(3);
-void highlight_set(HlAttrs attrs)
-  FUNC_API_SINCE(3) FUNC_API_REMOTE_IMPL FUNC_API_BRIDGE_IMPL;
-void put(String str)
-  FUNC_API_SINCE(3);
 void bell(void)
   FUNC_API_SINCE(3);
 void visual_bell(void)
   FUNC_API_SINCE(3);
 void flush(void)
   FUNC_API_SINCE(3) FUNC_API_REMOTE_IMPL;
-void update_fg(Integer fg)
-  FUNC_API_SINCE(3) FUNC_API_BRIDGE_IMPL;
-void update_bg(Integer bg)
-  FUNC_API_SINCE(3) FUNC_API_BRIDGE_IMPL;
-void update_sp(Integer sp)
-  FUNC_API_SINCE(3) FUNC_API_BRIDGE_IMPL;
-void default_colors_set(Integer rgb_fg, Integer rgb_bg, Integer rgb_sp,
-                        Integer cterm_fg, Integer cterm_bg)
-  FUNC_API_SINCE(4);
 void suspend(void)
   FUNC_API_SINCE(3) FUNC_API_BRIDGE_IMPL;
 void set_title(String title)
@@ -64,6 +39,49 @@ void set_icon(String icon)
 void option_set(String name, Object value)
   FUNC_API_SINCE(4) FUNC_API_BRIDGE_IMPL;
 
+// First revison of the grid protocol, used by default
+void update_fg(Integer fg)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void update_bg(Integer bg)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void update_sp(Integer sp)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void resize(Integer width, Integer height)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void clear(void)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void eol_clear(void)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void cursor_goto(Integer row, Integer col)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void highlight_set(HlAttrs attrs)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY FUNC_API_REMOTE_IMPL;
+void put(String str)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void set_scroll_region(Integer top, Integer bot, Integer left, Integer right)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+void scroll(Integer count)
+  FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
+
+// Second revison of the grid protocol, used with ext_newgrid ui option
+void default_colors_set(Integer rgb_fg, Integer rgb_bg, Integer rgb_sp,
+                        Integer cterm_fg, Integer cterm_bg)
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+void hl_attr_define(Integer id, HlAttrs rgb_attrs, HlAttrs cterm_attrs,
+                    Array info)
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL FUNC_API_BRIDGE_IMPL;
+void grid_resize(Integer grid, Integer width, Integer height)
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+void grid_clear(Integer grid)
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+void grid_cursor_goto(Integer grid, Integer row, Integer col)
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+void grid_line(Integer grid, Integer row, Integer col_start, Array data)
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_ONLY;
+void grid_scroll(Integer grid, Integer top, Integer bot,
+                 Integer left, Integer right, Integer rows, Integer cols)
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+
 void popupmenu_show(Array items, Integer selected, Integer row, Integer col)
   FUNC_API_SINCE(3) FUNC_API_REMOTE_ONLY;
 void popupmenu_hide(void)
diff --git a/src/nvim/event/defs.h b/src/nvim/event/defs.h
index cc875d74b9..55b2d277bb 100644
--- a/src/nvim/event/defs.h
+++ b/src/nvim/event/defs.h
@@ -4,7 +4,7 @@
 #include <assert.h>
 #include <stdarg.h>
 
-#define EVENT_HANDLER_MAX_ARGC 6
+#define EVENT_HANDLER_MAX_ARGC 9
 
 typedef void (*argv_callback)(void **argv);
 typedef struct message {
diff --git a/src/nvim/ex_docmd.c b/src/nvim/ex_docmd.c
index c1b9eff697..b077aefa1e 100644
--- a/src/nvim/ex_docmd.c
+++ b/src/nvim/ex_docmd.c
@@ -6320,8 +6320,10 @@ static void ex_stop(exarg_T *eap)
       autowrite_all();
     }
     apply_autocmds(EVENT_VIMSUSPEND, NULL, NULL, false, NULL);
+
+    // TODO(bfredl): the TUI should do this on suspend
     ui_cursor_goto((int)Rows - 1, 0);
-    ui_linefeed();
+    ui_call_grid_scroll(1, 0, Rows, 0, Columns, 1, 0);
     ui_flush();
     ui_call_suspend();  // call machine specific function
 
diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c
index 6b52ce7c80..775d002e58 100644
--- a/src/nvim/ex_getln.c
+++ b/src/nvim/ex_getln.c
@@ -215,6 +215,8 @@ static int hislen = 0;                  /* actual length of history tables */
 /// user interrupting highlight function to not interrupt command-line.
 static bool getln_interrupted_highlight = false;
 
+static bool need_cursor_update = false;
+
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "ex_getln.c.generated.h"
@@ -2944,30 +2946,22 @@ static void ui_ext_cmdline_show(CmdlineInfo *line)
     char *buf = xmallocz(len);
     memset(buf, '*', len);
     Array item = ARRAY_DICT_INIT;
-    ADD(item, DICTIONARY_OBJ((Dictionary)ARRAY_DICT_INIT));
+    ADD(item, INTEGER_OBJ(0));
     ADD(item, STRING_OBJ(((String) { .data = buf, .size = len })));
     ADD(content, ARRAY_OBJ(item));
   } else if (kv_size(line->last_colors.colors)) {
     for (size_t i = 0; i < kv_size(line->last_colors.colors); i++) {
       CmdlineColorChunk chunk = kv_A(line->last_colors.colors, i);
       Array item = ARRAY_DICT_INIT;
+      ADD(item, INTEGER_OBJ(chunk.attr));
 
-      if (chunk.attr) {
-        HlAttrs *aep = syn_attr2entry(chunk.attr);
-        // TODO(bfredl): this desicion could be delayed by making attr_code a
-        // recognized type
-        Dictionary rgb_attrs = hlattrs2dict(aep, true);
-        ADD(item, DICTIONARY_OBJ(rgb_attrs));
-      } else {
-        ADD(item, DICTIONARY_OBJ((Dictionary)ARRAY_DICT_INIT));
-      }
       ADD(item, STRING_OBJ(cbuf_to_string((char *)line->cmdbuff + chunk.start,
                                           chunk.end-chunk.start)));
       ADD(content, ARRAY_OBJ(item));
     }
   } else {
     Array item = ARRAY_DICT_INIT;
-    ADD(item, DICTIONARY_OBJ((Dictionary)ARRAY_DICT_INIT));
+    ADD(item, INTEGER_OBJ(0));
     ADD(item, STRING_OBJ(cstr_to_string((char *)(line->cmdbuff))));
     ADD(content, ARRAY_OBJ(item));
   }
@@ -3033,6 +3027,8 @@ void cmdline_screen_cleared(void)
     }
     prev_ccline = prev_ccline->prev_ccline;
   }
+
+  need_cursor_update = true;
 }
 
 /// called by ui_flush, do what redraws neccessary to keep cmdline updated.
@@ -3501,6 +3497,10 @@ static void cursorcmd(void)
     if (ccline.redraw_state < kCmdRedrawPos) {
       ccline.redraw_state = kCmdRedrawPos;
     }
+    if (need_cursor_update) {
+      need_cursor_update = false;
+      setcursor();
+    }
     return;
   }
 
diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/nvim/generators/gen_api_ui_events.lua
index 2666ca6e6f..e76b601d8a 100644
--- a/src/nvim/generators/gen_api_ui_events.lua
+++ b/src/nvim/generators/gen_api_ui_events.lua
@@ -132,19 +132,21 @@ for i = 1, #events do
     end
   end
 
-  call_output:write('void ui_call_'..ev.name)
-  write_signature(call_output, ev, '')
-  call_output:write('\n{\n')
-  if ev.remote_only then
-    write_arglist(call_output, ev, false)
-    call_output:write('  UI_LOG('..ev.name..', 0);\n')
-    call_output:write('  ui_event("'..ev.name..'", args);\n')
-  else
-    call_output:write('  UI_CALL')
-    write_signature(call_output, ev, ev.name, true)
-    call_output:write(";\n")
+  if not (ev.remote_only and ev.remote_impl) then
+    call_output:write('void ui_call_'..ev.name)
+    write_signature(call_output, ev, '')
+    call_output:write('\n{\n')
+    if ev.remote_only then
+      write_arglist(call_output, ev, false)
+      call_output:write('  UI_LOG('..ev.name..', 0);\n')
+      call_output:write('  ui_event("'..ev.name..'", args);\n')
+    else
+      call_output:write('  UI_CALL')
+      write_signature(call_output, ev, ev.name, true)
+      call_output:write(";\n")
+    end
+    call_output:write("}\n\n")
   end
-  call_output:write("}\n\n")
 
 end
 
diff --git a/src/nvim/highlight.c b/src/nvim/highlight.c
index 5efbafa128..0b39ba442e 100644
--- a/src/nvim/highlight.c
+++ b/src/nvim/highlight.c
@@ -38,7 +38,7 @@ void highlight_init(void)
 bool highlight_use_hlstate(void)
 {
   if (hlstate_active) {
-      return false;
+    return false;
   }
   hlstate_active = true;
   // hl tables must now be rebuilt.
@@ -89,9 +89,26 @@ static int get_attr_entry(HlEntry entry)
 
   map_put(HlEntry, int)(attr_entry_ids, entry, id);
 
+  Array inspect = hl_inspect(id);
+
+  // Note: internally we don't distinguish between cterm and rgb attributes,
+  // remote_ui_hl_attr_define will however.
+  ui_call_hl_attr_define(id, entry.attr, entry.attr, inspect);
+  api_free_array(inspect);
   return id;
 }
 
+/// When a UI connects, we need to send it the table of higlights used so far.
+void ui_send_all_hls(UI *ui)
+{
+  for (size_t i = 1; i < kv_size(attr_entries); i++) {
+    Array inspect = hl_inspect((int)i);
+    ui->hl_attr_define(ui, (Integer)i, kv_A(attr_entries, i).attr,
+                       kv_A(attr_entries, i).attr, inspect);
+    api_free_array(inspect);
+  }
+}
+
 /// Get attribute code for a syntax group.
 int hl_get_syn_attr(int idx, HlAttrs at_en)
 {
diff --git a/src/nvim/highlight_defs.h b/src/nvim/highlight_defs.h
index 5f637228ba..09d20c75ea 100644
--- a/src/nvim/highlight_defs.h
+++ b/src/nvim/highlight_defs.h
@@ -8,6 +8,8 @@
 typedef int32_t RgbValue;
 
 /// Highlighting attribute bits.
+///
+/// sign bit should not be used here, as it identifies invalid highlight
 typedef enum {
   HL_INVERSE     = 0x01,
   HL_BOLD        = 0x02,
@@ -35,6 +37,17 @@ typedef struct attr_entry {
   .cterm_bg_color = 0, \
 }
 
+// sentinel value that compares unequal to any valid highlight
+#define HLATTRS_INVALID (HlAttrs) { \
+  .rgb_ae_attr = -1, \
+  .cterm_ae_attr = -1, \
+  .rgb_fg_color = -1, \
+  .rgb_bg_color = -1, \
+  .rgb_sp_color = -1, \
+  .cterm_fg_color = 0, \
+  .cterm_bg_color = 0, \
+}
+
 /// Values for index in highlight_attr[].
 /// When making changes, also update hlf_names below!
 typedef enum {
diff --git a/src/nvim/lib/kvec.h b/src/nvim/lib/kvec.h
index 6d54c7f78d..93b2f053bc 100644
--- a/src/nvim/lib/kvec.h
+++ b/src/nvim/lib/kvec.h
@@ -98,14 +98,14 @@
     (*kv_pushp(v) = (x))
 
 #define kv_a(v, i) \
-    (((v).capacity <= (size_t) (i) \
+    (*(((v).capacity <= (size_t) (i) \
       ? ((v).capacity = (v).size = (i) + 1, \
          kv_roundup32((v).capacity), \
-         kv_resize((v), (v).capacity), 0) \
+         kv_resize((v), (v).capacity), 0UL) \
       : ((v).size <= (size_t) (i) \
          ? (v).size = (i) + 1 \
-         : 0)), \
-     (v).items[(i)])
+         : 0UL)), \
+     &(v).items[(i)]))
 
 /// Type of a vector with a few first members allocated on stack
 ///
diff --git a/src/nvim/main.c b/src/nvim/main.c
index 528fd921bc..96c2168bca 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -454,7 +454,6 @@ int main(int argc, char **argv)
   }
 
   setmouse();  // may start using the mouse
-  ui_reset_scroll_region();  // In case Rows changed
 
   if (exmode_active) {
     must_redraw = CLEAR;  // Don't clear the screen when starting in Ex mode.
@@ -1374,7 +1373,7 @@ static void handle_quickfix(mparm_T *paramp)
           paramp->use_ef, OPT_FREE, SID_CARG);
     vim_snprintf((char *)IObuff, IOSIZE, "cfile %s", p_ef);
     if (qf_init(NULL, p_ef, p_efm, true, IObuff, p_menc) < 0) {
-      ui_linefeed();
+      msg_putchar('\n');
       mch_exit(3);
     }
     TIME_MSG("reading errorfile");
diff --git a/src/nvim/message.c b/src/nvim/message.c
index 9d4d421941..46fc9115b4 100644
--- a/src/nvim/message.c
+++ b/src/nvim/message.c
@@ -1888,11 +1888,9 @@ static void msg_scroll_up(void)
                   fill_msgsep, fill_msgsep, HL_ATTR(HLF_MSGSEP));
     }
     int nscroll = MIN(msg_scrollsize()+1, Rows);
-    ui_call_set_scroll_region(Rows-nscroll, Rows-1, 0, Columns-1);
-    screen_del_lines(Rows-nscroll, 0, 1, nscroll, NULL);
-    ui_reset_scroll_region();
+    screen_del_lines(Rows-nscroll, 1, Rows, 0, Columns);
   } else {
-    screen_del_lines(0, 0, 1, (int)Rows, NULL);
+    screen_del_lines(0, 1, (int)Rows, 0, Columns);
   }
 }
 
@@ -2307,9 +2305,9 @@ static int do_more_prompt(int typed_char)
               mp_last = msg_sb_start(mp_last->sb_prev);
           }
 
-          if (toscroll == -1 && screen_ins_lines(0, 0, 1,
-                  (int)Rows, NULL) == OK) {
-            /* display line at top */
+          if (toscroll == -1
+              && screen_ins_lines(0, 1, (int)Rows, 0, (int)Columns) == OK) {
+            // display line at top
             (void)disp_sb_line(0, mp);
           } else {
             /* redisplay all lines */
diff --git a/src/nvim/misc1.c b/src/nvim/misc1.c
index 3d7399f151..684f486c04 100644
--- a/src/nvim/misc1.c
+++ b/src/nvim/misc1.c
@@ -2714,7 +2714,7 @@ int call_shell(char_u *cmd, ShellOpts opts, char_u *extra_shell_arg)
   if (p_verbose > 3) {
     verbose_enter();
     smsg(_("Calling shell to execute: \"%s\""), cmd == NULL ? p_sh : cmd);
-    ui_linefeed();
+    msg_putchar('\n');
     verbose_leave();
   }
 
diff --git a/src/nvim/popupmnu.c b/src/nvim/popupmnu.c
index 1182d3d902..5bd4b4ddff 100644
--- a/src/nvim/popupmnu.c
+++ b/src/nvim/popupmnu.c
@@ -341,6 +341,8 @@ void pum_redraw(void)
     idx = i + pum_first;
     attr = (idx == pum_selected) ? attr_select : attr_norm;