UI: add "compositor" layer to merge grids for TUI use in a correct way

Initially we will use this for the popupmenu, floating windows will
follow soon

NB: writedelay + compositor is weird, we need more flexible
redraw introspection.

15 files changed, 583 insertions, 112 deletions

diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index 928d473b04..8abc43c2aa 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -23,6 +23,7 @@ set(GENERATOR_DIR ${CMAKE_CURRENT_LIST_DIR}/generators)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
 set(API_DISPATCH_GENERATOR ${GENERATOR_DIR}/gen_api_dispatch.lua)
 set(API_UI_EVENTS_GENERATOR ${GENERATOR_DIR}/gen_api_ui_events.lua)
+set(GENERATOR_C_GRAMMAR ${GENERATOR_DIR}/c_grammar.lua)
 set(API_METADATA ${PROJECT_BINARY_DIR}/api_metadata.mpack)
 set(FUNCS_DATA ${PROJECT_BINARY_DIR}/funcs_data.mpack)
 set(MSGPACK_LUA_C_BINDINGS ${GENERATED_DIR}/msgpack_lua_c_bindings.generated.c)
@@ -269,6 +270,7 @@ add_custom_command(
     ${API_HEADERS}
     ${MSGPACK_RPC_HEADERS}
     ${API_DISPATCH_GENERATOR}
+    ${GENERATOR_C_GRAMMAR}
     ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
 )
 
@@ -300,6 +302,7 @@ add_custom_command(
                      ${GENERATED_UI_EVENTS_METADATA}
   DEPENDS
     ${API_UI_EVENTS_GENERATOR}
+    ${GENERATOR_C_GRAMMAR}
     ${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h
 )
 
diff --git a/src/nvim/api/ui.c b/src/nvim/api/ui.c
index bc8a1a941f..9d577db022 100644
--- a/src/nvim/api/ui.c
+++ b/src/nvim/api/ui.c
@@ -113,6 +113,8 @@ void nvim_ui_attach(uint64_t channel_id, Integer width, Integer height,
   ui->set_title = remote_ui_set_title;
   ui->set_icon = remote_ui_set_icon;
   ui->option_set = remote_ui_option_set;
+  ui->win_scroll_over_start = remote_ui_win_scroll_over_start;
+  ui->win_scroll_over_reset = remote_ui_win_scroll_over_reset;
   ui->event = remote_ui_event;
   ui->inspect = remote_ui_inspect;
 
diff --git a/src/nvim/api/ui_events.in.h b/src/nvim/api/ui_events.in.h
index 59a7780651..70f8b9d85c 100644
--- a/src/nvim/api/ui_events.in.h
+++ b/src/nvim/api/ui_events.in.h
@@ -38,6 +38,9 @@ void set_icon(String icon)
   FUNC_API_SINCE(3);
 void option_set(String name, Object value)
   FUNC_API_SINCE(4) FUNC_API_BRIDGE_IMPL;
+// Stop event is not exported as such, represented by EOF in the msgpack stream.
+void stop(void)
+  FUNC_API_NOEXPORT;
 
 // First revison of the grid protocol, used by default
 void update_fg(Integer fg)
@@ -71,28 +74,39 @@ void hl_attr_define(Integer id, HlAttrs rgb_attrs, HlAttrs cterm_attrs,
                     Array info)
   FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL FUNC_API_BRIDGE_IMPL;
 void grid_resize(Integer grid, Integer width, Integer height)
-  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL FUNC_API_COMPOSITOR_IMPL;
 void grid_clear(Integer grid)
-  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL FUNC_API_COMPOSITOR_IMPL;
 void grid_cursor_goto(Integer grid, Integer row, Integer col)
-  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL FUNC_API_COMPOSITOR_IMPL;
 void grid_line(Integer grid, Integer row, Integer col_start, Array data)
   FUNC_API_SINCE(5) FUNC_API_REMOTE_ONLY;
 void grid_scroll(Integer grid, Integer top, Integer bot,
                  Integer left, Integer right, Integer rows, Integer cols)
-  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL;
+  FUNC_API_SINCE(5) FUNC_API_REMOTE_IMPL FUNC_API_COMPOSITOR_IMPL;
 void grid_destroy(Integer grid)
   FUNC_API_SINCE(6) FUNC_API_REMOTE_ONLY;
 
+// For perfomance and simplicity, we use the dense screen representation
+// in internal code, such as compositor and TUI. The remote_ui module will
+// translate this in to the public grid_line format.
+void raw_line(Integer grid, Integer row, Integer startcol,
+              Integer endcol, Integer clearcol, Integer clearattr,
+              Boolean wrap, const schar_T *chunk, const sattr_T *attrs)
+  FUNC_API_NOEXPORT FUNC_API_COMPOSITOR_IMPL;
+
+void event(char *name, Array args, bool *args_consumed)
+  FUNC_API_NOEXPORT;
+
 void win_pos(Integer grid, Integer win, Integer startrow,
              Integer startcol, Integer width, Integer height)
   FUNC_API_SINCE(6) FUNC_API_REMOTE_ONLY;
 void win_hide(Integer grid)
   FUNC_API_SINCE(6) FUNC_API_REMOTE_ONLY;
 void win_scroll_over_start(void)
-  FUNC_API_SINCE(6) FUNC_API_REMOTE_ONLY;
+  FUNC_API_SINCE(6) FUNC_API_BRIDGE_IMPL FUNC_API_COMPOSITOR_IMPL;
 void win_scroll_over_reset(void)
-  FUNC_API_SINCE(6) FUNC_API_REMOTE_ONLY;
+  FUNC_API_SINCE(6) FUNC_API_BRIDGE_IMPL FUNC_API_COMPOSITOR_IMPL;
 
 void popupmenu_show(Array items, Integer selected,
                     Integer row, Integer col, Integer grid)
diff --git a/src/nvim/generators/c_grammar.lua b/src/nvim/generators/c_grammar.lua
index d3047e1a9c..40bdc3e30c 100644
--- a/src/nvim/generators/c_grammar.lua
+++ b/src/nvim/generators/c_grammar.lua
@@ -25,6 +25,7 @@ local c_id = (
 local c_void = P('void')
 local c_param_type = (
   ((P('Error') * fill * P('*') * fill) * Cc('error')) +
+  C((P('const ') ^ -1) * (c_id) * (ws ^ 1) * P('*')) +
   (C(c_id) * (ws ^ 1))
   )
 local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
@@ -43,6 +44,7 @@ local c_proto = Ct(
   (fill * Cg((P('FUNC_API_REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) *
   (fill * Cg((P('FUNC_API_REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) *
   (fill * Cg((P('FUNC_API_BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) *
+  (fill * Cg((P('FUNC_API_COMPOSITOR_IMPL') * Cc(true)), 'compositor_impl') ^ -1) *
   fill * P(';')
   )
 
diff --git a/src/nvim/generators/gen_api_ui_events.lua b/src/nvim/generators/gen_api_ui_events.lua
index e76b601d8a..c8ab310b02 100644
--- a/src/nvim/generators/gen_api_ui_events.lua
+++ b/src/nvim/generators/gen_api_ui_events.lua
@@ -54,7 +54,7 @@ for i = 1, #events do
   ev = events[i]
   assert(ev.return_type == 'void')
 
-  if ev.since == nil then
+  if ev.since == nil and not ev.noexport then
     print("Ui event "..ev.name.." lacks since field.\n")
     os.exit(1)
   end
@@ -65,7 +65,7 @@ for i = 1, #events do
     write_signature(proto_output, ev, 'UI *ui')
     proto_output:write(';\n')
 
-    if not ev.remote_impl then
+    if not ev.remote_impl and not ev.noexport then
       remote_output:write('static void remote_ui_'..ev.name)
       write_signature(remote_output, ev, 'UI *ui')
       remote_output:write('\n{\n')
@@ -74,8 +74,7 @@ for i = 1, #events do
       remote_output:write('}\n\n')
     end
 
-    if not ev.bridge_impl then
-
+    if not ev.bridge_impl and not ev.noexport then
       send, argv, recv, recv_argv, recv_cleanup = '', '', '', '', ''
       argc = 1
       for j = 1, #ev.parameters do
@@ -138,21 +137,36 @@ for i = 1, #events do
     call_output:write('\n{\n')
     if ev.remote_only then
       write_arglist(call_output, ev, false)
-      call_output:write('  UI_LOG('..ev.name..', 0);\n')
+      call_output:write('  UI_LOG('..ev.name..');\n')
       call_output:write('  ui_event("'..ev.name..'", args);\n')
+    elseif ev.compositor_impl then
+      call_output:write('  UI_CALL')
+      write_signature(call_output, ev, '!ui->composed, '..ev.name..', ui', true)
+      call_output:write(";\n")
     else
       call_output:write('  UI_CALL')
-      write_signature(call_output, ev, ev.name, true)
+      write_signature(call_output, ev, 'true, '..ev.name..', ui', true)
       call_output:write(";\n")
     end
     call_output:write("}\n\n")
   end
 
+  if ev.compositor_impl then
+    call_output:write('void ui_composed_call_'..ev.name)
+    write_signature(call_output, ev, '')
+    call_output:write('\n{\n')
+    call_output:write('  UI_CALL')
+    write_signature(call_output, ev, 'ui->composed, '..ev.name..', ui', true)
+    call_output:write(";\n")
+    call_output:write("}\n\n")
+  end
+
 end
 
 proto_output:close()
 call_output:close()
 remote_output:close()
+bridge_output:close()
 
 -- don't expose internal attributes like "impl_name" in public metadata
 exported_attributes = {'name', 'parameters',
@@ -168,7 +182,9 @@ for _,ev in ipairs(events) do
       p[1] = 'Dictionary'
     end
   end
-  exported_events[#exported_events+1] = ev_exported
+  if not ev.noexport then
+    exported_events[#exported_events+1] = ev_exported
+  end
 end
 
 packed = mpack.pack(exported_events)
diff --git a/src/nvim/grid_defs.h b/src/nvim/grid_defs.h
index cf9e550f48..d9ccdbbdab 100644
--- a/src/nvim/grid_defs.h
+++ b/src/nvim/grid_defs.h
@@ -1,6 +1,8 @@
 #ifndef NVIM_GRID_DEFS_H
 #define NVIM_GRID_DEFS_H
 
+#include <stdbool.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #include "nvim/types.h"
@@ -48,8 +50,15 @@ typedef struct {
   // offsets for the grid relative to the global screen
   int row_offset;
   int col_offset;
+
+  // state owned by the compositor.
+  int comp_row;
+  int comp_col;
+  size_t comp_index;
+  bool comp_disabled;
 } ScreenGrid;
 
-#define SCREEN_GRID_INIT { 0, NULL, NULL, NULL, NULL, 0, 0, 0, 0 }
+#define SCREEN_GRID_INIT { 0, NULL, NULL, NULL, NULL, 0, 0, 0, 0, 0, 0, 0, \
+                           false }
 
 #endif  // NVIM_GRID_DEFS_H
diff --git a/src/nvim/highlight.c b/src/nvim/highlight.c
index 41d60fa3ea..20cdbc7ec9 100644
--- a/src/nvim/highlight.c
+++ b/src/nvim/highlight.c
@@ -101,6 +101,9 @@ static int get_attr_entry(HlEntry entry)
 /// When a UI connects, we need to send it the table of highlights used so far.
 void ui_send_all_hls(UI *ui)
 {
+  if (!ui->hl_attr_define) {
+    return;
+  }
   for (size_t i = 1; i < kv_size(attr_entries); i++) {
     Array inspect = hl_inspect((int)i);
     ui->hl_attr_define(ui, (Integer)i, kv_A(attr_entries, i).attr,
diff --git a/src/nvim/main.c b/src/nvim/main.c
index 17f3a894d4..9c8711495c 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -154,6 +154,7 @@ void event_init(void)
   remote_ui_init();
   api_vim_init();
   terminal_init();
+  ui_init();
 }
 
 /// @returns false if main_loop could not be closed gracefully
diff --git a/src/nvim/os_unix.c b/src/nvim/os_unix.c
index 09ba718302..351350d939 100644
--- a/src/nvim/os_unix.c
+++ b/src/nvim/os_unix.c
@@ -139,7 +139,7 @@ void mch_exit(int r)
   exiting = true;
 
   ui_flush();
-  ui_builtin_stop();
+  ui_call_stop();
   ml_close_all(true);           // remove all memfiles
 
   if (!event_teardown() && r == 0) {
diff --git a/src/nvim/popupmnu.c b/src/nvim/popupmnu.c
index 056770f2c0..86cb66c6f3 100644
--- a/src/nvim/popupmnu.c
+++ b/src/nvim/popupmnu.c
@@ -19,6 +19,7 @@
 #include "nvim/move.h"
 #include "nvim/option.h"
 #include "nvim/screen.h"
+#include "nvim/ui_compositor.h"
 #include "nvim/search.h"
 #include "nvim/strings.h"
 #include "nvim/memory.h"
@@ -43,6 +44,8 @@ static int pum_col;                 // left column of pum
 static bool pum_is_visible = false;
 static bool pum_external = false;
 
+static ScreenGrid pum_grid = SCREEN_GRID_INIT;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "popupmnu.c.generated.h"
 #endif
@@ -317,7 +320,7 @@ void pum_display(pumitem_T *array, int size, int selected, bool array_changed)
 /// Redraw the popup menu, using "pum_first" and "pum_selected".
 void pum_redraw(void)
 {
-  int row = pum_row;
+  int row = 0;
   int col;
   int attr_norm = win_hl_attr(curwin, HLF_PNI);
   int attr_select = win_hl_attr(curwin, HLF_PSI);
@@ -334,6 +337,37 @@ void pum_redraw(void)
   int round;
   int n;
 
+  int grid_width = pum_width;
+  int col_off = 0;
+  bool extra_space = false;
+  if (curwin->w_p_rl) {
+    col_off = pum_width;
+    if (pum_col < curwin->w_wincol + curwin->w_width - 1) {
+      grid_width += 1;
+      extra_space = true;
+    }
+  } else if (pum_col > 0) {
+    grid_width += 1;
+    col_off = 1;
+    extra_space = true;
+  }
+  if (pum_scrollbar > 0) {
+    grid_width++;
+  }
+
+  grid_assign_handle(&pum_grid);
+  bool moved = ui_comp_put_grid(&pum_grid, pum_row, pum_col-col_off,
+                                pum_height, grid_width);
+
+  if (!pum_grid.chars
+      || pum_grid.Rows != pum_height || pum_grid.Columns != grid_width) {
+    grid_alloc(&pum_grid, pum_height, grid_width, !moved);
+    ui_call_grid_resize(pum_grid.handle, pum_grid.Columns, pum_grid.Rows);
+  } else if (moved) {
+    grid_invalidate(&pum_grid);
+  }
+
+
   // Never display more than we have
   if (pum_first > pum_size - pum_height) {
     pum_first = pum_size - pum_height;
@@ -356,17 +390,17 @@ void pum_redraw(void)
     screen_puts_line_start(row);
 
     // prepend a space if there is room
-    if (curwin->w_p_rl) {
-      if (pum_col < curwin->w_wincol + curwin->w_width - 1) {
-        grid_putchar(&default_grid, ' ', row, pum_col + 1, attr);
+    if (extra_space) {
+      if (curwin->w_p_rl) {
+        grid_putchar(&pum_grid, ' ', row, col_off + 1, attr);
+      } else {
+        grid_putchar(&pum_grid, ' ', row, col_off - 1, attr);
       }
-    } else if (pum_col > 0) {
-      grid_putchar(&default_grid, ' ', row, pum_col - 1, attr);
     }
 
     // Display each entry, use two spaces for a Tab.
     // Do this 3 times: For the main text, kind and extra info
-    col = pum_col;
+    col = col_off;
     totwidth = 0;
 
     for (round = 1; round <= 3; ++round) {
@@ -423,13 +457,13 @@ void pum_redraw(void)
                   size++;
                 }
               }
-              grid_puts_len(&default_grid, rt, (int)STRLEN(rt), row,
+              grid_puts_len(&pum_grid, rt, (int)STRLEN(rt), row,
                             col - size + 1, attr);
               xfree(rt_start);
               xfree(st);
               col -= width;
             } else {
-              grid_puts_len(&default_grid, st, (int)STRLEN(st), row, col, attr);
+              grid_puts_len(&pum_grid, st, (int)STRLEN(st), row, col, attr);
               xfree(st);
               col += width;
             }
@@ -440,11 +474,11 @@ void pum_redraw(void)
 
             // Display two spaces for a Tab.
             if (curwin->w_p_rl) {
-              grid_puts_len(&default_grid, (char_u *)"  ", 2, row, col - 1,
+              grid_puts_len(&pum_grid, (char_u *)"  ", 2, row, col - 1,
                             attr);
               col -= 2;
             } else {
-              grid_puts_len(&default_grid, (char_u *)"  ", 2, row, col, attr);
+              grid_puts_len(&pum_grid, (char_u *)"  ", 2, row, col, attr);
               col += 2;
             }
             totwidth += 2;
@@ -475,37 +509,37 @@ void pum_redraw(void)
       }
 
       if (curwin->w_p_rl) {
-        grid_fill(&default_grid, row, row + 1, pum_col - pum_base_width - n + 1,
+        grid_fill(&pum_grid, row, row + 1, col_off - pum_base_width - n + 1,
                   col + 1, ' ', ' ', attr);
-        col = pum_col - pum_base_width - n + 1;
+        col = col_off - pum_base_width - n + 1;
       } else {
-        grid_fill(&default_grid, row, row + 1, col,
-                  pum_col + pum_base_width + n, ' ', ' ', attr);
-        col = pum_col + pum_base_width + n;
+        grid_fill(&pum_grid, row, row + 1, col,
+                  col_off + pum_base_width + n, ' ', ' ', attr);
+        col = pum_base_width + n;
       }
       totwidth = pum_base_width + n;
     }
 
     if (curwin->w_p_rl) {
-      grid_fill(&default_grid, row, row + 1, pum_col - pum_width + 1, col + 1,
+      grid_fill(&pum_grid, row, row + 1, col_off - pum_width + 1, col + 1,
                 ' ', ' ', attr);
     } else {
-      grid_fill(&default_grid, row, row + 1, col, pum_col + pum_width, ' ', ' ',
+      grid_fill(&pum_grid, row, row + 1, col, col_off + pum_width, ' ', ' ',
                 attr);
     }
 
     if (pum_scrollbar > 0) {
       if (curwin->w_p_rl) {
-        grid_putchar(&default_grid, ' ', row, pum_col - pum_width,
+        grid_putchar(&pum_grid, ' ', row, col_off - pum_width,
                      i >= thumb_pos && i < thumb_pos + thumb_heigth
                      ? attr_thumb : attr_scroll);
       } else {
-        grid_putchar(&default_grid, ' ', row, pum_col + pum_width,
+        grid_putchar(&pum_grid, ' ', row, col_off + pum_width,
                      i >= thumb_pos && i < thumb_pos + thumb_heigth
                      ? attr_thumb : attr_scroll);
       }
     }
-    grid_puts_line_flush(&default_grid, false);
+    grid_puts_line_flush(&pum_grid, false);
     row++;
   }
 }
@@ -733,9 +767,9 @@ void pum_undisplay(void)
   if (pum_external) {
     ui_call_popupmenu_hide();
   } else {
-    redraw_all_later(SOME_VALID);
-    redraw_tabline = true;
-    status_redraw_all();
+    ui_comp_remove_grid(&pum_grid);
+    // TODO(bfredl): consider the possibility of keeping float grids allocated.
+    grid_free(&pum_grid);
   }
 }
 
diff --git a/src/nvim/screen.c b/src/nvim/screen.c
index 2038fb4d2c..309a8d5439 100644
--- a/src/nvim/screen.c
+++ b/src/nvim/screen.c
@@ -110,6 +110,7 @@
 #include "nvim/syntax.h"
 #include "nvim/terminal.h"
 #include "nvim/ui.h"
+#include "nvim/ui_compositor.h"
 #include "nvim/undo.h"
 #include "nvim/version.h"
 #include "nvim/window.h"
@@ -153,6 +154,8 @@ static bool highlights_invalid = false;
 
 static bool conceal_cursor_used = false;
 
+static bool floats_invalid = false;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "screen.c.generated.h"
 #endif
@@ -453,17 +456,20 @@ void update_screen(int type)
 
     /* redraw status line after the window to minimize cursor movement */
     if (wp->w_redr_status) {
-      win_redr_status(wp, true);  // any popup menu will be redrawn below
+      win_redr_status(wp);
     }
   }
-  send_grid_resize = false;
-  highlights_invalid = false;
+
   end_search_hl();
   // May need to redraw the popup menu.
-  if (pum_drawn()) {
+  if (pum_drawn() && floats_invalid) {
     pum_redraw();
   }
 
+  send_grid_resize = false;
+  highlights_invalid = false;
+  floats_invalid = false;
+
   /* Reset b_mod_set flags.  Going through all windows is probably faster
    * than going through all buffers (there could be many buffers). */
   FOR_ALL_WINDOWS_IN_TAB(wp, curtab) {
@@ -4287,7 +4293,7 @@ win_line (
 /// screen positions.
 static void screen_adjust_grid(ScreenGrid **grid, int *row_off, int *col_off)
 {
-  if (!ui_is_external(kUIMultigrid) && *grid != &default_grid) {
+  if (!(*grid)->chars && *grid != &default_grid) {
     *row_off += (*grid)->row_offset;
     *col_off += (*grid)->col_offset;
     *grid = &default_grid;
@@ -4537,7 +4543,7 @@ void redraw_statuslines(void)
 {
   FOR_ALL_WINDOWS_IN_TAB(wp, curtab) {
     if (wp->w_redr_status) {
-      win_redr_status(wp, false);
+      win_redr_status(wp);
     }
   }
   if (redraw_tabline)
@@ -4808,7 +4814,7 @@ win_redr_status_matches (
 /// If inversion is possible we use it. Else '=' characters are used.
 /// If "ignore_pum" is true, also redraw statusline when the popup menu is
 /// displayed.
-static void win_redr_status(win_T *wp, int ignore_pum)
+static void win_redr_status(win_T *wp)
 {
   int row;
   char_u      *p;
@@ -4831,7 +4837,7 @@ static void win_redr_status(win_T *wp, int ignore_pum)
   if (wp->w_status_height == 0) {
     // no status line, can only be last window
     redraw_cmdline = true;
-  } else if (!redrawing() || (!ignore_pum && pum_drawn())) {
+  } else if (!redrawing()) {
     // Don't redraw right now, do it later. Don't update status line when
     // popup menu is visible and may be drawn over it
     wp->w_redr_status = true;
@@ -5454,10 +5460,10 @@ void grid_puts_len(ScreenGrid *grid, char_u *text, int textlen, int row,
         grid->chars[off + 1][0] = 0;
         grid->attrs[off + 1] = attr;
       }
-      if (put_dirty_first == -1) {
+      if (put_dirty_first == -1 || col < put_dirty_first) {
         put_dirty_first = col;
       }
-      put_dirty_last = col+mbyte_cells;
+      put_dirty_last = MAX(put_dirty_last, col+mbyte_cells);
     }
 
     off += mbyte_cells;
@@ -5864,7 +5870,7 @@ void grid_fill(ScreenGrid *grid, int start_row, int end_row, int start_col,
     if (dirty_last > dirty_first) {
       // TODO(bfredl): support a cleared suffix even with a batched line?
       if (put_dirty_row == row) {
-        if (put_dirty_first == -1) {
+        if (put_dirty_first == -1 || dirty_first < put_dirty_first) {
           put_dirty_first = dirty_first;
         }
         put_dirty_last = MAX(put_dirty_last, dirty_last);
@@ -6030,6 +6036,10 @@ retry:
    */
   ++RedrawingDisabled;
 
+  // win_new_shellsize will recompute floats posititon, but tell the
+  // compositor to now redraw them yet
+  ui_comp_invalidate_screen();
+
   win_new_shellsize();      /* fit the windows in the new sized shell */
 
   comp_col();           /* recompute columns for shown command and ruler */
@@ -6188,6 +6198,7 @@ static void screenclear2(void)
     default_grid.line_wraps[i] = false;
   }
 
+  floats_invalid = true;
   ui_call_grid_clear(1);  // clear the display
   clear_cmdline = false;
   mode_displayed = false;
@@ -6218,7 +6229,7 @@ static void grid_clear_line(ScreenGrid *grid, unsigned off, int width,
   (void)memset(grid->attrs + off, fill, (size_t)width * sizeof(sattr_T));
 }
 
-static void grid_invalidate(ScreenGrid *grid)
+void grid_invalidate(ScreenGrid *grid)
 {
   (void)memset(grid->attrs, -1, grid->Rows * grid->Columns * sizeof(sattr_T));
 }
@@ -6916,11 +6927,6 @@ void showruler(int always)
 {
   if (!always && !redrawing())
     return;
-  if (pum_drawn()) {
-    // Don't redraw right now, do it later.
-    curwin->w_redr_status = true;
-    return;
-  }
   if ((*p_stl != NUL || *curwin->w_p_stl != NUL) && curwin->w_status_height) {
     redraw_custom_statusline(curwin);
   } else {
@@ -6955,10 +6961,6 @@ static void win_redr_ruler(win_T *wp, int always)
   if (wp == lastwin && lastwin->w_status_height == 0)
     if (edit_submode != NULL)
       return;
-  // Don't draw the ruler when the popup menu is visible, it may overlap.
-  if (pum_drawn()) {
-    return;
-  }
 
   if (*p_ruf) {
     int save_called_emsg = called_emsg;
diff --git a/src/nvim/ui.c b/src/nvim/ui.c
index b65f3be746..3dc6ccaf52 100644
--- a/src/nvim/ui.c
+++ b/src/nvim/ui.c
@@ -33,6 +33,7 @@
 #include "nvim/popupmnu.h"
 #include "nvim/screen.h"
 #include "nvim/highlight.h"
+#include "nvim/ui_compositor.h"
 #include "nvim/window.h"
 #include "nvim/cursor_shape.h"
 #ifdef FEAT_TUI
@@ -60,11 +61,11 @@ static bool pending_mode_update = false;
 static handle_T cursor_grid_handle = DEFAULT_GRID_HANDLE;
 
 #if MIN_LOG_LEVEL > DEBUG_LOG_LEVEL
-# define UI_LOG(funname, ...)
+# define UI_LOG(funname)
 #else
 static size_t uilog_seen = 0;
 static char uilog_last_event[1024] = { 0 };
-# define UI_LOG(funname, ...) \
+# define UI_LOG(funname) \
   do { \
     if (strequal(uilog_last_event, STR(funname))) { \
       uilog_seen++; \
@@ -80,42 +81,34 @@ static char uilog_last_event[1024] = { 0 };
   } while (0)
 #endif
 
-// UI_CALL invokes a function on all registered UI instances. The functions can
-// have 0-10 arguments (configurable by SELECT_NTH).
-//
-// See http://stackoverflow.com/a/11172679 for how it works.
-#ifdef _MSC_VER
-# define UI_CALL(funname, ...) \
-    do { \
-      UI_LOG(funname, 0); \
-      for (size_t i = 0; i < ui_count; i++) { \
-        UI *ui = uis[i]; \
-        UI_CALL_MORE(funname, __VA_ARGS__); \
-      } \
-    } while (0)
-#else
-# define UI_CALL(...) \
-    do { \
-      UI_LOG(__VA_ARGS__, 0); \
-      for (size_t i = 0; i < ui_count; i++) { \
-        UI *ui = uis[i]; \
-        UI_CALL_HELPER(CNT(__VA_ARGS__), __VA_ARGS__); \
+// UI_CALL invokes a function on all registered UI instances.
+// This is called by code generated by generators/gen_api_ui_events.lua
+// C code should use ui_call_{funname} instead.
+# define UI_CALL(cond, funname, ...) \
+  do { \
+    bool any_call = false; \
+    for (size_t i = 0; i < ui_count; i++) { \
+      UI *ui = uis[i]; \
+      if (ui->funname && (cond)) { \
+        ui->funname(__VA_ARGS__); \
+        any_call = true; \
       } \
-    } while (0)
-#endif
-#define CNT(...) SELECT_NTH(__VA_ARGS__, MORE, MORE, MORE, MORE, MORE, \
-                            MORE, MORE, MORE, MORE, ZERO, ignore)
-#define SELECT_NTH(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, ...) a11
-#define UI_CALL_HELPER(c, ...) UI_CALL_HELPER2(c, __VA_ARGS__)
-// Resolves to UI_CALL_MORE or UI_CALL_ZERO.
-#define UI_CALL_HELPER2(c, ...) UI_CALL_##c(__VA_ARGS__)
-#define UI_CALL_MORE(method, ...) if (ui->method) ui->method(ui, __VA_ARGS__)
-#define UI_CALL_ZERO(method) if (ui->method) ui->method(ui)
+    } \
+    if (any_call) { \
+      UI_LOG(funname); \
+    } \
+  } while (0)
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "ui_events_call.generated.h"
 #endif
 
+void ui_init(void)
+{
+  default_grid.handle = 1;
+  ui_comp_init();
+}
+
 void ui_builtin_start(void)
 {
 #ifdef FEAT_TUI
@@ -135,17 +128,12 @@ void ui_builtin_start(void)
 #endif
 }
 
-void ui_builtin_stop(void)
-{
-  UI_CALL(stop);
-}
-
 bool ui_rgb_attached(void)
 {
   if (!headless_mode && p_tgc) {
     return true;
   }
-  for (size_t i = 0; i < ui_count; i++) {
+  for (size_t i = 1; i < ui_count; i++) {
     if (uis[i]->rgb) {
       return true;
     }
@@ -155,13 +143,13 @@ bool ui_rgb_attached(void)
 
 bool ui_active(void)
 {
-  return ui_count != 0;
+  return ui_count > 1;
 }
 
 void ui_event(char *name, Array args)
 {
   bool args_consumed = false;
-  UI_CALL(event, name, args, &args_consumed);
+  ui_call_event(name, args, &args_consumed);
   if (!args_consumed) {
     api_free_array(args);
   }
@@ -257,6 +245,9 @@ void ui_attach_impl(UI *ui)
   if (ui_count == MAX_UI_COUNT) {
     abort();
   }
+  if (!ui->ui_ext[kUIMultigrid]) {
+    ui_comp_attach(ui);
+  }
 
   uis[ui_count++] = ui;
   ui_refresh_options();
@@ -303,6 +294,10 @@ void ui_detach_impl(UI *ui)
       && !exiting) {
     ui_schedule_refresh();
   }
+
+  if (!ui->ui_ext[kUIMultigrid]) {
+    ui_comp_detach(ui);
+  }
 }
 
 void ui_set_ext_option(UI *ui, UIExtension ext, bool active)
@@ -322,9 +317,9 @@ void ui_line(ScreenGrid *grid, int row, int startcol, int endcol, int clearcol,
 {
   size_t off = grid->line_offset[row] + (size_t)startcol;
 
-  UI_CALL(raw_line, grid->handle, row, startcol, endcol, clearcol, clearattr,
-          wrap, (const schar_T *)grid->chars + off,
-          (const sattr_T *)grid->attrs + off);
+  ui_call_raw_line(grid->handle, row, startcol, endcol,
+             clearcol, clearattr, wrap, (const schar_T *)grid->chars + off,
+             (const sattr_T *)grid->attrs + off);
 
   if (p_wd) {  // 'writedelay': flush & delay each time.
     int old_row = cursor_row, old_col = cursor_col;
@@ -421,7 +416,7 @@ bool ui_is_external(UIExtension widget)
 Array ui_array(void)
 {
   Array all_uis = ARRAY_DICT_INIT;
-  for (size_t i = 0; i < ui_count; i++) {
+  for (size_t i = 1; i < ui_count; i++) {
     UI *ui = uis[i];
     Dictionary info = ARRAY_DICT_INIT;
     PUT(info, "width", INTEGER_OBJ(ui->width));
diff --git a/src/nvim/ui.h b/src/nvim/ui.h
index 16237214cb..8aeb632f6d 100644
--- a/src/nvim/ui.h
+++ b/src/nvim/ui.h
@@ -36,6 +36,7 @@ typedef struct ui_t UI;
 
 struct ui_t {
   bool rgb;
+  bool composed;
   bool ui_ext[kUIExtCount];  ///< Externalized widgets
   int width, height;
   void *data;
@@ -44,14 +45,6 @@ struct ui_t {
 # include "ui_events.generated.h"
 #endif
 
-  // For perfomance and simplicity, we use the dense screen representation
-  // in the bridge and the TUI. The remote_ui module will translate this
-  // in to the public grid_line format.
-  void (*raw_line)(UI *ui, Integer grid, Integer row, Integer startcol,
-                   Integer endcol, Integer clearcol, Integer clearattr,
-                   Boolean wrap, const schar_T *chunk, const sattr_T *attrs);
-  void (*event)(UI *ui, char *name, Array args, bool *args_consumed);
-  void (*stop)(UI *ui);
   void (*inspect)(UI *ui, Dictionary *info);
 };
 
diff --git a/src/nvim/ui_compositor.c b/src/nvim/ui_compositor.c
new file mode 100644
index 0000000000..2ca0d1a6eb
--- /dev/null
+++ b/src/nvim/ui_compositor.c
@@ -0,0 +1,385 @@
+// This is an open source non-commercial project. Dear PVS-Studio, please check
+// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
+
+// Compositor: merge floating grids with the main grid for display in
+// TUI and non-multigrid UIs.
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <limits.h>
+
+#include "nvim/lib/kvec.h"
+#include "nvim/log.h"
+#include "nvim/main.h"
+#include "nvim/ascii.h"
+#include "nvim/vim.h"
+#include "nvim/ui.h"
+#include "nvim/memory.h"
+#include "nvim/ui_compositor.h"
+#include "nvim/ugrid.h"
+#include "nvim/screen.h"
+#include "nvim/syntax.h"
+#include "nvim/api/private/helpers.h"
+#include "nvim/os/os.h"
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "ui_compositor.c.generated.h"
+#endif
+
+static UI *compositor = NULL;
+static int composed_uis = 0;
+kvec_t(ScreenGrid *) layers = KV_INITIAL_VALUE;
+
+static size_t bufsize = 0;
+static schar_T *linebuf;
+static sattr_T *attrbuf;
+
+#ifndef NDEBUG
+static int chk_width = 0, chk_height = 0;
+#endif
+
+static ScreenGrid *curgrid;
+
+static bool valid_screen = true;
+static bool msg_scroll_mode = false;
+static int msg_first_invalid = 0;
+
+void ui_comp_init(void)
+{
+  if (compositor != NULL) {
+    return;
+  }
+  compositor = xcalloc(1, sizeof(UI));
+
+  compositor->rgb = true;
+  compositor->grid_resize = ui_comp_grid_resize;
+  compositor->grid_clear = ui_comp_grid_clear;
+  compositor->grid_scroll = ui_comp_grid_scroll;
+  compositor->grid_cursor_goto = ui_comp_grid_cursor_goto;
+  compositor->raw_line = ui_comp_raw_line;
+  compositor->win_scroll_over_start = ui_comp_win_scroll_over_start;
+  compositor->win_scroll_over_reset = ui_comp_win_scroll_over_reset;
+
+  // Be unopinionated: will be attached together with a "real" ui anyway
+  compositor->width = INT_MAX;
+  compositor->height = INT_MAX;
+  for (UIExtension i = 0; (int)i < kUIExtCount; i++) {
+    compositor->ui_ext[i] = true;
+  }
+
+  // TODO(bfredl): this will be more complicated if we implement
+  // hlstate per UI (i e reduce hl ids for non-hlstate UIs)
+  compositor->ui_ext[kUIHlState] = false;
+
+  kv_push(layers, &default_grid);
+  curgrid = &default_grid;
+
+  ui_attach_impl(compositor);
+}
+
+
+void ui_comp_attach(UI *ui)
+{
+  composed_uis++;
+  ui->composed = true;
+}
+
+void ui_comp_detach(UI *ui)
+{
+  composed_uis--;
+  if (composed_uis == 0) {
+    xfree(linebuf);
+    xfree(attrbuf);
+    linebuf = NULL;
+    attrbuf = NULL;
+    bufsize = 0;
+  }
+  ui->composed = false;
+}
+
+bool ui_comp_should_draw(void)
+{
+  return composed_uis != 0 && valid_screen;
+}
+
+/// TODO(bfredl): later on the compositor should just use win_float_pos events,
+/// though that will require slight event order adjustment: emit the win_pos
+/// events in the beginning of  update_screen(0), rather than in ui_flush()
+bool ui_comp_put_grid(ScreenGrid *grid, int row, int col, int height, int width)
+{
+  if (grid->comp_index != 0) {
+    bool moved = (row != grid->comp_row) || (col != grid->comp_col);
+    if (ui_comp_should_draw()) {
+      // Redraw the area covered by the old position, and is not covered
+      // by the new position. Disable the grid so that compose_area() will not
+      // use it.
+      grid->comp_disabled = true;
+      compose_area(grid->comp_row, row,
+                   grid->comp_col, grid->comp_col + grid->Columns);
+      if (grid->comp_col < col) {
+        compose_area(MAX(row, grid->comp_row),
+                     MIN(row+height, grid->comp_row+grid->Rows),
+                     grid->comp_col, col);
+      }
+      if (col+width < grid->comp_col+grid->Columns) {
+        compose_area(MAX(row, grid->comp_row),
+                     MIN(row+height, grid->comp_row+grid->Rows),
+                     col+width, grid->comp_col+grid->Columns);
+      }
+      compose_area(row+height, grid->comp_row+grid->Rows,
+                   grid->comp_col, grid->comp_col + grid->Columns);
+      grid->comp_disabled = false;
+    }
+    grid->comp_row = row;
+    grid->comp_col = col;
+    return moved;
+  }
+#ifndef NDEBUG
+  for (size_t i = 0; i < kv_size(layers); i++) {
+    if (kv_A(layers, i) == grid) {
+      assert(false);
+    }
+  }
+#endif
+  // not found: new grid
+  kv_push(layers, grid);
+  grid->comp_row = row;
+  grid->comp_col = col;
+  grid->comp_index = kv_size(layers)-1;
+  return true;
+}
+
+void ui_comp_remove_grid(ScreenGrid *grid)
+{
+  assert(grid != &default_grid);
+  if (grid->comp_index == 0) {
+    // grid wasn't present
+    return;
+  }
+
+  if (curgrid == grid) {
+    curgrid = &default_grid;
+  }
+
+  for (size_t i = grid->comp_index; i < kv_size(layers)-1; i++) {
+    kv_A(layers, i) = kv_A(layers, i+1);
+    kv_A(layers, i)->comp_index = i;
+  }
+  (void)kv_pop(layers);
+  grid->comp_index = 0;
+
+  if (ui_comp_should_draw()) {
+    // inefficent: only draw up to grid->comp_index
+    compose_area(grid->comp_row, grid->comp_row+grid->Rows,
+                 grid->comp_col, grid->comp_col+grid->Columns);
+  }
+}
+
+bool ui_comp_set_grid(handle_T handle)
+{
+  if (curgrid->handle == handle) {
+    return true;
+  }
+  ScreenGrid *grid = NULL;
+  for (size_t i = 0; i < kv_size(layers); i++) {
+    if (kv_A(layers, i)->handle == handle) {
+      grid = kv_A(layers, i);
+      break;
+    }
+  }
+  if (grid != NULL) {
+    curgrid = grid;
+    return true;
+  }
+  return false;
+}
+
+static void ui_comp_grid_cursor_goto(UI *ui, Integer grid_handle,
+                                     Integer r, Integer c)
+{
+  if (!ui_comp_should_draw() || !ui_comp_set_grid((int)grid_handle)) {
+    return;
+  }
+  int cursor_row = curgrid->comp_row+(int)r;
+  int cursor_col = curgrid->comp_col+(int)c;
+
+  if (cursor_col >= default_grid.Columns || cursor_row >= default_grid.Rows) {
+    // TODO(bfredl): this happens with 'writedelay', refactor?
+    // abort();
+    return;
+  }
+  ui_composed_call_grid_cursor_goto(1, cursor_row, cursor_col);
+}
+
+
+/// Baseline implementation. This is always correct, but we can sometimes
+/// do something more efficient (where efficiency means smaller deltas to
+/// the downstream UI.)
+static void compose_line(Integer row, Integer startcol, Integer endcol,
+                         bool wrap)
+{
+  int col = (int)startcol;
+  ScreenGrid *grid = NULL;
+
+  while (col < endcol) {
+    int until = 0;
+    for (size_t i = 0; i < kv_size(layers); i++) {
+      ScreenGrid *g = kv_A(layers, i);
+      if (g->comp_row > row || row >= g->comp_row + g->Rows
+          || g->comp_disabled) {
+        continue;
+      }
+      if (g->comp_col <= col && col < g->comp_col+g->Columns) {
+        grid = g;
+        until = g->comp_col+g->Columns;
+      } else if (g->comp_col > col) {
+        until = MIN(until, g->comp_col);
+      }
+    }
+    until = MIN(until, (int)endcol);
+
+    assert(grid != NULL);
+    assert(until > col);
+    assert(until <= default_grid.Columns);
+    size_t n = (size_t)(until-col);
+    size_t off = grid->line_offset[row-grid->comp_row]
+                 + (size_t)(col-grid->comp_col);
+    memcpy(linebuf+(col-startcol), grid->chars+off, n * sizeof(*linebuf));
+    memcpy(attrbuf+(col-startcol), grid->attrs+off, n * sizeof(*attrbuf));
+    col = until;
+  }
+  assert(endcol <= chk_width);
+  assert(row < chk_height);
+  // TODO(bfredl): too conservative, need check
+  // grid->line_wraps if grid->Width == Width
+  wrap = wrap && grid && grid->handle == 1;
+  ui_composed_call_raw_line(1, row, startcol, endcol, endcol, 0, wrap,
+                            (const schar_T *)linebuf, (const sattr_T *)attrbuf);
+}
+
+static void compose_area(Integer startrow, Integer endrow,
+                         Integer startcol, Integer endcol)
+{
+  endrow = MIN(endrow, default_grid.Rows);
+  endcol = MIN(endcol, default_grid.Columns);
+  for (int r = (int)startrow; r < endrow; r++) {
+    compose_line(r, startcol, endcol, false);
+  }
+}
+
+
+static void draw_line(ScreenGrid *grid, Integer row, Integer startcol,
+                      Integer endcol, Integer clearcol, Integer clearattr,
+                      bool wrap, const schar_T *chunk, const sattr_T *attrs)
+{
+  row += grid->comp_row;
+  startcol += grid->comp_col;
+  endcol += grid->comp_col;
+  clearcol += grid->comp_col;
+  wrap = wrap && grid->handle == 1;
+  assert(clearcol <= default_grid.Columns);
+  if (kv_size(layers) > grid->comp_index+1) {
+    compose_line(row, startcol, clearcol, wrap);
+  } else {
+    ui_composed_call_raw_line(1, row, startcol, endcol, clearcol, clearattr,
+                              wrap, chunk, attrs);
+  }
+}
+
+static void ui_comp_raw_line(UI *ui, Integer grid, Integer row,
+                             Integer startcol, Integer endcol,
+                             Integer clearcol, Integer clearattr, bool wrap,
+                             const schar_T *chunk, const sattr_T *attrs)
+{
+  if (!ui_comp_should_draw() || !ui_comp_set_grid((int)grid)) {
+    return;
+  }
+  draw_line(curgrid, row, startcol, endcol, clearcol, clearattr, wrap, chunk,
+            attrs);
+}
+
+/// The screen is invalid and will soon be cleared
+///
+/// Don't redraw floats until screen is cleared
+void ui_comp_invalidate_screen(void)
+{
+  valid_screen = false;
+}
+
+static void ui_comp_grid_clear(UI *ui, Integer grid)
+{
+  // By design, only first grid uses clearing.
+  assert(grid == 1);
+  ui_composed_call_grid_clear(1);
+  valid_screen = true;
+}
+
+// TODO(bfredl): These events are somewhat of a hack. multiline messages
+// should later on be a separate grid, then this would just be ordinary
+// ui_comp_put_grid and ui_comp_remove_grid calls.
+static void ui_comp_win_scroll_over_start(UI *ui)
+{
+  msg_scroll_mode = true;
+  msg_first_invalid = ui->height;
+}
+
+static void ui_comp_win_scroll_over_reset(UI *ui)
+{
+  msg_scroll_mode = false;
+  for (size_t i = 1; i < kv_size(layers); i++) {
+    ScreenGrid *grid = kv_A(layers, i);
+    if (grid->comp_row+grid->Rows > msg_first_invalid) {
+      compose_area(msg_first_invalid, grid->comp_row+grid->Rows,
+                   grid->comp_col, grid->comp_col+grid->Columns);
+    }
+  }
+}
+
+static void ui_comp_grid_scroll(UI *ui, Integer grid, Integer top,
+                                Integer bot, Integer left, Integer right,
+                                Integer rows, Integer cols)
+{
+  if (!ui_comp_should_draw() || !ui_comp_set_grid((int)grid)) {
+    return;
+  }
+  top += curgrid->comp_row;
+  bot += curgrid->comp_row;
+  left += curgrid->comp_col;
+  right += curgrid->comp_col;
+  if (!msg_scroll_mode && kv_size(layers) > curgrid->comp_index+1) {
+    // TODO(bfredl):
+    // 1. check if rectangles actually overlap
+    // 2. calulate subareas that can scroll.
+    if (rows > 0) {
+      bot -= rows;
+    } else {
+      top += (-rows);
+    }
+    compose_area(top, bot, left, right);
+  } else {
+    msg_first_invalid = MIN(msg_first_invalid, (int)top);
+    ui_composed_call_grid_scroll(1, top, bot, left, right, rows, cols);
+  }
+}
+
+static void ui_comp_grid_resize(UI *ui, Integer grid,
+                                Integer width, Integer height)
+{
+  if (grid == 1) {
+    ui_composed_call_grid_resize(1, width, height);
+#ifndef NDEBUG
+    chk_width = (int)width;
+    chk_height = (int)height;
+#endif
+    size_t new_bufsize = (size_t)width;
+    if (bufsize != new_bufsize) {
+      xfree(linebuf);
+      xfree(attrbuf);
+      linebuf = xmalloc(new_bufsize * sizeof(*linebuf));
+      attrbuf = xmalloc(new_bufsize * sizeof(*attrbuf));
+      bufsize = new_bufsize;
+    }
+  }
+}
+
diff --git a/src/nvim/ui_compositor.h b/src/nvim/ui_compositor.h
new file mode 100644
index 0000000000..b3780db532
--- /dev/null
+++ b/src/nvim/ui_compositor.h
@@ -0,0 +1,12 @@
+// Bridge for communication between a UI thread and nvim core.
+// Used by the built-in TUI and libnvim-based UIs.
+#ifndef NVIM_UI_COMPOSITOR_H
+#define NVIM_UI_COMPOSITOR_H
+
+#include "nvim/ui.h"
+#include "nvim/event/defs.h"
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "ui_compositor.h.generated.h"
+#endif
+#endif  // NVIM_UI_COMPOSITOR_H