Merge remote-tracking branch 'upstream/master' into rahm

1 files changed, 424 insertions, 132 deletions

diff --git a/src/nvim/api/ui.c b/src/nvim/api/ui.c
index d86aecc318..54ce838b9b 100644
--- a/src/nvim/api/ui.c
+++ b/src/nvim/api/ui.c
@@ -9,25 +9,46 @@
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/api/ui.h"
+#include "nvim/channel.h"
 #include "nvim/cursor_shape.h"
 #include "nvim/highlight.h"
 #include "nvim/map.h"
 #include "nvim/memory.h"
 #include "nvim/msgpack_rpc/channel.h"
+#include "nvim/msgpack_rpc/helpers.h"
+#include "nvim/option.h"
 #include "nvim/popupmnu.h"
 #include "nvim/screen.h"
 #include "nvim/ui.h"
 #include "nvim/vim.h"
 #include "nvim/window.h"
 
-#ifdef INCLUDE_GENERATED_DECLARATIONS
-# include "api/ui.c.generated.h"
-# include "ui_events_remote.generated.h"
-#endif
-
 typedef struct {
   uint64_t channel_id;
-  Array buffer;
+
+#define UI_BUF_SIZE 4096  ///< total buffer size for pending msgpack data.
+  /// guranteed size available for each new event (so packing of simple events
+  /// and the header of grid_line will never fail)
+#define EVENT_BUF_SIZE 256
+  char buf[UI_BUF_SIZE];  ///< buffer of packed but not yet sent msgpack data
+  char *buf_wptr;  ///< write head of buffer
+  const char *cur_event;  ///< name of current event (might get multiple arglists)
+  Array call_buf;  ///< buffer for constructing a single arg list (max 16 elements!)
+
+  // state for write_cb, while packing a single arglist to msgpack. This
+  // might fail due to buffer overflow.
+  size_t pack_totlen;
+  bool buf_overflow;
+  char *temp_buf;
+
+  // We start packing the two outermost msgpack arrays before knowing the total
+  // number of elements. Thus track the location where array size will need
+  // to be written in the msgpack buffer, once the specifc array is finished.
+  char *nevents_pos;
+  char *ncalls_pos;
+  uint32_t nevents;  ///< number of distinct events (top-level args to "redraw"
+  uint32_t ncalls;  ///< number of calls made to the current event (plus one for the name!)
+  bool flushed_events;  ///< events where sent to client without "flush" event
 
   int hl_id;  // Current highlight for legacy put event.
   Integer cursor_row, cursor_col;  // Intended visible cursor position.
@@ -37,8 +58,76 @@ typedef struct {
   bool wildmenu_active;
 } UIData;
 
+#define BUF_POS(data) ((size_t)((data)->buf_wptr - (data)->buf))
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "api/ui.c.generated.h"
+# include "ui_events_remote.generated.h"
+#endif
+
 static PMap(uint64_t) connected_uis = MAP_INIT;
 
+#define mpack_w(b, byte) *(*b)++ = (char)(byte);
+static void mpack_w2(char **b, uint32_t v)
+{
+  *(*b)++ = (char)((v >> 8) & 0xff);
+  *(*b)++ = (char)(v & 0xff);
+}
+
+static void mpack_w4(char **b, uint32_t v)
+{
+  *(*b)++ = (char)((v >> 24) & 0xff);
+  *(*b)++ = (char)((v >> 16) & 0xff);
+  *(*b)++ = (char)((v >> 8) & 0xff);
+  *(*b)++ = (char)(v & 0xff);
+}
+
+static void mpack_uint(char **buf, uint32_t val)
+{
+  if (val > 0xffff) {
+    mpack_w(buf, 0xce);
+    mpack_w4(buf, val);
+  } else if (val > 0xff) {
+    mpack_w(buf, 0xcd);
+    mpack_w2(buf, val);
+  } else if (val > 0x7f) {
+    mpack_w(buf, 0xcc);
+    mpack_w(buf, val);
+  } else {
+    mpack_w(buf, val);
+  }
+}
+
+static void mpack_array(char **buf, uint32_t len)
+{
+  if (len < 0x10) {
+    mpack_w(buf, 0x90 | len);
+  } else if (len < 0x10000) {
+    mpack_w(buf, 0xdc);
+    mpack_w2(buf, len);
+  } else {
+    mpack_w(buf, 0xdd);
+    mpack_w4(buf, len);
+  }
+}
+
+static char *mpack_array_dyn16(char **buf)
+{
+  mpack_w(buf, 0xdc);
+  char *pos = *buf;
+  mpack_w2(buf, 0xFFEF);
+  return pos;
+}
+
+static void mpack_str(char **buf, const char *str)
+{
+  assert(sizeof(schar_T) - 1 < 0x20);
+  size_t len = STRLEN(str);
+  mpack_w(buf, 0xa0 | len);
+  memcpy(*buf, str, len);
+  *buf += len;
+}
+
 void remote_ui_disconnect(uint64_t channel_id)
 {
   UI *ui = pmap_get(uint64_t)(&connected_uis, channel_id);
@@ -46,9 +135,9 @@ void remote_ui_disconnect(uint64_t channel_id)
     return;
   }
   UIData *data = ui->data;
-  api_free_array(data->buffer);  // Destroy pending screen updates.
+  kv_destroy(data->call_buf);
   pmap_del(uint64_t)(&connected_uis, channel_id);
-  xfree(ui->data);
+  xfree(data);
   ui->data = NULL;  // Flag UI as "stopped".
   ui_detach_impl(ui, channel_id);
   xfree(ui);
@@ -158,10 +247,19 @@ void nvim_ui_attach(uint64_t channel_id, Integer width, Integer height, Dictiona
 
   UIData *data = xmalloc(sizeof(UIData));
   data->channel_id = channel_id;
-  data->buffer = (Array)ARRAY_DICT_INIT;
+  data->cur_event = NULL;
   data->hl_id = 0;
   data->client_col = -1;
+  data->nevents_pos = NULL;
+  data->nevents = 0;
+  data->flushed_events = false;
+  data->ncalls_pos = NULL;
+  data->ncalls = 0;
+  data->buf_wptr = data->buf;
+  data->temp_buf = NULL;
   data->wildmenu_active = false;
+  data->call_buf = (Array)ARRAY_DICT_INIT;
+  kv_ensure_space(data->call_buf, 16);
   ui->data = data;
 
   pmap_put(uint64_t)(&connected_uis, channel_id, ui);
@@ -195,7 +293,6 @@ void nvim_ui_detach(uint64_t channel_id, Error *err)
   remote_ui_disconnect(channel_id);
 }
 
-
 void nvim_ui_try_resize(uint64_t channel_id, Integer width, Integer height, Error *err)
   FUNC_API_SINCE(1) FUNC_API_REMOTE_ONLY
 {
@@ -255,6 +352,49 @@ static void ui_set_option(UI *ui, bool init, String name, Object value, Error *e
     return;
   }
 
+  if (strequal(name.data, "term_name")) {
+    if (value.type != kObjectTypeString) {
+      api_set_error(error, kErrorTypeValidation, "term_name must be a String");
+      return;
+    }
+    set_tty_option("term", string_to_cstr(value.data.string));
+    return;
+  }
+
+  if (strequal(name.data, "term_colors")) {
+    if (value.type != kObjectTypeInteger) {
+      api_set_error(error, kErrorTypeValidation, "term_colors must be a Integer");
+      return;
+    }
+    t_colors = (int)value.data.integer;
+    return;
+  }
+
+  if (strequal(name.data, "term_background")) {
+    if (value.type != kObjectTypeString) {
+      api_set_error(error, kErrorTypeValidation, "term_background must be a String");
+      return;
+    }
+    set_tty_background(value.data.string.data);
+    return;
+  }
+
+  if (strequal(name.data, "stdin_fd")) {
+    if (value.type != kObjectTypeInteger || value.data.integer < 0) {
+      api_set_error(error, kErrorTypeValidation, "stdin_fd must be a non-negative Integer");
+      return;
+    }
+
+    if (starting != NO_SCREEN) {
+      api_set_error(error, kErrorTypeValidation,
+                    "stdin_fd can only be used with first attached ui");
+      return;
+    }
+
+    stdin_fd = (int)value.data.integer;
+    return;
+  }
+
   // LEGACY: Deprecated option, use `ext_cmdline` instead.
   bool is_popupmenu = strequal(name.data, "popupmenu_external");
 
@@ -305,7 +445,11 @@ void nvim_ui_try_resize_grid(uint64_t channel_id, Integer grid, Integer width, I
     return;
   }
 
-  ui_grid_resize((handle_T)grid, (int)width, (int)height, err);
+  if (grid == DEFAULT_GRID_HANDLE) {
+    nvim_ui_try_resize(channel_id, width, height, err);
+  } else {
+    ui_grid_resize((handle_T)grid, (int)width, (int)height, err);
+  }
 }
 
 /// Tells Nvim the number of elements displaying in the popumenu, to decide
@@ -386,34 +530,128 @@ void nvim_ui_pum_set_bounds(uint64_t channel_id, Float width, Float height, Floa
   ui->pum_pos = true;
 }
 
-/// Pushes data into UI.UIData, to be consumed later by remote_ui_flush().
-static void push_call(UI *ui, const char *name, Array args)
+static void flush_event(UIData *data)
+{
+  if (data->cur_event) {
+    mpack_w2(&data->ncalls_pos, data->ncalls);
+    data->cur_event = NULL;
+  }
+  if (!data->nevents_pos) {
+    assert(BUF_POS(data) == 0);
+    char **buf = &data->buf_wptr;
+    // [2, "redraw", [...]]
+    mpack_array(buf, 3);
+    mpack_uint(buf, 2);
+    mpack_str(buf, "redraw");
+    data->nevents_pos = mpack_array_dyn16(buf);
+  }
+}
+
+static inline int write_cb(void *vdata, const char *buf, size_t len)
+{
+  UIData *data = (UIData *)vdata;
+  if (!buf) {
+    return 0;
+  }
+
+  data->pack_totlen += len;
+  if (!data->temp_buf && UI_BUF_SIZE - BUF_POS(data) < len) {
+    data->buf_overflow = true;
+    return 0;
+  }
+
+  memcpy(data->buf_wptr, buf, len);
+  data->buf_wptr += len;
+
+  return 0;
+}
+
+static bool prepare_call(UI *ui, const char *name)
 {
-  Array call = ARRAY_DICT_INIT;
   UIData *data = ui->data;
 
-  // To optimize data transfer(especially for "put"), we bundle adjacent
+  if (BUF_POS(data) > UI_BUF_SIZE - EVENT_BUF_SIZE) {
+    remote_ui_flush_buf(ui);
+  }
+
+  // To optimize data transfer(especially for "grid_line"), we bundle adjacent
   // calls to same method together, so only add a new call entry if the last
   // method call is different from "name"
-  if (kv_size(data->buffer)) {
-    call = kv_A(data->buffer, kv_size(data->buffer) - 1).data.array;
-  }
 
-  if (!kv_size(call) || strcmp(kv_A(call, 0).data.string.data, name)) {
-    call = (Array)ARRAY_DICT_INIT;
-    ADD(data->buffer, ARRAY_OBJ(call));
-    ADD(call, STRING_OBJ(cstr_to_string(name)));
+  if (!data->cur_event || !strequal(data->cur_event, name)) {
+    flush_event(data);
+    data->cur_event = name;
+    char **buf = &data->buf_wptr;
+    data->ncalls_pos = mpack_array_dyn16(buf);
+    mpack_str(buf, name);
+    data->nevents++;
+    data->ncalls = 1;
+    return true;
   }
 
-  ADD(call, ARRAY_OBJ(args));
-  kv_A(data->buffer, kv_size(data->buffer) - 1).data.array = call;
+  return false;
+}
+
+/// Pushes data into UI.UIData, to be consumed later by remote_ui_flush().
+static void push_call(UI *ui, const char *name, Array args)
+{
+  UIData *data = ui->data;
+  bool pending = data->nevents_pos;
+  char *buf_pos_save = data->buf_wptr;
+
+  bool new_event = prepare_call(ui, name);
+
+  msgpack_packer pac;
+  data->pack_totlen = 0;
+  data->buf_overflow = false;
+  msgpack_packer_init(&pac, data, write_cb);
+  msgpack_rpc_from_array(args, &pac);
+  if (data->buf_overflow) {
+    data->buf_wptr = buf_pos_save;
+    if (new_event) {
+      data->cur_event = NULL;
+      data->nevents--;
+    }
+    if (pending) {
+      remote_ui_flush_buf(ui);
+    }
+
+    if (data->pack_totlen > UI_BUF_SIZE - strlen(name) - 20) {
+      // TODO(bfredl): manually testable by setting UI_BUF_SIZE to 1024 (mode_info_set)
+      data->temp_buf = xmalloc(20 + strlen(name) + data->pack_totlen);
+      data->buf_wptr = data->temp_buf;
+      char **buf = &data->buf_wptr;
+      mpack_array(buf, 3);
+      mpack_uint(buf, 2);
+      mpack_str(buf, "redraw");
+      mpack_array(buf, 1);
+      mpack_array(buf, 2);
+      mpack_str(buf, name);
+    } else {
+      prepare_call(ui, name);
+    }
+    data->pack_totlen = 0;
+    data->buf_overflow = false;
+    msgpack_rpc_from_array(args, &pac);
+
+    if (data->temp_buf) {
+      size_t size = (size_t)(data->buf_wptr - data->temp_buf);
+      WBuffer *buf = wstream_new_buffer(data->temp_buf, size, 1, xfree);
+      rpc_write_raw(data->channel_id, buf);
+      data->temp_buf = NULL;
+      data->buf_wptr = data->buf;
+      data->nevents_pos = NULL;
+    }
+  }
+  data->ncalls++;
 }
 
 static void remote_ui_grid_clear(UI *ui, Integer grid)
 {
-  Array args = ARRAY_DICT_INIT;
+  UIData *data = ui->data;
+  Array args = data->call_buf;
   if (ui->ui_ext[kUILinegrid]) {
-    ADD(args, INTEGER_OBJ(grid));
+    ADD_C(args, INTEGER_OBJ(grid));
   }
   const char *name = ui->ui_ext[kUILinegrid] ? "grid_clear" : "clear";
   push_call(ui, name, args);
@@ -421,12 +659,13 @@ static void remote_ui_grid_clear(UI *ui, Integer grid)
 
 static void remote_ui_grid_resize(UI *ui, Integer grid, Integer width, Integer height)
 {
-  Array args = ARRAY_DICT_INIT;
+  UIData *data = ui->data;
+  Array args = data->call_buf;
   if (ui->ui_ext[kUILinegrid]) {
-    ADD(args, INTEGER_OBJ(grid));
+    ADD_C(args, INTEGER_OBJ(grid));
   }
-  ADD(args, INTEGER_OBJ(width));
-  ADD(args, INTEGER_OBJ(height));
+  ADD_C(args, INTEGER_OBJ(width));
+  ADD_C(args, INTEGER_OBJ(height));
   const char *name = ui->ui_ext[kUILinegrid] ? "grid_resize" : "resize";
   push_call(ui, name, args);
 }
@@ -434,35 +673,36 @@ static void remote_ui_grid_resize(UI *ui, Integer grid, Integer width, Integer h
 static void remote_ui_grid_scroll(UI *ui, Integer grid, Integer top, Integer bot, Integer left,
                                   Integer right, Integer rows, Integer cols)
 {
+  UIData *data = ui->data;
   if (ui->ui_ext[kUILinegrid]) {
-    Array args = ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(grid));
-    ADD(args, INTEGER_OBJ(top));
-    ADD(args, INTEGER_OBJ(bot));
-    ADD(args, INTEGER_OBJ(left));
-    ADD(args, INTEGER_OBJ(right));
-    ADD(args, INTEGER_OBJ(rows));
-    ADD(args, INTEGER_OBJ(cols));
+    Array args = data->call_buf;
+    ADD_C(args, INTEGER_OBJ(grid));
+    ADD_C(args, INTEGER_OBJ(top));
+    ADD_C(args, INTEGER_OBJ(bot));
+    ADD_C(args, INTEGER_OBJ(left));
+    ADD_C(args, INTEGER_OBJ(right));
+    ADD_C(args, INTEGER_OBJ(rows));
+    ADD_C(args, INTEGER_OBJ(cols));
     push_call(ui, "grid_scroll", args);
   } else {
-    Array args = ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(top));
-    ADD(args, INTEGER_OBJ(bot-1));
-    ADD(args, INTEGER_OBJ(left));
-    ADD(args, INTEGER_OBJ(right-1));
+    Array args = data->call_buf;
+    ADD_C(args, INTEGER_OBJ(top));
+    ADD_C(args, INTEGER_OBJ(bot - 1));
+    ADD_C(args, INTEGER_OBJ(left));
+    ADD_C(args, INTEGER_OBJ(right - 1));
     push_call(ui, "set_scroll_region", args);
 
-    args = (Array)ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(rows));
+    args = data->call_buf;
+    ADD_C(args, INTEGER_OBJ(rows));
     push_call(ui, "scroll", args);
 
     // some clients have "clear" being affected by scroll region,
     // so reset it.
-    args = (Array)ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(0));
-    ADD(args, INTEGER_OBJ(ui->height-1));
-    ADD(args, INTEGER_OBJ(0));
-    ADD(args, INTEGER_OBJ(ui->width-1));
+    args = data->call_buf;
+    ADD_C(args, INTEGER_OBJ(0));
+    ADD_C(args, INTEGER_OBJ(ui->height - 1));
+    ADD_C(args, INTEGER_OBJ(0));
+    ADD_C(args, INTEGER_OBJ(ui->width - 1));
     push_call(ui, "set_scroll_region", args);
   }
 }
@@ -473,26 +713,27 @@ static void remote_ui_default_colors_set(UI *ui, Integer rgb_fg, Integer rgb_bg,
   if (!ui->ui_ext[kUITermColors]) {
     HL_SET_DEFAULT_COLORS(rgb_fg, rgb_bg, rgb_sp);
   }
-  Array args = ARRAY_DICT_INIT;
-  ADD(args, INTEGER_OBJ(rgb_fg));
-  ADD(args, INTEGER_OBJ(rgb_bg));
-  ADD(args, INTEGER_OBJ(rgb_sp));
-  ADD(args, INTEGER_OBJ(cterm_fg));
-  ADD(args, INTEGER_OBJ(cterm_bg));
+  UIData *data = ui->data;
+  Array args = data->call_buf;
+  ADD_C(args, INTEGER_OBJ(rgb_fg));
+  ADD_C(args, INTEGER_OBJ(rgb_bg));
+  ADD_C(args, INTEGER_OBJ(rgb_sp));
+  ADD_C(args, INTEGER_OBJ(cterm_fg));
+  ADD_C(args, INTEGER_OBJ(cterm_bg));
   push_call(ui, "default_colors_set", args);
 
   // Deprecated
   if (!ui->ui_ext[kUILinegrid]) {
-    args = (Array)ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(ui->rgb ? rgb_fg : cterm_fg - 1));
+    args = data->call_buf;
+    ADD_C(args, INTEGER_OBJ(ui->rgb ? rgb_fg : cterm_fg - 1));
     push_call(ui, "update_fg", args);
 
-    args = (Array)ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(ui->rgb ? rgb_bg : cterm_bg - 1));
+    args = data->call_buf;
+    ADD_C(args, INTEGER_OBJ(ui->rgb ? rgb_bg : cterm_bg - 1));
     push_call(ui, "update_bg", args);
 
-    args = (Array)ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(ui->rgb ? rgb_sp : -1));
+    args = data->call_buf;
+    ADD_C(args, INTEGER_OBJ(ui->rgb ? rgb_sp : -1));
     push_call(ui, "update_sp", args);
   }
 }
@@ -503,26 +744,29 @@ static void remote_ui_hl_attr_define(UI *ui, Integer id, HlAttrs rgb_attrs, HlAt
   if (!ui->ui_ext[kUILinegrid]) {
     return;
   }
-  Array args = ARRAY_DICT_INIT;
 
-  ADD(args, INTEGER_OBJ(id));
-  ADD(args, DICTIONARY_OBJ(hlattrs2dict(rgb_attrs, true)));
-  ADD(args, DICTIONARY_OBJ(hlattrs2dict(cterm_attrs, false)));
+  UIData *data = ui->data;
+  Array args = data->call_buf;
+  ADD_C(args, INTEGER_OBJ(id));
+  ADD_C(args, DICTIONARY_OBJ(hlattrs2dict(rgb_attrs, true)));
+  ADD_C(args, DICTIONARY_OBJ(hlattrs2dict(cterm_attrs, false)));
 
   if (ui->ui_ext[kUIHlState]) {
-    ADD(args, ARRAY_OBJ(copy_array(info)));
+    ADD_C(args, ARRAY_OBJ(info));
   } else {
-    ADD(args, ARRAY_OBJ((Array)ARRAY_DICT_INIT));
+    ADD_C(args, ARRAY_OBJ((Array)ARRAY_DICT_INIT));
   }
 
   push_call(ui, "hl_attr_define", args);
+  // TODO(bfredl): could be elided
+  api_free_dictionary(kv_A(args, 1).data.dictionary);
+  api_free_dictionary(kv_A(args, 2).data.dictionary);
 }
 
 static void remote_ui_highlight_set(UI *ui, int id)
 {
-  Array args = ARRAY_DICT_INIT;
   UIData *data = ui->data;
-
+  Array args = data->call_buf;
 
   if (data->hl_id == id) {
     return;
@@ -530,18 +774,20 @@ static void remote_ui_highlight_set(UI *ui, int id)
   data->hl_id = id;
   Dictionary hl = hlattrs2dict(syn_attr2entry(id), ui->rgb);
 
-  ADD(args, DICTIONARY_OBJ(hl));
+  ADD_C(args, DICTIONARY_OBJ(hl));
   push_call(ui, "highlight_set", args);
+  api_free_dictionary(kv_A(args, 0).data.dictionary);
 }
 
 /// "true" cursor used only for input focus
 static void remote_ui_grid_cursor_goto(UI *ui, Integer grid, Integer row, Integer col)
 {
   if (ui->ui_ext[kUILinegrid]) {
-    Array args = ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(grid));
-    ADD(args, INTEGER_OBJ(row));
-    ADD(args, INTEGER_OBJ(col));
+    UIData *data = ui->data;
+    Array args = data->call_buf;
+    ADD_C(args, INTEGER_OBJ(grid));
+    ADD_C(args, INTEGER_OBJ(row));
+    ADD_C(args, INTEGER_OBJ(col));
     push_call(ui, "grid_cursor_goto", args);
   } else {
     UIData *data = ui->data;
@@ -560,9 +806,9 @@ static void remote_ui_cursor_goto(UI *ui, Integer row, Integer col)
   }
   data->client_row = row;
   data->client_col = col;
-  Array args = ARRAY_DICT_INIT;
-  ADD(args, INTEGER_OBJ(row));
-  ADD(args, INTEGER_OBJ(col));
+  Array args = data->call_buf;
+  ADD_C(args, INTEGER_OBJ(row));
+  ADD_C(args, INTEGER_OBJ(col));
   push_call(ui, "cursor_goto", args);
 }
 
@@ -570,8 +816,8 @@ static void remote_ui_put(UI *ui, const char *cell)
 {
   UIData *data = ui->data;
   data->client_col++;
-  Array args = ARRAY_DICT_INIT;
-  ADD(args, STRING_OBJ(cstr_to_string(cell)));
+  Array args = data->call_buf;
+  ADD_C(args, STRING_OBJ(cstr_as_string((char *)cell)));
   push_call(ui, "put", args);
 }
 
@@ -581,47 +827,70 @@ static void remote_ui_raw_line(UI *ui, Integer grid, Integer row, Integer startc
 {
   UIData *data = ui->data;
   if (ui->ui_ext[kUILinegrid]) {
-    Array args = ARRAY_DICT_INIT;
-    ADD(args, INTEGER_OBJ(grid));
-    ADD(args, INTEGER_OBJ(row));
-    ADD(args, INTEGER_OBJ(startcol));
-    Array cells = ARRAY_DICT_INIT;
-    int repeat = 0;
-    size_t ncells = (size_t)(endcol-startcol);
+    prepare_call(ui, "grid_line");
+    data->ncalls++;
+
+    char **buf = &data->buf_wptr;
+    mpack_array(buf, 4);
+    mpack_uint(buf, (uint32_t)grid);
+    mpack_uint(buf, (uint32_t)row);
+    mpack_uint(buf, (uint32_t)startcol);
+    char *lenpos = mpack_array_dyn16(buf);
+
+    uint32_t repeat = 0;
+    size_t ncells = (size_t)(endcol - startcol);
     int last_hl = -1;
+    uint32_t nelem = 0;
     for (size_t i = 0; i < ncells; i++) {
       repeat++;
-      if (i == ncells-1 || attrs[i] != attrs[i+1]
-          || STRCMP(chunk[i], chunk[i+1])) {
-        Array cell = ARRAY_DICT_INIT;
-        ADD(cell, STRING_OBJ(cstr_to_string((const char *)chunk[i])));
-        if (attrs[i] != last_hl || repeat > 1) {
-          ADD(cell, INTEGER_OBJ(attrs[i]));
-          last_hl = attrs[i];
+      if (i == ncells - 1 || attrs[i] != attrs[i + 1]
+          || STRCMP(chunk[i], chunk[i + 1])) {
+        if (UI_BUF_SIZE - BUF_POS(data) < 2 * (1 + 2 + sizeof(schar_T) + 5 + 5)) {
+          // close to overflowing the redraw buffer. finish this event,
+          // flush, and start a new "grid_line" event at the current position.
+          // For simplicity leave place for the final "clear" element
+          // as well, hence the factor of 2 in the check.
+          mpack_w2(&lenpos, nelem);
+          remote_ui_flush_buf(ui);
+
+          prepare_call(ui, "grid_line");
+          data->ncalls++;
+          mpack_array(buf, 4);
+          mpack_uint(buf, (uint32_t)grid);
+          mpack_uint(buf, (uint32_t)row);
+          mpack_uint(buf, (uint32_t)startcol + (uint32_t)i - repeat + 1);
+          lenpos = mpack_array_dyn16(buf);
+          nelem = 0;
+          last_hl = -1;
         }
-        if (repeat > 1) {
-          ADD(cell, INTEGER_OBJ(repeat));
+        uint32_t csize = (repeat > 1) ? 3 : ((attrs[i] != last_hl) ? 2 : 1);
+        nelem++;
+        mpack_array(buf, csize);
+        mpack_str(buf, (const char *)chunk[i]);
+        if (csize >= 2) {
+          mpack_uint(buf, (uint32_t)attrs[i]);
+          if (csize >= 3) {
+            mpack_uint(buf, repeat);
+          }
         }
-        ADD(cells, ARRAY_OBJ(cell));
+        last_hl = attrs[i];
         repeat = 0;
       }
     }
     if (endcol < clearcol) {
-      Array cell = ARRAY_DICT_INIT;
-      ADD(cell, STRING_OBJ(cstr_to_string(" ")));
-      ADD(cell, INTEGER_OBJ(clearattr));
-      ADD(cell, INTEGER_OBJ(clearcol-endcol));
-      ADD(cells, ARRAY_OBJ(cell));
+      nelem++;
+      mpack_array(buf, 3);
+      mpack_str(buf, " ");
+      mpack_uint(buf, (uint32_t)clearattr);
+      mpack_uint(buf, (uint32_t)(clearcol - endcol));
     }
-    ADD(args, ARRAY_OBJ(cells));
-
-    push_call(ui, "grid_line", args);
+    mpack_w2(&lenpos, nelem);
   } else {
-    for (int i = 0; i < endcol-startcol; i++) {
-      remote_ui_cursor_goto(ui, row, startcol+i);
+    for (int i = 0; i < endcol - startcol; i++) {
+      remote_ui_cursor_goto(ui, row, startcol + i);
       remote_ui_highlight_set(ui, attrs[i]);
       remote_ui_put(ui, (const char *)chunk[i]);
-      if (utf_ambiguous_width(utf_ptr2char(chunk[i]))) {
+      if (utf_ambiguous_width(utf_ptr2char((char *)chunk[i]))) {
         data->client_col = -1;  // force cursor update
       }
     }
@@ -642,16 +911,47 @@ static void remote_ui_raw_line(UI *ui, Integer grid, Integer row, Integer startc
   }
 }
 
+/// Flush the internal packing buffer to the client.
+///
+/// This might happen multiple times before the actual ui_flush, if the
+/// total redraw size is large!
+static void remote_ui_flush_buf(UI *ui)
+{
+  UIData *data = ui->data;
+  if (!data->nevents_pos) {
+    return;
+  }
+  if (data->cur_event) {
+    flush_event(data);
+  }
+  mpack_w2(&data->nevents_pos, data->nevents);
+  data->nevents = 0;
+  data->nevents_pos = NULL;
+
+  // TODO(bfredl): elide copy by a length one free-list like the arena
+  size_t size = BUF_POS(data);
+  WBuffer *buf = wstream_new_buffer(xmemdup(data->buf, size), size, 1, xfree);
+  rpc_write_raw(data->channel_id, buf);
+  data->buf_wptr = data->buf;
+  // we have sent events to the client, but possibly not yet the final "flush"
+  // event.
+  data->flushed_events = true;
+}
+
+/// An intentional flush (vsync) when Nvim is finished redrawing the screen
+///
+/// Clients can know this happened by a final "flush" event at the end of the
+/// "redraw" batch.
 static void remote_ui_flush(UI *ui)
 {
   UIData *data = ui->data;
-  if (data->buffer.size > 0) {
+  if (data->nevents > 0 || data->flushed_events) {
     if (!ui->ui_ext[kUILinegrid]) {
       remote_ui_cursor_goto(ui, data->cursor_row, data->cursor_col);
     }
     push_call(ui, "flush", (Array)ARRAY_DICT_INIT);
-    rpc_send_event(data->channel_id, "redraw", data->buffer);
-    data->buffer = (Array)ARRAY_DICT_INIT;
+    remote_ui_flush_buf(ui);
+    data->flushed_events = false;
   }
 }
 
@@ -686,7 +986,7 @@ static Array translate_firstarg(UI *ui, Array args)
   return new_args;
 }
 
-static void remote_ui_event(UI *ui, char *name, Array args, bool *args_consumed)
+static void remote_ui_event(UI *ui, char *name, Array args)
 {
   UIData *data = ui->data;
   if (!ui->ui_ext[kUILinegrid]) {
@@ -695,21 +995,24 @@ static void remote_ui_event(UI *ui, char *name, Array args, bool *args_consumed)
     if (strequal(name, "cmdline_show")) {
       Array new_args = translate_firstarg(ui, args);
       push_call(ui, name, new_args);
+      api_free_array(new_args);
       return;
     } else if (strequal(name, "cmdline_block_show")) {
-      Array new_args = ARRAY_DICT_INIT;
+      Array new_args = data->call_buf;
       Array block = args.items[0].data.array;
       Array new_block = ARRAY_DICT_INIT;
       for (size_t i = 0; i < block.size; i++) {
         ADD(new_block,
             ARRAY_OBJ(translate_contents(ui, block.items[i].data.array)));
       }
-      ADD(new_args, ARRAY_OBJ(new_block));
+      ADD_C(new_args, ARRAY_OBJ(new_block));
       push_call(ui, name, new_args);
+      api_free_array(new_block);
       return;
     } else if (strequal(name, "cmdline_block_append")) {
       Array new_args = translate_firstarg(ui, args);
       push_call(ui, name, new_args);
+      api_free_array(new_args);
       return;
     }
   }
@@ -720,18 +1023,19 @@ static void remote_ui_event(UI *ui, char *name, Array args, bool *args_consumed)
       data->wildmenu_active = (args.items[4].data.integer == -1)
                               || !ui->ui_ext[kUIPopupmenu];
       if (data->wildmenu_active) {
-        Array new_args = ARRAY_DICT_INIT;
+        Array new_args = data->call_buf;
         Array items = args.items[0].data.array;
         Array new_items = ARRAY_DICT_INIT;
         for (size_t i = 0; i < items.size; i++) {
           ADD(new_items, copy_object(items.items[i].data.array.items[0]));
         }
-        ADD(new_args, ARRAY_OBJ(new_items));
+        ADD_C(new_args, ARRAY_OBJ(new_items));
         push_call(ui, "wildmenu_show", new_args);
+        api_free_array(new_items);
         if (args.items[1].data.integer != -1) {
-          Array new_args2 = ARRAY_DICT_INIT;
-          ADD(new_args2, args.items[1]);
-          push_call(ui, "wildmenu_select", new_args);
+          Array new_args2 = data->call_buf;
+          ADD_C(new_args2, args.items[1]);
+          push_call(ui, "wildmenu_select", new_args2);
         }
         return;
       }
@@ -746,19 +1050,7 @@ static void remote_ui_event(UI *ui, char *name, Array args, bool *args_consumed)
     }
   }
 
-
-  Array my_args = ARRAY_DICT_INIT;
-  // Objects are currently single-reference
-  // make a copy, but only if necessary
-  if (*args_consumed) {
-    for (size_t i = 0; i < args.size; i++) {
-      ADD(my_args, copy_object(args.items[i]));
-    }
-  } else {
-    my_args = args;
-    *args_consumed = true;
-  }
-  push_call(ui, name, my_args);
+  push_call(ui, name, args);
 }
 
 static void remote_ui_inspect(UI *ui, Dictionary *info)