feat(ui): allow to set the highlight namespace per window

- reimplement 'winhl' in terms of highlight namespaces
- check for EOF in screen tests (to indicate a likely crash)

30 files changed, 591 insertions, 239 deletions

diff --git a/runtime/doc/api.txt b/runtime/doc/api.txt
index 12edaa1f08..a388592981 100644
--- a/runtime/doc/api.txt
+++ b/runtime/doc/api.txt
@@ -603,20 +603,6 @@ nvim__inspect_cell({grid}, {row}, {col})                *nvim__inspect_cell()*
     NB: if your UI doesn't use hlstate, this will not return hlstate first
     time.
 
-nvim__set_hl_ns({ns_id})                                   *nvim__set_hl_ns()*
-    Set active namespace for highlights.
-
-    NB: this function can be called from async contexts, but the semantics are
-    not yet well-defined. To start with |nvim_set_decoration_provider| on_win
-    and on_line callbacks are explicitly allowed to change the namespace
-    during a redraw cycle.
-
-    Attributes: ~
-        |api-fast|
-
-    Parameters: ~
-        {ns_id}  the namespace to activate
-
 nvim__stats()                                                  *nvim__stats()*
     Gets internal stats.
 
@@ -1414,6 +1400,26 @@ nvim_set_hl({ns_id}, {name}, {*val})                           *nvim_set_hl()*
                    set, cterm attributes will match those from the attribute
                    map documented above.
 
+nvim_set_hl_ns({ns_id})                                     *nvim_set_hl_ns()*
+    Set active namespace for highlights. This can be set for a single window,
+    see |nvim_win_set_hl_ns|.
+
+    Parameters: ~
+        {ns_id}  the namespace to use
+
+nvim_set_hl_ns_fast({ns_id})                           *nvim_set_hl_ns_fast()*
+    Set active namespace for highlights while redrawing.
+
+    This function meant to be called while redrawing, primarily from
+    |nvim_set_decoration_provider| on_win and on_line callbacks, which are
+    allowed to change the namespace during a redraw cycle.
+
+    Attributes: ~
+        |api-fast|
+
+    Parameters: ~
+        {ns_id}  the namespace to activate
+
 nvim_set_keymap({mode}, {lhs}, {rhs}, {*opts})             *nvim_set_keymap()*
     Sets a global |mapping| for the given mode.
 
@@ -2875,6 +2881,15 @@ nvim_win_set_height({window}, {height})                *nvim_win_set_height()*
         {window}  Window handle, or 0 for current window
         {height}  Height as a count of rows
 
+nvim_win_set_hl_ns({window}, {ns_id})                   *nvim_win_set_hl_ns()*
+    Set highlight namespace for a window. This will use highlights defined in
+    this namespace, but fall back to global highlights (ns=0) when missing.
+
+    This takes predecence over the 'winhighlight' option.
+
+    Parameters: ~
+        {ns_id}  the namespace to use
+
 nvim_win_set_var({window}, {name}, {value})               *nvim_win_set_var()*
     Sets a window-scoped (w:) variable
 
diff --git a/runtime/doc/options.txt b/runtime/doc/options.txt
index 349c442c9f..a1f2eac5ed 100644
--- a/runtime/doc/options.txt
+++ b/runtime/doc/options.txt
@@ -7096,10 +7096,12 @@ A jump table for the options with a short description can be found at |Q_op|.
 'winhighlight' 'winhl'	string (default empty)
 			local to window
 	Window-local highlights.  Comma-delimited list of highlight
-	|group-name| pairs "{hl-builtin}:{hl},..." where each {hl-builtin} is
-	a built-in |highlight-groups| item to be overridden by {hl} group in
-	the window.  Only built-in |highlight-groups| are supported, not
-	syntax highlighting (use |:ownsyntax| for that).
+	|group-name| pairs "{hl-from}:{hl-to},..." where each {hl-from} is
+	a |highlight-groups| item to be overridden by {hl-to} group in
+	the window.
+
+	Note: highlight namespaces take precedence over 'winhighlight'.
+	See |nvim_win_set_hl_ns| and |nvim_set_hl|.
 
 	Highlights of vertical separators are determined by the window to the
 	left of the separator.  The 'tabline' highlight of a tabpage is
diff --git a/src/nvim/api/extmark.c b/src/nvim/api/extmark.c
index c36f127507..b8295601c1 100644
--- a/src/nvim/api/extmark.c
+++ b/src/nvim/api/extmark.c
@@ -1031,6 +1031,8 @@ void nvim_set_decoration_provider(Integer ns_id, DictionaryOf(LuaRef) opts, Erro
   }
 
   p->active = true;
+  p->hl_valid++;
+  p->hl_cached = false;
   return;
 error:
   decor_provider_clear(p);
diff --git a/src/nvim/api/keysets.lua b/src/nvim/api/keysets.lua
index 4f4ac40ce9..6fad52ba75 100644
--- a/src/nvim/api/keysets.lua
+++ b/src/nvim/api/keysets.lua
@@ -104,7 +104,6 @@ return {
     "reverse";
     "nocombine";
     "default";
-    "global";
     "cterm";
     "foreground"; "fg";
     "background"; "bg";
@@ -112,9 +111,9 @@ return {
     "ctermbg";
     "special"; "sp";
     "link";
+    "global_link";
     "fallback";
     "blend";
-    "temp";
   };
   highlight_cterm = {
     "bold";
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 3f9730c569..25f6e822b3 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -93,7 +93,6 @@ Dictionary nvim_get_hl_by_name(String name, Boolean rgb, Error *err)
 }
 
 /// Gets a highlight definition by id. |hlID()|
-///
 /// @param hl_id Highlight id as returned by |hlID()|
 /// @param rgb Export RGB colors
 /// @param[out] err Error details, if any
@@ -182,35 +181,38 @@ void nvim_set_hl(Integer ns_id, String name, Dict(highlight) *val, Error *err)
   }
 }
 
-/// Set active namespace for highlights.
-///
-/// NB: this function can be called from async contexts, but the
-/// semantics are not yet well-defined. To start with
-/// |nvim_set_decoration_provider| on_win and on_line callbacks
-/// are explicitly allowed to change the namespace during a redraw cycle.
+/// Set active namespace for highlights. This can be set for a single window,
+/// see |nvim_win_set_hl_ns|.
 ///
-/// @param ns_id the namespace to activate
+/// @param ns_id the namespace to use
 /// @param[out] err Error details, if any
-void nvim__set_hl_ns(Integer ns_id, Error *err)
-  FUNC_API_FAST
+void nvim_set_hl_ns(Integer ns_id, Error *err)
+  FUNC_API_SINCE(10)
 {
-  if (ns_id >= 0) {
-    ns_hl_active = (NS)ns_id;
+  if (ns_id < 0) {
+    api_set_error(err, kErrorTypeValidation, "no such namespace");
+    return;
   }
 
-  // TODO(bfredl): this is a little bit hackish.  Eventually we want a standard
-  // event path for redraws caused by "fast" events. This could tie in with
-  // better throttling of async events causing redraws, such as non-batched
-  // nvim_buf_set_extmark calls from async contexts.
-  if (!provider_active && !ns_hl_changed && must_redraw < NOT_VALID) {
-    multiqueue_put(main_loop.events, on_redraw_event, 0);
-  }
-  ns_hl_changed = true;
+  ns_hl_global = (NS)ns_id;
+  hl_check_ns();
+  redraw_all_later(NOT_VALID);
 }
 
-static void on_redraw_event(void **argv)
+/// Set active namespace for highlights while redrawing.
+///
+/// This function meant to be called while redrawing, primarily from
+/// |nvim_set_decoration_provider| on_win and on_line callbacks, which
+/// are allowed to change the namespace during a redraw cycle.
+///
+/// @param ns_id the namespace to activate
+/// @param[out] err Error details, if any
+void nvim_set_hl_ns_fast(Integer ns_id, Error *err)
+  FUNC_API_SINCE(10)
+  FUNC_API_FAST
 {
-  redraw_all_later(NOT_VALID);
+  ns_hl_fast = (NS)ns_id;
+  hl_check_ns();
 }
 
 /// Sends input-keys to Nvim, subject to various quirks controlled by `mode`
@@ -480,7 +482,7 @@ Object nvim_notify(String msg, Integer log_level, Dictionary opts, Error *err)
   ADD_C(args, INTEGER_OBJ(log_level));
   ADD_C(args, DICTIONARY_OBJ(opts));
 
-  return nlua_exec(STATIC_CSTR_AS_STRING("return vim.notify(...)"), args, err);
+  return NLUA_EXEC_STATIC("return vim.notify(...)", args, err);
 }
 
 /// Calculates the number of display cells occupied by `text`.
@@ -1835,11 +1837,9 @@ Array nvim_get_proc_children(Integer pid, Error *err)
   if (rv == 2) {
     // syscall failed (possibly because of kernel options), try shelling out.
     DLOG("fallback to vim._os_proc_children()");
-    Array a = ARRAY_DICT_INIT;
+    MAXSIZE_TEMP_ARRAY(a, 1);
     ADD(a, INTEGER_OBJ(pid));
-    String s = STATIC_CSTR_AS_STRING("return vim._os_proc_children(...)");
-    Object o = nlua_exec(s, a, err);
-    api_free_array(a);
+    Object o = NLUA_EXEC_STATIC("return vim._os_proc_children(...)", a, err);
     if (o.type == kObjectTypeArray) {
       rvobj = o.data.array;
     } else if (!ERROR_SET(err)) {
@@ -1880,12 +1880,9 @@ Object nvim_get_proc(Integer pid, Error *err)
   }
 #else
   // Cross-platform process info APIs are miserable, so use `ps` instead.
-  Array a = ARRAY_DICT_INIT;
+  MAXSIZE_TEMP_ARRAY(a, 1);
   ADD(a, INTEGER_OBJ(pid));
-  String s = cstr_to_string("return vim._os_proc_info(select(1, ...))");
-  Object o = nlua_exec(s, a, err);
-  api_free_string(s);
-  api_free_array(a);
+  Object o = NLUA_EXEC_STATIC("return vim._os_proc_info(...)", a, err);
   if (o.type == kObjectTypeArray && o.data.array.size == 0) {
     return NIL;  // Process not found.
   } else if (o.type == kObjectTypeDictionary) {
diff --git a/src/nvim/api/window.c b/src/nvim/api/window.c
index 5a4ff70257..d0a2ff766e 100644
--- a/src/nvim/api/window.c
+++ b/src/nvim/api/window.c
@@ -426,3 +426,28 @@ Object nvim_win_call(Window window, LuaRef fun, Error *err)
   try_end(err);
   return res;
 }
+
+/// Set highlight namespace for a window. This will use highlights defined in
+/// this namespace, but fall back to global highlights (ns=0) when missing.
+///
+/// This takes predecence over the 'winhighlight' option.
+///
+/// @param ns_id the namespace to use
+/// @param[out] err Error details, if any
+void nvim_win_set_hl_ns(Window window, Integer ns_id, Error *err)
+  FUNC_API_SINCE(10)
+{
+  win_T *win = find_window_by_handle(window, err);
+  if (!win) {
+    return;
+  }
+
+  // -1 is allowed as inherit global namespace
+  if (ns_id < -1) {
+    api_set_error(err, kErrorTypeValidation, "no such namespace");
+  }
+
+  win->w_ns_hl = (NS)ns_id;
+  win->w_hl_needs_update = true;
+  redraw_later(win, NOT_VALID);
+}
diff --git a/src/nvim/buffer_defs.h b/src/nvim/buffer_defs.h
index f5dd86cb98..15b964f5ed 100644
--- a/src/nvim/buffer_defs.h
+++ b/src/nvim/buffer_defs.h
@@ -1143,11 +1143,14 @@ struct window_S {
 
   synblock_T *w_s;                 ///< for :ownsyntax
 
+  int w_ns_hl;
+  int w_ns_hl_winhl;
+  int w_ns_hl_active;
+  int *w_ns_hl_attr;
+
   int w_hl_id_normal;               ///< 'winhighlight' normal id
   int w_hl_attr_normal;             ///< 'winhighlight' normal final attrs
-
-  int w_hl_ids[HLF_COUNT];          ///< 'winhighlight' id
-  int w_hl_attrs[HLF_COUNT];        ///< 'winhighlight' final attrs
+  int w_hl_attr_normalnc;           ///< 'winhighlight' NormalNC final attrs
 
   int w_hl_needs_update;            ///< attrs need to be recalculated
 
@@ -1469,11 +1472,6 @@ struct window_S {
   size_t w_winbar_click_defs_size;
 };
 
-static inline int win_hl_attr(win_T *wp, int hlf)
-{
-  return wp->w_hl_attrs[hlf];
-}
-
 /// Macros defined in Vim, but not in Neovim
 #define CHANGEDTICK(buf) \
   (=== Include buffer.h & use buf_(get|set|inc) _changedtick ===)
diff --git a/src/nvim/decoration_provider.c b/src/nvim/decoration_provider.c
index 04d875c4e3..95e13b4240 100644
--- a/src/nvim/decoration_provider.c
+++ b/src/nvim/decoration_provider.c
@@ -14,7 +14,7 @@ static kvec_t(DecorProvider) decor_providers = KV_INITIAL_VALUE;
 #define DECORATION_PROVIDER_INIT(ns_id) (DecorProvider) \
   { ns_id, false, LUA_NOREF, LUA_NOREF, \
     LUA_NOREF, LUA_NOREF, LUA_NOREF, \
-    LUA_NOREF, -1 }
+    LUA_NOREF, -1, false }
 
 static bool decor_provider_invoke(NS ns_id, const char *name, LuaRef ref, Array args,
                                   bool default_true, char **perr)
@@ -107,8 +107,6 @@ void decor_providers_invoke_win(win_T *wp, DecorProviders *providers,
       }
     }
   }
-
-  win_check_ns_hl(wp);
 }
 
 /// For each provider invoke the 'line' callback for a given window row.
@@ -135,7 +133,7 @@ void providers_invoke_line(win_T *wp, DecorProviders *providers, int row, bool *
         kv_A(*providers, k) = NULL;
       }
 
-      win_check_ns_hl(wp);
+      hl_check_ns();
     }
   }
 }
diff --git a/src/nvim/decoration_provider.h b/src/nvim/decoration_provider.h
index 3ec7c80357..dd1ed6c581 100644
--- a/src/nvim/decoration_provider.h
+++ b/src/nvim/decoration_provider.h
@@ -13,6 +13,7 @@ typedef struct {
   LuaRef redraw_end;
   LuaRef hl_def;
   int hl_valid;
+  bool hl_cached;
 } DecorProvider;
 
 typedef kvec_withinit_t(DecorProvider *, 4) DecorProviders;
diff --git a/src/nvim/globals.h b/src/nvim/globals.h
index a8e15dfdd9..954b62883e 100644
--- a/src/nvim/globals.h
+++ b/src/nvim/globals.h
@@ -96,9 +96,6 @@ EXTERN struct nvim_stats_s {
 EXTERN int Rows INIT(= DFLT_ROWS);     // nr of rows in the screen
 EXTERN int Columns INIT(= DFLT_COLS);  // nr of columns in the screen
 
-EXTERN NS ns_hl_active INIT(= 0);         // current ns that defines highlights
-EXTERN bool ns_hl_changed INIT(= false);  // highlight need update
-
 // We use 64-bit file functions here, if available.  E.g. ftello() returns
 // off_t instead of long, which helps if long is 32 bit and off_t is 64 bit.
 // We assume that when fseeko() is available then ftello() is too.
diff --git a/src/nvim/hardcopy.c b/src/nvim/hardcopy.c
index fa357f864e..d811a78ca8 100644
--- a/src/nvim/hardcopy.c
+++ b/src/nvim/hardcopy.c
@@ -900,7 +900,7 @@ static colnr_T hardcopy_line(prt_settings_T *psettings, int page_line, prt_pos_T
     }
     // syntax highlighting stuff.
     if (psettings->do_syntax) {
-      id = syn_get_id(curwin, ppos->file_line, col, 1, NULL, FALSE);
+      id = syn_get_id(curwin, ppos->file_line, col, 1, NULL, false);
       if (id > 0) {
         id = syn_get_final_id(id);
       } else {
diff --git a/src/nvim/highlight.c b/src/nvim/highlight.c
index dfd077840e..504cae9a51 100644
--- a/src/nvim/highlight.c
+++ b/src/nvim/highlight.c
@@ -32,7 +32,9 @@ static Map(int, int) blend_attr_entries = MAP_INIT;
 static Map(int, int) blendthrough_attr_entries = MAP_INIT;
 
 /// highlight entries private to a namespace
-static Map(ColorKey, ColorItem) ns_hl;
+static Map(ColorKey, ColorItem) ns_hls;
+typedef int NSHlAttr[HLF_COUNT + 1];
+static PMap(handle_T) ns_hl_attr;
 
 void highlight_init(void)
 {
@@ -147,42 +149,46 @@ int hl_get_syn_attr(int ns_id, int idx, HlAttrs at_en)
 
 void ns_hl_def(NS ns_id, int hl_id, HlAttrs attrs, int link_id, Dict(highlight) *dict)
 {
-  if ((attrs.rgb_ae_attr & HL_DEFAULT)
-      && map_has(ColorKey, ColorItem)(&ns_hl, ColorKey(ns_id, hl_id))) {
-    return;
-  }
   if (ns_id == 0) {
     assert(dict);
     // set in global (':highlight') namespace
     set_hl_group(hl_id, attrs, dict, link_id);
     return;
   }
+  if ((attrs.rgb_ae_attr & HL_DEFAULT)
+      && map_has(ColorKey, ColorItem)(&ns_hls, ColorKey(ns_id, hl_id))) {
+    return;
+  }
   DecorProvider *p = get_decor_provider(ns_id, true);
   int attr_id = link_id > 0 ? -1 : hl_get_syn_attr(ns_id, hl_id, attrs);
   ColorItem it = { .attr_id = attr_id,
                    .link_id = link_id,
                    .version = p->hl_valid,
-                   .is_default = (attrs.rgb_ae_attr & HL_DEFAULT) };
-  map_put(ColorKey, ColorItem)(&ns_hl, ColorKey(ns_id, hl_id), it);
+                   .is_default = (attrs.rgb_ae_attr & HL_DEFAULT),
+                   .link_global = (attrs.rgb_ae_attr & HL_GLOBAL) };
+  map_put(ColorKey, ColorItem)(&ns_hls, ColorKey(ns_id, hl_id), it);
+  p->hl_cached = false;
 }
 
-int ns_get_hl(NS ns_id, int hl_id, bool link, bool nodefault)
+int ns_get_hl(NS *ns_hl, int hl_id, bool link, bool nodefault)
 {
   static int recursive = 0;
 
-  if (ns_id < 0) {
+  if (*ns_hl < 0) {
     if (ns_hl_active <= 0) {
       return -1;
     }
-    ns_id = ns_hl_active;
+    *ns_hl = ns_hl_active;
   }
 
+  int ns_id = *ns_hl;
+
   DecorProvider *p = get_decor_provider(ns_id, true);
-  ColorItem it = map_get(ColorKey, ColorItem)(&ns_hl, ColorKey(ns_id, hl_id));
+  ColorItem it = map_get(ColorKey, ColorItem)(&ns_hls, ColorKey(ns_id, hl_id));
   // TODO(bfredl): map_ref true even this?
-  bool valid_cache = it.version >= p->hl_valid;
+  bool valid_item = it.version >= p->hl_valid;
 
-  if (!valid_cache && p->hl_def != LUA_NOREF && !recursive) {
+  if (!valid_item && p->hl_def != LUA_NOREF && !recursive) {
     MAXSIZE_TEMP_ARRAY(args, 3);
     ADD_C(args, INTEGER_OBJ((Integer)ns_id));
     ADD_C(args, STRING_OBJ(cstr_to_string((char *)syn_id2name(hl_id))));
@@ -215,44 +221,76 @@ int ns_get_hl(NS ns_id, int hl_id, bool link, bool nodefault)
     it.attr_id = fallback ? -1 : hl_get_syn_attr((int)ns_id, hl_id, attrs);
     it.version = p->hl_valid - tmp;
     it.is_default = attrs.rgb_ae_attr & HL_DEFAULT;
-    map_put(ColorKey, ColorItem)(&ns_hl, ColorKey(ns_id, hl_id), it);
+    it.link_global = attrs.rgb_ae_attr & HL_GLOBAL;
+    map_put(ColorKey, ColorItem)(&ns_hls, ColorKey(ns_id, hl_id), it);
+    valid_item = true;
   }
 
-  if (it.is_default && nodefault) {
+  if ((it.is_default && nodefault) || !valid_item) {
     return -1;
   }
 
   if (link) {
-    return it.attr_id >= 0 ? 0 : it.link_id;
+    if (it.attr_id >= 0) {
+      return 0;
+    } else {
+      if (it.link_global) {
+        *ns_hl = 0;
+      }
+      return it.link_id;
+    }
   } else {
     return it.attr_id;
   }
 }
 
-bool win_check_ns_hl(win_T *wp)
+bool hl_check_ns(void)
 {
-  if (ns_hl_changed) {
-    highlight_changed();
-    if (wp) {
-      update_window_hl(wp, true);
+  int ns = 0;
+  if (ns_hl_fast > 0) {
+    ns = ns_hl_fast;
+  } else if (ns_hl_win >= 0) {
+    ns = ns_hl_win;
+  } else {
+    ns = ns_hl_global;
+  }
+  if (ns_hl_active == ns) {
+    return false;
+  }
+
+  ns_hl_active = ns;
+  hl_attr_active = highlight_attr;
+  if (ns > 0) {
+    update_ns_hl(ns);
+    NSHlAttr *hl_def = (NSHlAttr *)pmap_get(handle_T)(&ns_hl_attr, ns);
+    if (hl_def) {
+      hl_attr_active = *hl_def;
     }
-    ns_hl_changed = false;
-    return true;
   }
-  return false;
+  need_highlight_changed = true;
+  return true;
+}
+
+/// prepare for drawing window `wp` or global elements if NULL
+///
+/// Note: pum should be drawn in the context of the current window!
+bool win_check_ns_hl(win_T *wp)
+{
+  ns_hl_win = wp ? wp->w_ns_hl : -1;
+  return hl_check_ns();
 }
 
 /// Get attribute code for a builtin highlight group.
 ///
 /// The final syntax group could be modified by hi-link or 'winhighlight'.
-int hl_get_ui_attr(int idx, int final_id, bool optional)
+int hl_get_ui_attr(int ns_id, int idx, int final_id, bool optional)
 {
   HlAttrs attrs = HLATTRS_INIT;
   bool available = false;
 
   if (final_id > 0) {
-    int syn_attr = syn_id2attr(final_id);
-    if (syn_attr != 0) {
+    int syn_attr = syn_ns_id2attr(ns_id, final_id, optional);
+    if (syn_attr > 0) {
       attrs = syn_attr2entry(syn_attr);
       available = true;
     }
@@ -265,7 +303,7 @@ int hl_get_ui_attr(int idx, int final_id, bool optional)
     if (pum_drawn()) {
       must_redraw_pum = true;
     }
-  } else if (idx == HLF_MSG) {
+  } else if (idx == HLF_MSG && ns_id == -1) {
     msg_grid.blending = attrs.hl_blend > -1;
   }
 
@@ -278,6 +316,21 @@ int hl_get_ui_attr(int idx, int final_id, bool optional)
 
 void update_window_hl(win_T *wp, bool invalid)
 {
+  int ns_id = wp->w_ns_hl;
+
+  update_ns_hl(ns_id);
+  if (ns_id != wp->w_ns_hl_active) {
+    wp->w_ns_hl_active = ns_id;
+
+    wp->w_ns_hl_attr = *(NSHlAttr *)pmap_get(handle_T)(&ns_hl_attr, ns_id);
+    if (!wp->w_ns_hl_attr) {
+      // No specific highlights, use the defaults.
+      wp->w_ns_hl_attr = highlight_attr;
+    }
+  }
+
+  int *hl_def = wp->w_ns_hl_attr;
+
   if (!wp->w_hl_needs_update && !invalid) {
     return;
   }
@@ -285,34 +338,17 @@ void update_window_hl(win_T *wp, bool invalid)
 
   // If a floating window is blending it always have a named
   // wp->w_hl_attr_normal group. HL_ATTR(HLF_NFLOAT) is always named.
-  bool has_blend = wp->w_floating && wp->w_p_winbl != 0;
 
   // determine window specific background set in 'winhighlight'
   bool float_win = wp->w_floating && !wp->w_float_config.external;
-  if (wp != curwin && wp->w_hl_ids[HLF_INACTIVE] != 0) {
-    wp->w_hl_attr_normal = hl_get_ui_attr(HLF_INACTIVE,
-                                          wp->w_hl_ids[HLF_INACTIVE],
-                                          !has_blend);
-  } else if (float_win && wp->w_hl_ids[HLF_NFLOAT] != 0) {
-    wp->w_hl_attr_normal = hl_get_ui_attr(HLF_NFLOAT,
-                                          wp->w_hl_ids[HLF_NFLOAT], !has_blend);
-  } else if (wp->w_hl_id_normal != 0) {
-    wp->w_hl_attr_normal = hl_get_ui_attr(-1, wp->w_hl_id_normal, !has_blend);
+  if (float_win && hl_def[HLF_NFLOAT] != 0) {
+    wp->w_hl_attr_normal = hl_def[HLF_NFLOAT];
+  } else if (hl_def[HLF_COUNT] > 0) {
+    wp->w_hl_attr_normal = hl_def[HLF_COUNT];
   } else {
     wp->w_hl_attr_normal = float_win ? HL_ATTR(HLF_NFLOAT) : 0;
   }
 
-  // NOOOO! You cannot just pretend that "Normal" is just like any other
-  // syntax group! It needs at least 10 layers of special casing! Noooooo!
-  //
-  // haha, theme engine go brrr
-  int normality = syn_check_group(S_LEN("Normal"));
-  int ns_attr = ns_get_hl(-1, normality, false, false);
-  if (ns_attr > 0) {
-    // TODO(bfredl): hantera NormalNC and so on
-    wp->w_hl_attr_normal = ns_attr;
-  }
-
   // if blend= attribute is not set, 'winblend' value overrides it.
   if (wp->w_floating && wp->w_p_winbl > 0) {
     HlEntry entry = kv_A(attr_entries, wp->w_hl_attr_normal);
@@ -322,28 +358,13 @@ void update_window_hl(win_T *wp, bool invalid)
     }
   }
 
-  if (wp != curwin && wp->w_hl_ids[HLF_INACTIVE] == 0) {
-    wp->w_hl_attr_normal = hl_combine_attr(HL_ATTR(HLF_INACTIVE),
-                                           wp->w_hl_attr_normal);
-  }
-
-  for (int hlf = 0; hlf < HLF_COUNT; hlf++) {
-    int attr;
-    if (wp->w_hl_ids[hlf] != 0) {
-      attr = hl_get_ui_attr(hlf, wp->w_hl_ids[hlf], false);
-    } else {
-      attr = HL_ATTR(hlf);
-    }
-    wp->w_hl_attrs[hlf] = attr;
-  }
-
   wp->w_float_config.shadow = false;
   if (wp->w_floating && wp->w_float_config.border) {
     for (int i = 0; i < 8; i++) {
-      int attr = wp->w_hl_attrs[HLF_BORDER];
+      int attr = hl_def[HLF_BORDER];
       if (wp->w_float_config.border_hl_ids[i]) {
-        attr = hl_get_ui_attr(HLF_BORDER, wp->w_float_config.border_hl_ids[i],
-                              false);
+        attr = hl_get_ui_attr(ns_id, HLF_BORDER,
+                              wp->w_float_config.border_hl_ids[i], false);
         HlAttrs a = syn_attr2entry(attr);
         if (a.hl_blend) {
           wp->w_float_config.shadow = true;
@@ -355,6 +376,65 @@ void update_window_hl(win_T *wp, bool invalid)
 
   // shadow might cause blending
   check_blending(wp);
+
+  // TODO(bfredl): this a bit ad-hoc. move it from highlight ns logic to 'winhl'
+  // implementation?
+  if (hl_def[HLF_INACTIVE] == 0) {
+    wp->w_hl_attr_normalnc = hl_combine_attr(HL_ATTR(HLF_INACTIVE),
+                                             wp->w_hl_attr_normal);
+  } else {
+    wp->w_hl_attr_normalnc = hl_def[HLF_INACTIVE];
+  }
+}
+
+void update_ns_hl(int ns_id)
+{
+  if (ns_id <= 0) {
+    return;
+  }
+  DecorProvider *p = get_decor_provider(ns_id, true);
+  if (p->hl_cached) {
+    return;
+  }
+
+  NSHlAttr **alloc = (NSHlAttr **)pmap_ref(handle_T)(&ns_hl_attr, ns_id, true);
+  if (*alloc == NULL) {
+    *alloc = xmalloc(sizeof(**alloc));
+  }
+  int *hl_attrs = **alloc;
+
+  for (int hlf = 0; hlf < (int)HLF_COUNT; hlf++) {
+    int id = syn_check_group(hlf_names[hlf], STRLEN(hlf_names[hlf]));
+    bool optional = (hlf == HLF_INACTIVE || hlf == HLF_NFLOAT);
+    hl_attrs[hlf] = hl_get_ui_attr(ns_id, hlf, id, optional);
+  }
+
+  // NOOOO! You cannot just pretend that "Normal" is just like any other
+  // syntax group! It needs at least 10 layers of special casing! Noooooo!
+  //
+  // haha, tema engine go brrr
+  int normality = syn_check_group(S_LEN("Normal"));
+  hl_attrs[HLF_COUNT] = hl_get_ui_attr(ns_id, -1, normality, true);
+
+  // hl_get_ui_attr might have invalidated the decor provider
+  p = get_decor_provider(ns_id, true);
+  p->hl_cached = true;
+}
+
+int win_bg_attr(win_T *wp)
+{
+  if (ns_hl_fast < 0) {
+    int local = (wp == curwin) ? wp->w_hl_attr_normal : wp->w_hl_attr_normalnc;
+    if (local) {
+      return local;
+    }
+  }
+
+  if (wp == curwin || hl_attr_active[HLF_INACTIVE] == 0) {
+    return hl_attr_active[HLF_COUNT];
+  } else {
+    return hl_attr_active[HLF_INACTIVE];
+  }
 }
 
 /// Gets HL_UNDERLINE highlight.
@@ -403,7 +483,7 @@ void clear_hl_tables(bool reinit)
     map_destroy(int, int)(&combine_attr_entries);
     map_destroy(int, int)(&blend_attr_entries);
     map_destroy(int, int)(&blendthrough_attr_entries);
-    map_destroy(ColorKey, ColorItem)(&ns_hl);
+    map_destroy(ColorKey, ColorItem)(&ns_hls);
   }
 }
 
@@ -852,7 +932,6 @@ HlAttrs dict2hlattrs(Dict(highlight) *dict, bool use_rgb, int *link_id, Error *e
   CHECK_FLAG(dict, mask, strikethrough, , HL_STRIKETHROUGH);
   CHECK_FLAG(dict, mask, nocombine, , HL_NOCOMBINE);
   CHECK_FLAG(dict, mask, default, _, HL_DEFAULT);
-  CHECK_FLAG(dict, mask, global, , HL_GLOBAL);
 
   if (HAS_KEY(dict->fg)) {
     fg = object_to_color(dict->fg, "fg", true, err);
@@ -895,14 +974,21 @@ HlAttrs dict2hlattrs(Dict(highlight) *dict, bool use_rgb, int *link_id, Error *e
     return hlattrs;
   }
 
-  if (HAS_KEY(dict->link)) {
+  if (HAS_KEY(dict->link) || HAS_KEY(dict->global_link)) {
     if (link_id) {
-      *link_id = object_to_hl_id(dict->link, "link", err);
+      if (HAS_KEY(dict->global_link)) {
+        *link_id = object_to_hl_id(dict->global_link, "link", err);
+        mask |= HL_GLOBAL;
+      } else {
+        *link_id = object_to_hl_id(dict->link, "link", err);
+      }
+
       if (ERROR_SET(err)) {
         return hlattrs;
       }
     } else {
-      api_set_error(err, kErrorTypeValidation, "Invalid Key: 'link'");
+      api_set_error(err, kErrorTypeValidation, "Invalid Key: '%s'",
+                    HAS_KEY(dict->global_link) ? "global_link" : "link");
     }
   }
 
diff --git a/src/nvim/highlight.h b/src/nvim/highlight.h
index ae63f83d65..50299bb91c 100644
--- a/src/nvim/highlight.h
+++ b/src/nvim/highlight.h
@@ -4,6 +4,7 @@
 #include <stdbool.h>
 
 #include "nvim/api/private/defs.h"
+#include "nvim/buffer_defs.h"
 #include "nvim/highlight_defs.h"
 #include "nvim/ui.h"
 
@@ -11,6 +12,13 @@
 # include "highlight.h.generated.h"
 #endif
 
+static inline int win_hl_attr(win_T *wp, int hlf)
+{
+  // wp->w_ns_hl_attr might be null if we check highlights
+  // prior to entering redraw
+  return ((wp->w_ns_hl_attr && ns_hl_fast < 0) ? wp->w_ns_hl_attr : hl_attr_active)[hlf];
+}
+
 #define HL_SET_DEFAULT_COLORS(rgb_fg, rgb_bg, rgb_sp) \
   do { \
     bool dark_ = (*p_bg == 'd'); \
diff --git a/src/nvim/highlight_defs.h b/src/nvim/highlight_defs.h
index da6d5284c2..ffcb0f3f22 100644
--- a/src/nvim/highlight_defs.h
+++ b/src/nvim/highlight_defs.h
@@ -180,7 +180,7 @@ EXTERN const char *hlf_names[] INIT(= {
   [HLF_CU] = "Cursor",
 });
 
-EXTERN int highlight_attr[HLF_COUNT];       // Highl. attr for each context.
+EXTERN int highlight_attr[HLF_COUNT + 1];     // Highl. attr for each context.
 EXTERN int highlight_attr_last[HLF_COUNT];  // copy for detecting changed groups
 EXTERN int highlight_user[9];                   // User[1-9] attributes
 EXTERN int highlight_stlnc[9];                  // On top of user
@@ -190,6 +190,13 @@ EXTERN RgbValue normal_fg INIT(= -1);
 EXTERN RgbValue normal_bg INIT(= -1);
 EXTERN RgbValue normal_sp INIT(= -1);
 
+EXTERN NS ns_hl_global INIT(= 0);  // global highlight namespace
+EXTERN NS ns_hl_win INIT(= -1);    // highlight namespace for the current window
+EXTERN NS ns_hl_fast INIT(= -1);   // highlight namespace specified in a fast callback
+EXTERN NS ns_hl_active INIT(= 0);  // currently active/cached namespace
+
+EXTERN int *hl_attr_active INIT(= highlight_attr);
+
 typedef enum {
   kHlUnknown,
   kHlUI,
@@ -219,9 +226,10 @@ typedef struct {
   int link_id;
   int version;
   bool is_default;
+  bool link_global;
 } ColorItem;
-#define COLOR_ITEM_INITIALIZER { .attr_id = -1, .link_id = -1, \
-                                 .version = -1, .is_default = false }
+#define COLOR_ITEM_INITIALIZER { .attr_id = -1, .link_id = -1, .version = -1, \
+                                 .is_default = false, .link_global = false }
 
 /// highlight attributes with associated priorities
 typedef struct {
diff --git a/src/nvim/highlight_group.c b/src/nvim/highlight_group.c
index d79283d3e3..616c2a670d 100644
--- a/src/nvim/highlight_group.c
+++ b/src/nvim/highlight_group.c
@@ -1790,11 +1790,18 @@ static int syn_add_group(const char *name, size_t len)
 /// @see syn_attr2entry
 int syn_id2attr(int hl_id)
 {
-  hl_id = syn_get_final_id(hl_id);
+  return syn_ns_id2attr(-1, hl_id, false);
+}
+
+int syn_ns_id2attr(int ns_id, int hl_id, bool optional)
+{
+  hl_id = syn_ns_get_final_id(&ns_id, hl_id);
   HlGroup *sgp = &hl_table[hl_id - 1];  // index is ID minus one
 
-  int attr = ns_get_hl(-1, hl_id, false, sgp->sg_set);
-  if (attr >= 0) {
+  int attr = ns_get_hl(&ns_id, hl_id, false, sgp->sg_set);
+
+  // if a highlight group is optional, don't use the global value
+  if (attr >= 0 || (optional && ns_id > 0)) {
     return attr;
   }
   return sgp->sg_attr;
@@ -1803,10 +1810,16 @@ int syn_id2attr(int hl_id)
 /// Translate a group ID to the final group ID (following links).
 int syn_get_final_id(int hl_id)
 {
+  int id = curwin->w_ns_hl_active;
+  return syn_ns_get_final_id(&id, hl_id);
+}
+
+int syn_ns_get_final_id(int *ns_id, int hl_id)
+{
   int count;
 
   if (hl_id > highlight_ga.ga_len || hl_id < 1) {
-    return 0;                           // Can be called from eval!!
+    return 0;  // Can be called from eval!!
   }
 
   // Follow links until there is no more.
@@ -1814,10 +1827,10 @@ int syn_get_final_id(int hl_id)
   for (count = 100; --count >= 0;) {
     HlGroup *sgp = &hl_table[hl_id - 1];  // index is ID minus one
 
-    // ACHTUNG: when using "tmp" attribute (no link) the function might be
+    // TODO(bfredl): when using "tmp" attribute (no link) the function might be
     // called twice. it needs be smart enough to remember attr only to
     // syn_id2attr time
-    int check = ns_get_hl(-1, hl_id, true, sgp->sg_set);
+    int check = ns_get_hl(ns_id, hl_id, true, sgp->sg_set);
     if (check == 0) {
       return hl_id;  // how dare! it broke the link!
     } else if (check > 0) {
@@ -1915,14 +1928,15 @@ void highlight_changed(void)
     if (id == 0) {
       abort();
     }
-    int final_id = syn_get_final_id(id);
+    int ns_id = -1;
+    int final_id = syn_ns_get_final_id(&ns_id, id);
     if (hlf == HLF_SNC) {
       id_SNC = final_id;
     } else if (hlf == HLF_S) {
       id_S = final_id;
     }
 
-    highlight_attr[hlf] = hl_get_ui_attr(hlf, final_id,
+    highlight_attr[hlf] = hl_get_ui_attr(ns_id, hlf, final_id,
                                          (hlf == HLF_INACTIVE || hlf == HLF_LC));
 
     if (highlight_attr[hlf] != highlight_attr_last[hlf]) {
@@ -1935,6 +1949,9 @@ void highlight_changed(void)
     }
   }
 
+  // sentinel value. used when no hightlight namespace is active
+  highlight_attr[HLF_COUNT] = 0;
+
   //
   // Setup the user highlights
   //
diff --git a/src/nvim/lua/executor.h b/src/nvim/lua/executor.h
index 2afbbebfe7..78346fd81f 100644
--- a/src/nvim/lua/executor.h
+++ b/src/nvim/lua/executor.h
@@ -24,6 +24,8 @@ typedef struct {
 #endif
 } nlua_ref_state_t;
 
+#define NLUA_EXEC_STATIC(cstr, arg, err) nlua_exec(STATIC_CSTR_AS_STRING(cstr), arg, err)
+
 #define NLUA_CLEAR_REF(x) \
   do { \
     /* Take the address to avoid double evaluation. #1375 */ \
diff --git a/src/nvim/match.c b/src/nvim/match.c
index f085d7cdb4..6cfc7d1f24 100644
--- a/src/nvim/match.c
+++ b/src/nvim/match.c
@@ -11,6 +11,7 @@
 #include "nvim/eval/funcs.h"
 #include "nvim/ex_docmd.h"
 #include "nvim/fold.h"
+#include "nvim/highlight.h"
 #include "nvim/highlight_group.h"
 #include "nvim/match.h"
 #include "nvim/memline.h"
@@ -696,7 +697,7 @@ int update_search_hl(win_T *wp, linenr_T lnum, colnr_T col, char_u **line, match
         }
         // Highlight the match were the cursor is using the CurSearch
         // group.
-        if (shl == search_hl && shl->has_cursor && (HL_ATTR(HLF_LC) || wp->w_hl_ids[HLF_LC])) {
+        if (shl == search_hl && shl->has_cursor && (HL_ATTR(HLF_LC) || win_hl_attr(wp, HLF_LC))) {
           shl->attr_cur = win_hl_attr(wp, HLF_LC) ? win_hl_attr(wp, HLF_LC) : HL_ATTR(HLF_LC);
         } else {
           shl->attr_cur = shl->attr;
diff --git a/src/nvim/move.c b/src/nvim/move.c
index 66bb9ef834..564a9ca5df 100644
--- a/src/nvim/move.c
+++ b/src/nvim/move.c
@@ -25,6 +25,7 @@
 #include "nvim/fold.h"
 #include "nvim/getchar.h"
 #include "nvim/grid.h"
+#include "nvim/highlight.h"
 #include "nvim/mbyte.h"
 #include "nvim/memline.h"
 #include "nvim/move.h"
@@ -115,7 +116,7 @@ static void redraw_for_cursorcolumn(win_T *wp)
   FUNC_ATTR_NONNULL_ALL
 {
   if ((wp->w_valid & VALID_VIRTCOL) == 0 && !pum_visible()) {
-    if (wp->w_p_cuc || ((HL_ATTR(HLF_LC) || wp->w_hl_ids[HLF_LC]) && using_hlsearch())) {
+    if (wp->w_p_cuc || ((HL_ATTR(HLF_LC) || win_hl_attr(wp, HLF_LC)) && using_hlsearch())) {
       // When 'cursorcolumn' is set or "CurSearch" is in use
       // need to redraw with SOME_VALID.
       redraw_later(wp, SOME_VALID);
diff --git a/src/nvim/normal.c b/src/nvim/normal.c
index 38baa77eec..4e608ca25b 100644
--- a/src/nvim/normal.c
+++ b/src/nvim/normal.c
@@ -1282,7 +1282,7 @@ static void normal_redraw(NormalState *s)
 
   if (VIsual_active) {
     redraw_curbuf_later(INVERTED);  // update inverted part
-    update_screen(INVERTED);
+    update_screen(0);
   } else if (must_redraw) {
     update_screen(0);
   } else if (redraw_cmdline || clear_cmdline || redraw_mode) {
@@ -1838,7 +1838,8 @@ bool do_mouse(oparg_T *oap, int c, int dir, long count, bool fixindent)
       }
       if (jump_flags) {
         jump_flags = jump_to_mouse(jump_flags, NULL, which_button);
-        update_curbuf(VIsual_active ? INVERTED : VALID);
+        redraw_curbuf_later(VIsual_active ? INVERTED : VALID);
+        update_screen(0);
         setcursor();
         ui_flush();  // Update before showing popup menu
       }
diff --git a/src/nvim/option.c b/src/nvim/option.c
index af86958702..82dac029f4 100644
--- a/src/nvim/option.c
+++ b/src/nvim/option.c
@@ -33,6 +33,7 @@
 #include "nvim/charset.h"
 #include "nvim/cursor.h"
 #include "nvim/cursor_shape.h"
+#include "nvim/decoration_provider.h"
 #include "nvim/diff.h"
 #include "nvim/digraph.h"
 #include "nvim/edit.h"
@@ -84,7 +85,9 @@
 #ifdef WIN32
 # include "nvim/os/pty_conpty_win.h"
 #endif
+#include "nvim/api/extmark.h"
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/vim.h"
 #include "nvim/lua/executor.h"
 #include "nvim/os/input.h"
 #include "nvim/os/lang.h"
@@ -3945,13 +3948,29 @@ static char *compile_cap_prog(synblock_T *synblock)
 }
 
 /// Handle setting `winhighlight' in window "wp"
-static bool parse_winhl_opt(win_T *wp)
+bool parse_winhl_opt(win_T *wp)
 {
-  int w_hl_id_normal = 0;
-  int w_hl_ids[HLF_COUNT] = { 0 };
-  int hlf;
-
   const char *p = (const char *)wp->w_p_winhl;
+
+  if (!*p) {
+    if (wp->w_ns_hl_winhl && wp->w_ns_hl == wp->w_ns_hl_winhl) {
+      wp->w_ns_hl = 0;
+      wp->w_hl_needs_update = true;
+    }
+
+    return true;
+  }
+
+  if (wp->w_ns_hl_winhl == 0) {
+    wp->w_ns_hl_winhl = (int)nvim_create_namespace(NULL_STRING);
+  } else {
+    // namespace already exist. invalidate existing items
+    DecorProvider *dp = get_decor_provider(wp->w_ns_hl_winhl, true);
+    dp->hl_valid++;
+  }
+  wp->w_ns_hl = wp->w_ns_hl_winhl;
+  int ns_hl = wp->w_ns_hl;
+
   while (*p) {
     char *colon = strchr(p, ':');
     if (!colon) {
@@ -3962,27 +3981,15 @@ static bool parse_winhl_opt(win_T *wp)
     char *commap = xstrchrnul(hi, ',');
     size_t len = (size_t)(commap - hi);
     int hl_id = len ? syn_check_group(hi, len) : -1;
+    int hl_id_link = nlen ? syn_check_group(p, nlen) : 0;
 
-    if (strncmp("Normal", p, nlen) == 0) {
-      w_hl_id_normal = hl_id;
-    } else {
-      for (hlf = 0; hlf < HLF_COUNT; hlf++) {
-        if (strlen(hlf_names[hlf]) == nlen
-            && strncmp(hlf_names[hlf], p, nlen) == 0) {
-          w_hl_ids[hlf] = hl_id;
-          break;
-        }
-      }
-      if (hlf == HLF_COUNT) {
-        return false;
-      }
-    }
+    HlAttrs attrs = HLATTRS_INIT;
+    attrs.rgb_ae_attr |= HL_GLOBAL;
+    ns_hl_def(ns_hl, hl_id_link, attrs, hl_id, NULL);
 
     p = *commap ? commap + 1 : "";
   }
 
-  wp->w_hl_id_normal = w_hl_id_normal;
-  memcpy(wp->w_hl_ids, w_hl_ids, sizeof(w_hl_ids));
   wp->w_hl_needs_update = true;
   return true;
 }
diff --git a/src/nvim/popupmnu.c b/src/nvim/popupmnu.c
index f8828b42ed..c137675502 100644
--- a/src/nvim/popupmnu.c
+++ b/src/nvim/popupmnu.c
@@ -17,6 +17,7 @@
 #include "nvim/eval/typval.h"
 #include "nvim/ex_cmds.h"
 #include "nvim/grid.h"
+#include "nvim/highlight.h"
 #include "nvim/insexpand.h"
 #include "nvim/memline.h"
 #include "nvim/memory.h"
diff --git a/src/nvim/screen.c b/src/nvim/screen.c
index bc26efdae2..f5158256e9 100644
--- a/src/nvim/screen.c
+++ b/src/nvim/screen.c
@@ -293,13 +293,6 @@ void redraw_win_signcol(win_T *wp)
   }
 }
 
-/// Update all windows that are editing the current buffer.
-void update_curbuf(int type)
-{
-  redraw_curbuf_later(type);
-  update_screen(type);
-}
-
 /// Redraw the parts of the screen that is marked for redraw.
 ///
 /// Most code shouldn't call this directly, rather use redraw_later() and
@@ -470,6 +463,8 @@ int update_screen(int type)
 
   ui_comp_set_screen_valid(true);
 
+  ns_hl_fast = -1;
+
   DecorProviders providers;
   decor_providers_start(&providers, type, &provider_err);
 
@@ -541,8 +536,7 @@ int update_screen(int type)
     }
   }
 
-  // Go from top to bottom through the windows, redrawing the ones that need
-  // it.
+  // Go from top to bottom through the windows, redrawing the ones that need it.
   bool did_one = false;
   search_hl.rm.regprog = NULL;
 
@@ -552,6 +546,8 @@ int update_screen(int type)
       wp->w_redr_type = NOT_VALID;
     }
 
+    win_check_ns_hl(wp);
+
     // reallocate grid if needed.
     win_grid_alloc(wp);
 
@@ -578,9 +574,12 @@ int update_screen(int type)
 
   // May need to redraw the popup menu.
   if (pum_drawn() && must_redraw_pum) {
+    win_check_ns_hl(curwin);
     pum_redraw();
   }
 
+  win_check_ns_hl(NULL);
+
   // Reset b_mod_set flags.  Going through all windows is probably faster
   // than going through all buffers (there could be many buffers).
   FOR_ALL_WINDOWS_IN_TAB(wp, curtab) {
@@ -1286,6 +1285,8 @@ win_update_start:
 
   bool cursorline_standout = win_cursorline_standout(wp);
 
+  win_check_ns_hl(wp);
+
   for (;;) {
     // stop updating when reached the end of the window (check for _past_
     // the end of the window is at the end of the loop)
@@ -1578,8 +1579,7 @@ win_update_start:
   wp->w_empty_rows = 0;
   wp->w_filler_rows = 0;
   if (!eof && !didline) {
-    int at_attr = hl_combine_attr(wp->w_hl_attr_normal,
-                                  win_hl_attr(wp, HLF_AT));
+    int at_attr = hl_combine_attr(win_bg_attr(wp), win_hl_attr(wp, HLF_AT));
     if (lnum == wp->w_topline) {
       // Single line that does not fit!
       // Don't overwrite it, it can be edited.
@@ -1751,7 +1751,7 @@ static void win_draw_end(win_T *wp, int c1, int c2, bool draw_margin, int row, i
     }
   }
 
-  int attr = hl_combine_attr(wp->w_hl_attr_normal, win_hl_attr(wp, (int)hl));
+  int attr = hl_combine_attr(win_bg_attr(wp), win_hl_attr(wp, (int)hl));
 
   if (wp->w_p_rl) {
     grid_fill(&wp->w_grid, row, endrow, wp->w_wincol, W_ENDCOL(wp) - 1 - n,
@@ -2322,6 +2322,8 @@ static int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool noc
     }
   }
 
+  int bg_attr = win_bg_attr(wp);
+
   filler_lines = diff_check(wp, lnum);
   if (filler_lines < 0) {
     if (filler_lines == -1) {
@@ -2842,8 +2844,7 @@ static int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool noc
          || (number_only && draw_state > WL_NR))
         && filler_todo <= 0) {
       draw_virt_text(wp, buf, win_col_offset, &col, grid->cols, row);
-      grid_put_linebuf(grid, row, 0, col, -grid->cols, wp->w_p_rl, wp,
-                       wp->w_hl_attr_normal, false);
+      grid_put_linebuf(grid, row, 0, col, -grid->cols, wp->w_p_rl, wp, bg_attr, false);
       // Pretend we have finished updating the window.  Except when
       // 'cursorcolumn' is set.
       if (wp->w_p_cuc) {
@@ -3924,8 +3925,7 @@ static int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool noc
       }
 
       draw_virt_text(wp, buf, win_col_offset, &col, grid->cols, row);
-      grid_put_linebuf(grid, row, 0, col, grid->cols, wp->w_p_rl, wp,
-                       wp->w_hl_attr_normal, false);
+      grid_put_linebuf(grid, row, 0, col, grid->cols, wp->w_p_rl, wp, bg_attr, false);
       row++;
 
       // Update w_cline_height and w_cline_folded if the cursor line was
@@ -4163,8 +4163,7 @@ static int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool noc
         draw_virt_text(wp, buf, win_col_offset, &draw_col, grid->cols, row);
       }
 
-      grid_put_linebuf(grid, row, 0, draw_col, grid->cols, wp->w_p_rl,
-                       wp, wp->w_hl_attr_normal, wrap);
+      grid_put_linebuf(grid, row, 0, draw_col, grid->cols, wp->w_p_rl, wp, bg_attr, wrap);
       if (wrap) {
         ScreenGrid *current_grid = grid;
         int current_row = row, dummy_col = 0;  // dummy_col unused
diff --git a/src/nvim/vim.h b/src/nvim/vim.h
index b2c7df4cbb..09b949bb20 100644
--- a/src/nvim/vim.h
+++ b/src/nvim/vim.h
@@ -275,8 +275,8 @@ enum { FOLD_TEXT_LEN = 51, };  //!< buffer size for get_foldtext()
                                           (const char *)(y), \
                                           (size_t)(n))
 
-// Enums need a typecast to be used as array index (for Ultrix).
-#define HL_ATTR(n)      highlight_attr[(int)(n)]
+// Enums need a typecast to be used as array index.
+#define HL_ATTR(n)      hl_attr_active[(int)(n)]
 
 /// Maximum number of bytes in a multi-byte character.  It can be one 32-bit
 /// character of up to 6 bytes, or one 16-bit character of up to three bytes
diff --git a/src/nvim/window.c b/src/nvim/window.c
index d39a21eb42..6c9466f711 100644
--- a/src/nvim/window.c
+++ b/src/nvim/window.c
@@ -6,6 +6,7 @@
 #include <stdbool.h>
 
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/vim.h"
 #include "nvim/arglist.h"
 #include "nvim/ascii.h"
 #include "nvim/buffer.h"
@@ -28,6 +29,7 @@
 #include "nvim/globals.h"
 #include "nvim/grid.h"
 #include "nvim/hashtab.h"
+#include "nvim/highlight.h"
 #include "nvim/main.h"
 #include "nvim/mapping.h"
 #include "nvim/mark.h"
@@ -700,7 +702,7 @@ win_T *win_new_float(win_T *wp, bool last, FloatConfig fconfig, Error *err)
     win_remove(wp, NULL);
     win_append(lastwin_nofloating(), wp);
   }
-  wp->w_floating = 1;
+  wp->w_floating = true;
   wp->w_status_height = 0;
   wp->w_winbar_height = 0;
   wp->w_hsep_height = 0;
@@ -730,13 +732,15 @@ void win_set_minimal_style(win_T *wp)
                    : concat_str(old, (char_u *)",eob: "));
     free_string_option(old);
   }
-  if (wp->w_hl_ids[HLF_EOB] != -1) {
-    char_u *old = wp->w_p_winhl;
-    wp->w_p_winhl = ((*old == NUL)
-                     ? (char_u *)xstrdup("EndOfBuffer:")
-                     : concat_str(old, (char_u *)",EndOfBuffer:"));
-    free_string_option(old);
-  }
+
+  // TODO(bfredl): this could use a highlight namespace directly,
+  // and avoid pecularities around window options
+  char_u *old = wp->w_p_winhl;
+  wp->w_p_winhl = ((*old == NUL)
+                   ? (char_u *)xstrdup("EndOfBuffer:")
+                   : concat_str(old, (char_u *)",EndOfBuffer:"));
+  free_string_option(old);
+  parse_winhl_opt(wp);
 
   // signcolumn: use 'auto'
   if (wp->w_p_scl[0] != 'a' || STRLEN(wp->w_p_scl) >= 8) {
@@ -4865,10 +4869,17 @@ static void win_enter_ext(win_T *const wp, const int flags)
     redraw_later(curwin, VALID);  // causes status line redraw
   }
 
-  if (HL_ATTR(HLF_INACTIVE)
-      || (prevwin && prevwin->w_hl_ids[HLF_INACTIVE])
-      || curwin->w_hl_ids[HLF_INACTIVE]) {
-    redraw_all_later(NOT_VALID);
+  // change background color according to NormalNC,
+  // but only if actually defined (otherwise no extra redraw)
+  if (curwin->w_hl_attr_normal != curwin->w_hl_attr_normalnc) {
+    // TODO(bfredl): eventually we should be smart enough
+    // to only recompose the window, not redraw it.
+    redraw_later(curwin, NOT_VALID);
+  }
+  if (prevwin) {
+    if (prevwin->w_hl_attr_normal != prevwin->w_hl_attr_normalnc) {
+      redraw_later(prevwin, NOT_VALID);
+    }
   }
 
   // set window height to desired minimal value
@@ -5037,6 +5048,8 @@ static win_T *win_alloc(win_T *after, bool hidden)
   new_wp->w_float_config = FLOAT_CONFIG_INIT;
   new_wp->w_viewport_invalid = true;
 
+  new_wp->w_ns_hl = -1;
+
   // use global option for global-local options
   new_wp->w_p_so = -1;
   new_wp->w_p_siso = -1;
diff --git a/test/functional/api/highlight_spec.lua b/test/functional/api/highlight_spec.lua
index c4197f0b3e..2730f7e23d 100644
--- a/test/functional/api/highlight_spec.lua
+++ b/test/functional/api/highlight_spec.lua
@@ -243,7 +243,7 @@ describe("API: set highlight", function()
 
   local function get_ns()
     local ns = meths.create_namespace('Test_set_hl')
-    meths._set_hl_ns(ns)
+    meths.set_hl_ns(ns)
     return ns
   end
 
diff --git a/test/functional/helpers.lua b/test/functional/helpers.lua
index 8c5a60657a..981cfc306e 100644
--- a/test/functional/helpers.lua
+++ b/test/functional/helpers.lua
@@ -244,10 +244,12 @@ function module.run_session(lsession, request_cb, notification_cb, setup_cb, tim
     last_error = nil
     error(err)
   end
+
+  return session.eof_err
 end
 
 function module.run(request_cb, notification_cb, setup_cb, timeout)
-  module.run_session(session, request_cb, notification_cb, setup_cb, timeout)
+  return module.run_session(session, request_cb, notification_cb, setup_cb, timeout)
 end
 
 function module.stop()
diff --git a/test/functional/ui/decorations_spec.lua b/test/functional/ui/decorations_spec.lua
index 68eebaf3a2..789f1c6487 100644
--- a/test/functional/ui/decorations_spec.lua
+++ b/test/functional/ui/decorations_spec.lua
@@ -193,7 +193,7 @@ describe('decorations providers', function()
                                               |
     ]]}
 
-    meths._set_hl_ns(ns1)
+    meths.set_hl_ns(ns1)
     screen:expect{grid=[[
       {10:  1 }{11:// just to see if there was an accid}|
       {10:    }{11:ent}                                 |
@@ -219,7 +219,7 @@ describe('decorations providers', function()
       local ns2 = a.nvim_create_namespace 'ns2'
       a.nvim_set_decoration_provider (ns2, {
         on_win = function (_, win, buf)
-          a.nvim__set_hl_ns(win == thewin and _G.ns1 or ns2)
+          a.nvim_set_hl_ns_fast(win == thewin and _G.ns1 or ns2)
         end;
       })
     ]]
@@ -266,7 +266,7 @@ describe('decorations providers', function()
     ]]}
 
     meths.set_hl(ns1, 'LinkGroup', {fg = 'Blue'})
-    meths._set_hl_ns(ns1)
+    meths.set_hl_ns(ns1)
 
     screen:expect{grid=[[
       // just to see if there was an accident |
@@ -302,7 +302,7 @@ describe('decorations providers', function()
     ]]}
 
     meths.set_hl(ns1, 'LinkGroup', {fg = 'Blue', default=true})
-    meths._set_hl_ns(ns1)
+    meths.set_hl_ns(ns1)
     feed 'k'
 
     screen:expect{grid=[[
diff --git a/test/functional/ui/highlight_spec.lua b/test/functional/ui/highlight_spec.lua
index e065a727f3..01e57e8ae0 100644
--- a/test/functional/ui/highlight_spec.lua
+++ b/test/functional/ui/highlight_spec.lua
@@ -3,9 +3,10 @@ local Screen = require('test.functional.ui.screen')
 local os = require('os')
 local clear, feed, insert = helpers.clear, helpers.feed, helpers.insert
 local command, exec = helpers.command, helpers.exec
-local eval, exc_exec = helpers.eval, helpers.exc_exec
+local eval = helpers.eval
 local feed_command, eq = helpers.feed_command, helpers.eq
 local curbufmeths = helpers.curbufmeths
+local meths = helpers.meths
 
 describe('colorscheme compatibility', function()
   before_each(function()
@@ -1787,6 +1788,7 @@ describe("'winhighlight' highlight", function()
       [26] = {background = Screen.colors.Red},
       [27] = {background = Screen.colors.DarkBlue, bold = true, foreground = Screen.colors.Green1},
       [28] = {bold = true, foreground = Screen.colors.Brown},
+      [29] = {foreground = Screen.colors.Blue1, background = Screen.colors.Red, bold = true};
     })
     command("hi Background1 guibg=DarkBlue")
     command("hi Background2 guibg=DarkGreen")
@@ -1820,7 +1822,7 @@ describe("'winhighlight' highlight", function()
     ]])
   end)
 
-  it('handles invalid values', function()
+  it('handles undefined groups', function()
     command("set winhl=Normal:Background1")
     screen:expect([[
       {1:^                    }|
@@ -1833,19 +1835,44 @@ describe("'winhighlight' highlight", function()
                           |
     ]])
 
-    eq('Vim(set):E474: Invalid argument: winhl=xxx:yyy',
-       exc_exec("set winhl=xxx:yyy"))
-    eq('Normal:Background1', eval('&winhl'))
+    command("set winhl=xxx:yyy")
+    eq('xxx:yyy', eval('&winhl'))
     screen:expect{grid=[[
-      {1:^                    }|
-      {2:~                   }|
-      {2:~                   }|
-      {2:~                   }|
-      {2:~                   }|
-      {2:~                   }|
-      {2:~                   }|
+      ^                    |
+      {0:~                   }|
+      {0:~                   }|
+      {0:~                   }|
+      {0:~                   }|
+      {0:~                   }|
+      {0:~                   }|
                           |
-    ]], unchanged=true}
+    ]]}
+  end)
+
+  it('can be changed to define different groups', function()
+    command("set winhl=EndOfBuffer:Background1")
+    screen:expect{grid=[[
+      ^                    |
+      {1:~                   }|
+      {1:~                   }|
+      {1:~                   }|
+      {1:~                   }|
+      {1:~                   }|
+      {1:~                   }|
+                          |
+    ]]}
+
+    command("set winhl=Normal:ErrorMsg")
+    screen:expect{grid=[[
+      {15:^                    }|
+      {29:~                   }|
+      {29:~                   }|
+      {29:~                   }|
+      {29:~                   }|
+      {29:~                   }|
+      {29:~                   }|
+                          |
+    ]]}
   end)
 
   it('works local to the window', function()
@@ -2270,4 +2297,145 @@ describe("'winhighlight' highlight", function()
                           |
     ]])
   end)
+
+
+  it("can override syntax groups", function()
+    command('syntax on')
+    command('syntax keyword Foobar foobar')
+    command('syntax keyword Article the')
+    command('hi Foobar guibg=#FF0000')
+    command('hi Article guifg=#00FF00 gui=bold')
+    insert('the foobar was foobar')
+    screen:expect([[
+      {25:the} {26:foobar} was {26:fooba}|
+      {26:^r}                   |
+      {0:~                   }|
+      {0:~                   }|
+      {0:~                   }|
+      {0:~                   }|
+      {0:~                   }|
+                          |
+    ]])
+
+    command('split')
+    command('set winhl=Foobar:Background1,Article:ErrorMsg')
+    screen:expect{grid=[[
+      {15:the} {1:foobar} was {1:fooba}|
+      {1:^r}                   |
+      {0:~                   }|
+      {3:[No Name] [+]       }|
+      {25:the} {26:foobar} was {26:fooba}|
+      {26:r}                   |
+      {4:[No Name] [+]       }|
+                          |
+    ]]}
+  end)
+end)
+
+describe('highlight namespaces', function()
+  local screen
+  local ns1, ns2
+
+  before_each(function()
+    clear()
+    screen = Screen.new(25,10)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = {foreground = Screen.colors.Blue, bold = true};
+      [2] = {background = Screen.colors.DarkGrey};
+      [3] = {italic = true, foreground = Screen.colors.DarkCyan, background = Screen.colors.DarkOrange4};
+      [4] = {background = Screen.colors.Magenta4};
+      [5] = {background = Screen.colors.Magenta4, foreground = Screen.colors.Crimson};
+      [6] = {bold = true, reverse = true};
+      [7] = {reverse = true};
+    }
+
+    ns1 = meths.create_namespace 'grungy'
+    ns2 = meths.create_namespace 'ultrared'
+
+    meths.set_hl(ns1, 'Normal', {bg='DarkGrey'})
+    meths.set_hl(ns1, 'NonText', {bg='DarkOrange4', fg='DarkCyan', italic=true})
+    meths.set_hl(ns2, 'Normal', {bg='DarkMagenta'})
+    meths.set_hl(ns2, 'NonText', {fg='Crimson'})
+  end)
+
+  it('can be used globally', function()
+    screen:expect{grid=[[
+      ^                         |
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+                               |
+    ]]}
+
+    meths.set_hl_ns(ns1)
+    screen:expect{grid=[[
+      {2:^                         }|
+      {3:~                        }|
+      {3:~                        }|
+      {3:~                        }|
+      {3:~                        }|
+      {3:~                        }|
+      {3:~                        }|
+      {3:~                        }|
+      {3:~                        }|
+                               |
+    ]]}
+
+    meths.set_hl_ns(ns2)
+    screen:expect{grid=[[
+      {4:^                         }|
+      {5:~                        }|
+      {5:~                        }|
+      {5:~                        }|
+      {5:~                        }|
+      {5:~                        }|
+      {5:~                        }|
+      {5:~                        }|
+      {5:~                        }|
+                               |
+    ]]}
+
+    meths.set_hl_ns(0)
+    screen:expect{grid=[[
+      ^                         |
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+      {1:~                        }|
+                               |
+    ]]}
+  end)
+
+  it('can be used per window', function()
+    local win1 = meths.get_current_win()
+    command 'split'
+    local win2 = meths.get_current_win()
+    command 'split'
+
+    meths.win_set_hl_ns(win1, ns1)
+    meths.win_set_hl_ns(win2, ns2)
+
+    screen:expect{grid=[[
+      ^                         |
+      {1:~                        }|
+      {6:[No Name]                }|
+      {4:                         }|
+      {5:~                        }|
+      {7:[No Name]                }|
+      {2:                         }|
+      {3:~                        }|
+      {7:[No Name]                }|
+                               |
+    ]]}
+  end)
 end)
diff --git a/test/functional/ui/hlstate_spec.lua b/test/functional/ui/hlstate_spec.lua
index df7f34aa7f..dc74d6d401 100644
--- a/test/functional/ui/hlstate_spec.lua
+++ b/test/functional/ui/hlstate_spec.lua
@@ -119,13 +119,15 @@ describe('ext_hlstate detailed highlights', function()
         [3] = {{bold = true, reverse = true}, {{hi_name = "StatusLine", ui_name = "StatusLine", kind = "ui"}}},
         [4] = {{reverse = true}, {{hi_name = "StatusLineNC", ui_name = "StatusLineNC", kind = "ui"}}},
         [5] = {{background = Screen.colors.Red, foreground = Screen.colors.Grey100}, {{hi_name = "ErrorMsg", ui_name = "LineNr", kind = "ui"}}},
-        [6] = {{bold = true, reverse = true}, {{hi_name = "MsgSeparator", ui_name = "Normal", kind = "ui"}}},
+        [6] = {{bold = true, reverse = true}, {{hi_name = "Normal", ui_name = "Normal", kind = "ui"}}},
         [7] = {{foreground = Screen.colors.Brown, bold = true, reverse = true}, {6, 1}},
-        [8] = {{foreground = Screen.colors.Blue1, bold = true, reverse = true}, {6, 2}},
-        [9] = {{bold = true, foreground = Screen.colors.Brown}, {{hi_name = "Statement", ui_name = "NormalNC", kind = "ui"}}},
+        [8] = {{foreground = Screen.colors.Blue1, bold = true, reverse = true}, {6, 14}},
+        [9] = {{bold = true, foreground = Screen.colors.Brown}, {{hi_name = "NormalNC", ui_name = "NormalNC", kind = "ui"}}},
         [10] = {{bold = true, foreground = Screen.colors.Brown}, {9, 1}},
-        [11] = {{bold = true, foreground = Screen.colors.Blue1}, {9, 2}},
+        [11] = {{bold = true, foreground = Screen.colors.Blue1}, {9, 14}},
         [12] = {{}, {{hi_name = "MsgArea", ui_name = "MsgArea", kind = "ui"}}},
+        [13] = {{background = Screen.colors.Red1, foreground = Screen.colors.Gray100}, {{ui_name = "LineNr", kind = "ui", hi_name = "LineNr"}}};
+        [14] = {{bold = true, foreground = Screen.colors.Blue}, {{ui_name = "EndOfBuffer", kind = "ui", hi_name = "EndOfBuffer"}}};
     })
 
     command("set number")
@@ -143,16 +145,16 @@ describe('ext_hlstate detailed highlights', function()
     ]])
 
     command("set winhl=LineNr:ErrorMsg")
-    screen:expect([[
-      {5:  1 }^                                    |
-      {2:~                                       }|
-      {2:~                                       }|
+    screen:expect{grid=[[
+      {13:  1 }^                                    |
+      {14:~                                       }|
+      {14:~                                       }|
       {3:[No Name]                               }|
       {1:  1 }                                    |
       {2:~                                       }|
       {4:[No Name]                               }|
       {12:                                        }|
-    ]])
+    ]]}
 
     command("set winhl=Normal:MsgSeparator,NormalNC:Statement")
     screen:expect([[
diff --git a/test/functional/ui/screen.lua b/test/functional/ui/screen.lua
index ea98705394..6ee9e7b393 100644
--- a/test/functional/ui/screen.lua
+++ b/test/functional/ui/screen.lua
@@ -546,7 +546,7 @@ function Screen:_wait(check, flags)
 
     return true
   end
-  run_session(self._session, flags.request_cb, notification_cb, nil, minimal_timeout)
+  local eof = run_session(self._session, flags.request_cb, notification_cb, nil, minimal_timeout)
   if not did_flush then
     err = "no flush received"
   elseif not checked then
@@ -557,9 +557,9 @@ function Screen:_wait(check, flags)
     end
   end
 
-  if not success_seen then
+  if not success_seen and not eof then
     did_miminal_timeout = true
-    run_session(self._session, flags.request_cb, notification_cb, nil, timeout-minimal_timeout)
+    eof = run_session(self._session, flags.request_cb, notification_cb, nil, timeout-minimal_timeout)
   end
 
   local did_warn = false
@@ -600,8 +600,10 @@ between asynchronous (feed(), nvim_input()) and synchronous API calls.
 
 
   if err then
+    if eof then err = err..'\n\n'..eof[2] end
     busted.fail(err, 3)
   elseif did_warn then
+    if eof then print(eof[2]) end
     local tb = debug.traceback()
     local index = string.find(tb, '\n%s*%[C]')
     print(string.sub(tb,1,index))