feat(ui): allow to set the highlight namespace per window

- reimplement 'winhl' in terms of highlight namespaces
- check for EOF in screen tests (to indicate a likely crash)

22 files changed, 351 insertions, 190 deletions

diff --git a/src/nvim/api/extmark.c b/src/nvim/api/extmark.c
index c36f127507..b8295601c1 100644
--- a/src/nvim/api/extmark.c
+++ b/src/nvim/api/extmark.c
@@ -1031,6 +1031,8 @@ void nvim_set_decoration_provider(Integer ns_id, DictionaryOf(LuaRef) opts, Erro
   }
 
   p->active = true;
+  p->hl_valid++;
+  p->hl_cached = false;
   return;
 error:
   decor_provider_clear(p);
diff --git a/src/nvim/api/keysets.lua b/src/nvim/api/keysets.lua
index 4f4ac40ce9..6fad52ba75 100644
--- a/src/nvim/api/keysets.lua
+++ b/src/nvim/api/keysets.lua
@@ -104,7 +104,6 @@ return {
     "reverse";
     "nocombine";
     "default";
-    "global";
     "cterm";
     "foreground"; "fg";
     "background"; "bg";
@@ -112,9 +111,9 @@ return {
     "ctermbg";
     "special"; "sp";
     "link";
+    "global_link";
     "fallback";
     "blend";
-    "temp";
   };
   highlight_cterm = {
     "bold";
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 3f9730c569..25f6e822b3 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -93,7 +93,6 @@ Dictionary nvim_get_hl_by_name(String name, Boolean rgb, Error *err)
 }
 
 /// Gets a highlight definition by id. |hlID()|
-///
 /// @param hl_id Highlight id as returned by |hlID()|
 /// @param rgb Export RGB colors
 /// @param[out] err Error details, if any
@@ -182,35 +181,38 @@ void nvim_set_hl(Integer ns_id, String name, Dict(highlight) *val, Error *err)
   }
 }
 
-/// Set active namespace for highlights.
-///
-/// NB: this function can be called from async contexts, but the
-/// semantics are not yet well-defined. To start with
-/// |nvim_set_decoration_provider| on_win and on_line callbacks
-/// are explicitly allowed to change the namespace during a redraw cycle.
+/// Set active namespace for highlights. This can be set for a single window,
+/// see |nvim_win_set_hl_ns|.
 ///
-/// @param ns_id the namespace to activate
+/// @param ns_id the namespace to use
 /// @param[out] err Error details, if any
-void nvim__set_hl_ns(Integer ns_id, Error *err)
-  FUNC_API_FAST
+void nvim_set_hl_ns(Integer ns_id, Error *err)
+  FUNC_API_SINCE(10)
 {
-  if (ns_id >= 0) {
-    ns_hl_active = (NS)ns_id;
+  if (ns_id < 0) {
+    api_set_error(err, kErrorTypeValidation, "no such namespace");
+    return;
   }
 
-  // TODO(bfredl): this is a little bit hackish.  Eventually we want a standard
-  // event path for redraws caused by "fast" events. This could tie in with
-  // better throttling of async events causing redraws, such as non-batched
-  // nvim_buf_set_extmark calls from async contexts.
-  if (!provider_active && !ns_hl_changed && must_redraw < NOT_VALID) {
-    multiqueue_put(main_loop.events, on_redraw_event, 0);
-  }
-  ns_hl_changed = true;
+  ns_hl_global = (NS)ns_id;
+  hl_check_ns();
+  redraw_all_later(NOT_VALID);
 }
 
-static void on_redraw_event(void **argv)
+/// Set active namespace for highlights while redrawing.
+///
+/// This function meant to be called while redrawing, primarily from
+/// |nvim_set_decoration_provider| on_win and on_line callbacks, which
+/// are allowed to change the namespace during a redraw cycle.
+///
+/// @param ns_id the namespace to activate
+/// @param[out] err Error details, if any
+void nvim_set_hl_ns_fast(Integer ns_id, Error *err)
+  FUNC_API_SINCE(10)
+  FUNC_API_FAST
 {
-  redraw_all_later(NOT_VALID);
+  ns_hl_fast = (NS)ns_id;
+  hl_check_ns();
 }
 
 /// Sends input-keys to Nvim, subject to various quirks controlled by `mode`
@@ -480,7 +482,7 @@ Object nvim_notify(String msg, Integer log_level, Dictionary opts, Error *err)
   ADD_C(args, INTEGER_OBJ(log_level));
   ADD_C(args, DICTIONARY_OBJ(opts));
 
-  return nlua_exec(STATIC_CSTR_AS_STRING("return vim.notify(...)"), args, err);
+  return NLUA_EXEC_STATIC("return vim.notify(...)", args, err);
 }
 
 /// Calculates the number of display cells occupied by `text`.
@@ -1835,11 +1837,9 @@ Array nvim_get_proc_children(Integer pid, Error *err)
   if (rv == 2) {
     // syscall failed (possibly because of kernel options), try shelling out.
     DLOG("fallback to vim._os_proc_children()");
-    Array a = ARRAY_DICT_INIT;
+    MAXSIZE_TEMP_ARRAY(a, 1);
     ADD(a, INTEGER_OBJ(pid));
-    String s = STATIC_CSTR_AS_STRING("return vim._os_proc_children(...)");
-    Object o = nlua_exec(s, a, err);
-    api_free_array(a);
+    Object o = NLUA_EXEC_STATIC("return vim._os_proc_children(...)", a, err);
     if (o.type == kObjectTypeArray) {
       rvobj = o.data.array;
     } else if (!ERROR_SET(err)) {
@@ -1880,12 +1880,9 @@ Object nvim_get_proc(Integer pid, Error *err)
   }
 #else
   // Cross-platform process info APIs are miserable, so use `ps` instead.
-  Array a = ARRAY_DICT_INIT;
+  MAXSIZE_TEMP_ARRAY(a, 1);
   ADD(a, INTEGER_OBJ(pid));
-  String s = cstr_to_string("return vim._os_proc_info(select(1, ...))");
-  Object o = nlua_exec(s, a, err);
-  api_free_string(s);
-  api_free_array(a);
+  Object o = NLUA_EXEC_STATIC("return vim._os_proc_info(...)", a, err);
   if (o.type == kObjectTypeArray && o.data.array.size == 0) {
     return NIL;  // Process not found.
   } else if (o.type == kObjectTypeDictionary) {
diff --git a/src/nvim/api/window.c b/src/nvim/api/window.c
index 5a4ff70257..d0a2ff766e 100644
--- a/src/nvim/api/window.c
+++ b/src/nvim/api/window.c
@@ -426,3 +426,28 @@ Object nvim_win_call(Window window, LuaRef fun, Error *err)
   try_end(err);
   return res;
 }
+
+/// Set highlight namespace for a window. This will use highlights defined in
+/// this namespace, but fall back to global highlights (ns=0) when missing.
+///
+/// This takes predecence over the 'winhighlight' option.
+///
+/// @param ns_id the namespace to use
+/// @param[out] err Error details, if any
+void nvim_win_set_hl_ns(Window window, Integer ns_id, Error *err)
+  FUNC_API_SINCE(10)
+{
+  win_T *win = find_window_by_handle(window, err);
+  if (!win) {
+    return;
+  }
+
+  // -1 is allowed as inherit global namespace
+  if (ns_id < -1) {
+    api_set_error(err, kErrorTypeValidation, "no such namespace");
+  }
+
+  win->w_ns_hl = (NS)ns_id;
+  win->w_hl_needs_update = true;
+  redraw_later(win, NOT_VALID);
+}
diff --git a/src/nvim/buffer_defs.h b/src/nvim/buffer_defs.h
index f5dd86cb98..15b964f5ed 100644
--- a/src/nvim/buffer_defs.h
+++ b/src/nvim/buffer_defs.h
@@ -1143,11 +1143,14 @@ struct window_S {
 
   synblock_T *w_s;                 ///< for :ownsyntax
 
+  int w_ns_hl;
+  int w_ns_hl_winhl;
+  int w_ns_hl_active;
+  int *w_ns_hl_attr;
+
   int w_hl_id_normal;               ///< 'winhighlight' normal id
   int w_hl_attr_normal;             ///< 'winhighlight' normal final attrs
-
-  int w_hl_ids[HLF_COUNT];          ///< 'winhighlight' id
-  int w_hl_attrs[HLF_COUNT];        ///< 'winhighlight' final attrs
+  int w_hl_attr_normalnc;           ///< 'winhighlight' NormalNC final attrs
 
   int w_hl_needs_update;            ///< attrs need to be recalculated
 
@@ -1469,11 +1472,6 @@ struct window_S {
   size_t w_winbar_click_defs_size;
 };
 
-static inline int win_hl_attr(win_T *wp, int hlf)
-{
-  return wp->w_hl_attrs[hlf];
-}
-
 /// Macros defined in Vim, but not in Neovim
 #define CHANGEDTICK(buf) \
   (=== Include buffer.h & use buf_(get|set|inc) _changedtick ===)
diff --git a/src/nvim/decoration_provider.c b/src/nvim/decoration_provider.c
index 04d875c4e3..95e13b4240 100644
--- a/src/nvim/decoration_provider.c
+++ b/src/nvim/decoration_provider.c
@@ -14,7 +14,7 @@ static kvec_t(DecorProvider) decor_providers = KV_INITIAL_VALUE;
 #define DECORATION_PROVIDER_INIT(ns_id) (DecorProvider) \
   { ns_id, false, LUA_NOREF, LUA_NOREF, \
     LUA_NOREF, LUA_NOREF, LUA_NOREF, \
-    LUA_NOREF, -1 }
+    LUA_NOREF, -1, false }
 
 static bool decor_provider_invoke(NS ns_id, const char *name, LuaRef ref, Array args,
                                   bool default_true, char **perr)
@@ -107,8 +107,6 @@ void decor_providers_invoke_win(win_T *wp, DecorProviders *providers,
       }
     }
   }
-
-  win_check_ns_hl(wp);
 }
 
 /// For each provider invoke the 'line' callback for a given window row.
@@ -135,7 +133,7 @@ void providers_invoke_line(win_T *wp, DecorProviders *providers, int row, bool *
         kv_A(*providers, k) = NULL;
       }
 
-      win_check_ns_hl(wp);
+      hl_check_ns();
     }
   }
 }
diff --git a/src/nvim/decoration_provider.h b/src/nvim/decoration_provider.h
index 3ec7c80357..dd1ed6c581 100644
--- a/src/nvim/decoration_provider.h
+++ b/src/nvim/decoration_provider.h
@@ -13,6 +13,7 @@ typedef struct {
   LuaRef redraw_end;
   LuaRef hl_def;
   int hl_valid;
+  bool hl_cached;
 } DecorProvider;
 
 typedef kvec_withinit_t(DecorProvider *, 4) DecorProviders;
diff --git a/src/nvim/globals.h b/src/nvim/globals.h
index a8e15dfdd9..954b62883e 100644
--- a/src/nvim/globals.h
+++ b/src/nvim/globals.h
@@ -96,9 +96,6 @@ EXTERN struct nvim_stats_s {
 EXTERN int Rows INIT(= DFLT_ROWS);     // nr of rows in the screen
 EXTERN int Columns INIT(= DFLT_COLS);  // nr of columns in the screen
 
-EXTERN NS ns_hl_active INIT(= 0);         // current ns that defines highlights
-EXTERN bool ns_hl_changed INIT(= false);  // highlight need update
-
 // We use 64-bit file functions here, if available.  E.g. ftello() returns
 // off_t instead of long, which helps if long is 32 bit and off_t is 64 bit.
 // We assume that when fseeko() is available then ftello() is too.
diff --git a/src/nvim/hardcopy.c b/src/nvim/hardcopy.c
index fa357f864e..d811a78ca8 100644
--- a/src/nvim/hardcopy.c
+++ b/src/nvim/hardcopy.c
@@ -900,7 +900,7 @@ static colnr_T hardcopy_line(prt_settings_T *psettings, int page_line, prt_pos_T
     }
     // syntax highlighting stuff.
     if (psettings->do_syntax) {
-      id = syn_get_id(curwin, ppos->file_line, col, 1, NULL, FALSE);
+      id = syn_get_id(curwin, ppos->file_line, col, 1, NULL, false);
       if (id > 0) {
         id = syn_get_final_id(id);
       } else {
diff --git a/src/nvim/highlight.c b/src/nvim/highlight.c
index dfd077840e..504cae9a51 100644
--- a/src/nvim/highlight.c
+++ b/src/nvim/highlight.c
@@ -32,7 +32,9 @@ static Map(int, int) blend_attr_entries = MAP_INIT;
 static Map(int, int) blendthrough_attr_entries = MAP_INIT;
 
 /// highlight entries private to a namespace
-static Map(ColorKey, ColorItem) ns_hl;
+static Map(ColorKey, ColorItem) ns_hls;
+typedef int NSHlAttr[HLF_COUNT + 1];
+static PMap(handle_T) ns_hl_attr;
 
 void highlight_init(void)
 {
@@ -147,42 +149,46 @@ int hl_get_syn_attr(int ns_id, int idx, HlAttrs at_en)
 
 void ns_hl_def(NS ns_id, int hl_id, HlAttrs attrs, int link_id, Dict(highlight) *dict)
 {
-  if ((attrs.rgb_ae_attr & HL_DEFAULT)
-      && map_has(ColorKey, ColorItem)(&ns_hl, ColorKey(ns_id, hl_id))) {
-    return;
-  }
   if (ns_id == 0) {
     assert(dict);
     // set in global (':highlight') namespace
     set_hl_group(hl_id, attrs, dict, link_id);
     return;
   }
+  if ((attrs.rgb_ae_attr & HL_DEFAULT)
+      && map_has(ColorKey, ColorItem)(&ns_hls, ColorKey(ns_id, hl_id))) {
+    return;
+  }
   DecorProvider *p = get_decor_provider(ns_id, true);
   int attr_id = link_id > 0 ? -1 : hl_get_syn_attr(ns_id, hl_id, attrs);
   ColorItem it = { .attr_id = attr_id,
                    .link_id = link_id,
                    .version = p->hl_valid,
-                   .is_default = (attrs.rgb_ae_attr & HL_DEFAULT) };
-  map_put(ColorKey, ColorItem)(&ns_hl, ColorKey(ns_id, hl_id), it);
+                   .is_default = (attrs.rgb_ae_attr & HL_DEFAULT),
+                   .link_global = (attrs.rgb_ae_attr & HL_GLOBAL) };
+  map_put(ColorKey, ColorItem)(&ns_hls, ColorKey(ns_id, hl_id), it);
+  p->hl_cached = false;
 }
 
-int ns_get_hl(NS ns_id, int hl_id, bool link, bool nodefault)
+int ns_get_hl(NS *ns_hl, int hl_id, bool link, bool nodefault)
 {
   static int recursive = 0;
 
-  if (ns_id < 0) {
+  if (*ns_hl < 0) {
     if (ns_hl_active <= 0) {
       return -1;
     }
-    ns_id = ns_hl_active;
+    *ns_hl = ns_hl_active;
   }
 
+  int ns_id = *ns_hl;
+
   DecorProvider *p = get_decor_provider(ns_id, true);
-  ColorItem it = map_get(ColorKey, ColorItem)(&ns_hl, ColorKey(ns_id, hl_id));
+  ColorItem it = map_get(ColorKey, ColorItem)(&ns_hls, ColorKey(ns_id, hl_id));
   // TODO(bfredl): map_ref true even this?
-  bool valid_cache = it.version >= p->hl_valid;
+  bool valid_item = it.version >= p->hl_valid;
 
-  if (!valid_cache && p->hl_def != LUA_NOREF && !recursive) {
+  if (!valid_item && p->hl_def != LUA_NOREF && !recursive) {
     MAXSIZE_TEMP_ARRAY(args, 3);
     ADD_C(args, INTEGER_OBJ((Integer)ns_id));
     ADD_C(args, STRING_OBJ(cstr_to_string((char *)syn_id2name(hl_id))));
@@ -215,44 +221,76 @@ int ns_get_hl(NS ns_id, int hl_id, bool link, bool nodefault)
     it.attr_id = fallback ? -1 : hl_get_syn_attr((int)ns_id, hl_id, attrs);
     it.version = p->hl_valid - tmp;
     it.is_default = attrs.rgb_ae_attr & HL_DEFAULT;
-    map_put(ColorKey, ColorItem)(&ns_hl, ColorKey(ns_id, hl_id), it);
+    it.link_global = attrs.rgb_ae_attr & HL_GLOBAL;
+    map_put(ColorKey, ColorItem)(&ns_hls, ColorKey(ns_id, hl_id), it);
+    valid_item = true;
   }
 
-  if (it.is_default && nodefault) {
+  if ((it.is_default && nodefault) || !valid_item) {
     return -1;
   }
 
   if (link) {
-    return it.attr_id >= 0 ? 0 : it.link_id;
+    if (it.attr_id >= 0) {
+      return 0;
+    } else {
+      if (it.link_global) {
+        *ns_hl = 0;
+      }
+      return it.link_id;
+    }
   } else {
     return it.attr_id;
   }
 }
 
-bool win_check_ns_hl(win_T *wp)
+bool hl_check_ns(void)
 {
-  if (ns_hl_changed) {
-    highlight_changed();
-    if (wp) {
-      update_window_hl(wp, true);
+  int ns = 0;
+  if (ns_hl_fast > 0) {
+    ns = ns_hl_fast;
+  } else if (ns_hl_win >= 0) {
+    ns = ns_hl_win;
+  } else {
+    ns = ns_hl_global;
+  }
+  if (ns_hl_active == ns) {
+    return false;
+  }
+
+  ns_hl_active = ns;
+  hl_attr_active = highlight_attr;
+  if (ns > 0) {
+    update_ns_hl(ns);
+    NSHlAttr *hl_def = (NSHlAttr *)pmap_get(handle_T)(&ns_hl_attr, ns);
+    if (hl_def) {
+      hl_attr_active = *hl_def;
     }
-    ns_hl_changed = false;
-    return true;
   }
-  return false;
+  need_highlight_changed = true;
+  return true;
+}
+
+/// prepare for drawing window `wp` or global elements if NULL
+///
+/// Note: pum should be drawn in the context of the current window!
+bool win_check_ns_hl(win_T *wp)
+{
+  ns_hl_win = wp ? wp->w_ns_hl : -1;
+  return hl_check_ns();
 }
 
 /// Get attribute code for a builtin highlight group.
 ///
 /// The final syntax group could be modified by hi-link or 'winhighlight'.
-int hl_get_ui_attr(int idx, int final_id, bool optional)
+int hl_get_ui_attr(int ns_id, int idx, int final_id, bool optional)
 {
   HlAttrs attrs = HLATTRS_INIT;
   bool available = false;
 
   if (final_id > 0) {
-    int syn_attr = syn_id2attr(final_id);
-    if (syn_attr != 0) {
+    int syn_attr = syn_ns_id2attr(ns_id, final_id, optional);
+    if (syn_attr > 0) {
       attrs = syn_attr2entry(syn_attr);
       available = true;
     }
@@ -265,7 +303,7 @@ int hl_get_ui_attr(int idx, int final_id, bool optional)
     if (pum_drawn()) {
       must_redraw_pum = true;
     }
-  } else if (idx == HLF_MSG) {
+  } else if (idx == HLF_MSG && ns_id == -1) {
     msg_grid.blending = attrs.hl_blend > -1;
   }
 
@@ -278,6 +316,21 @@ int hl_get_ui_attr(int idx, int final_id, bool optional)
 
 void update_window_hl(win_T *wp, bool invalid)
 {
+  int ns_id = wp->w_ns_hl;
+
+  update_ns_hl(ns_id);
+  if (ns_id != wp->w_ns_hl_active) {
+    wp->w_ns_hl_active = ns_id;
+
+    wp->w_ns_hl_attr = *(NSHlAttr *)pmap_get(handle_T)(&ns_hl_attr, ns_id);
+    if (!wp->w_ns_hl_attr) {
+      // No specific highlights, use the defaults.
+      wp->w_ns_hl_attr = highlight_attr;
+    }
+  }
+
+  int *hl_def = wp->w_ns_hl_attr;
+
   if (!wp->w_hl_needs_update && !invalid) {
     return;
   }
@@ -285,34 +338,17 @@ void update_window_hl(win_T *wp, bool invalid)
 
   // If a floating window is blending it always have a named
   // wp->w_hl_attr_normal group. HL_ATTR(HLF_NFLOAT) is always named.
-  bool has_blend = wp->w_floating && wp->w_p_winbl != 0;
 
   // determine window specific background set in 'winhighlight'
   bool float_win = wp->w_floating && !wp->w_float_config.external;
-  if (wp != curwin && wp->w_hl_ids[HLF_INACTIVE] != 0) {
-    wp->w_hl_attr_normal = hl_get_ui_attr(HLF_INACTIVE,
-                                          wp->w_hl_ids[HLF_INACTIVE],
-                                          !has_blend);
-  } else if (float_win && wp->w_hl_ids[HLF_NFLOAT] != 0) {
-    wp->w_hl_attr_normal = hl_get_ui_attr(HLF_NFLOAT,
-                                          wp->w_hl_ids[HLF_NFLOAT], !has_blend);
-  } else if (wp->w_hl_id_normal != 0) {
-    wp->w_hl_attr_normal = hl_get_ui_attr(-1, wp->w_hl_id_normal, !has_blend);
+  if (float_win && hl_def[HLF_NFLOAT] != 0) {
+    wp->w_hl_attr_normal = hl_def[HLF_NFLOAT];
+  } else if (hl_def[HLF_COUNT] > 0) {
+    wp->w_hl_attr_normal = hl_def[HLF_COUNT];
   } else {
     wp->w_hl_attr_normal = float_win ? HL_ATTR(HLF_NFLOAT) : 0;
   }
 
-  // NOOOO! You cannot just pretend that "Normal" is just like any other
-  // syntax group! It needs at least 10 layers of special casing! Noooooo!
-  //
-  // haha, theme engine go brrr
-  int normality = syn_check_group(S_LEN("Normal"));
-  int ns_attr = ns_get_hl(-1, normality, false, false);
-  if (ns_attr > 0) {
-    // TODO(bfredl): hantera NormalNC and so on
-    wp->w_hl_attr_normal = ns_attr;
-  }
-
   // if blend= attribute is not set, 'winblend' value overrides it.
   if (wp->w_floating && wp->w_p_winbl > 0) {
     HlEntry entry = kv_A(attr_entries, wp->w_hl_attr_normal);
@@ -322,28 +358,13 @@ void update_window_hl(win_T *wp, bool invalid)
     }
   }
 
-  if (wp != curwin && wp->w_hl_ids[HLF_INACTIVE] == 0) {
-    wp->w_hl_attr_normal = hl_combine_attr(HL_ATTR(HLF_INACTIVE),
-                                           wp->w_hl_attr_normal);
-  }
-
-  for (int hlf = 0; hlf < HLF_COUNT; hlf++) {
-    int attr;
-    if (wp->w_hl_ids[hlf] != 0) {
-      attr = hl_get_ui_attr(hlf, wp->w_hl_ids[hlf], false);
-    } else {
-      attr = HL_ATTR(hlf);
-    }
-    wp->w_hl_attrs[hlf] = attr;
-  }
-
   wp->w_float_config.shadow = false;
   if (wp->w_floating && wp->w_float_config.border) {
     for (int i = 0; i < 8; i++) {
-      int attr = wp->w_hl_attrs[HLF_BORDER];
+      int attr = hl_def[HLF_BORDER];
       if (wp->w_float_config.border_hl_ids[i]) {
-        attr = hl_get_ui_attr(HLF_BORDER, wp->w_float_config.border_hl_ids[i],
-                              false);
+        attr = hl_get_ui_attr(ns_id, HLF_BORDER,
+                              wp->w_float_config.border_hl_ids[i], false);
         HlAttrs a = syn_attr2entry(attr);
         if (a.hl_blend) {
           wp->w_float_config.shadow = true;
@@ -355,6 +376,65 @@ void update_window_hl(win_T *wp, bool invalid)
 
   // shadow might cause blending
   check_blending(wp);
+
+  // TODO(bfredl): this a bit ad-hoc. move it from highlight ns logic to 'winhl'
+  // implementation?
+  if (hl_def[HLF_INACTIVE] == 0) {
+    wp->w_hl_attr_normalnc = hl_combine_attr(HL_ATTR(HLF_INACTIVE),
+                                             wp->w_hl_attr_normal);
+  } else {
+    wp->w_hl_attr_normalnc = hl_def[HLF_INACTIVE];
+  }
+}
+
+void update_ns_hl(int ns_id)
+{
+  if (ns_id <= 0) {
+    return;
+  }
+  DecorProvider *p = get_decor_provider(ns_id, true);
+  if (p->hl_cached) {
+    return;
+  }
+
+  NSHlAttr **alloc = (NSHlAttr **)pmap_ref(handle_T)(&ns_hl_attr, ns_id, true);
+  if (*alloc == NULL) {
+    *alloc = xmalloc(sizeof(**alloc));
+  }
+  int *hl_attrs = **alloc;
+
+  for (int hlf = 0; hlf < (int)HLF_COUNT; hlf++) {
+    int id = syn_check_group(hlf_names[hlf], STRLEN(hlf_names[hlf]));
+    bool optional = (hlf == HLF_INACTIVE || hlf == HLF_NFLOAT);
+    hl_attrs[hlf] = hl_get_ui_attr(ns_id, hlf, id, optional);
+  }
+
+  // NOOOO! You cannot just pretend that "Normal" is just like any other
+  // syntax group! It needs at least 10 layers of special casing! Noooooo!
+  //
+  // haha, tema engine go brrr
+  int normality = syn_check_group(S_LEN("Normal"));
+  hl_attrs[HLF_COUNT] = hl_get_ui_attr(ns_id, -1, normality, true);
+
+  // hl_get_ui_attr might have invalidated the decor provider
+  p = get_decor_provider(ns_id, true);
+  p->hl_cached = true;
+}
+
+int win_bg_attr(win_T *wp)
+{
+  if (ns_hl_fast < 0) {
+    int local = (wp == curwin) ? wp->w_hl_attr_normal : wp->w_hl_attr_normalnc;
+    if (local) {
+      return local;
+    }
+  }
+
+  if (wp == curwin || hl_attr_active[HLF_INACTIVE] == 0) {
+    return hl_attr_active[HLF_COUNT];
+  } else {
+    return hl_attr_active[HLF_INACTIVE];
+  }
 }
 
 /// Gets HL_UNDERLINE highlight.
@@ -403,7 +483,7 @@ void clear_hl_tables(bool reinit)
     map_destroy(int, int)(&combine_attr_entries);
     map_destroy(int, int)(&blend_attr_entries);
     map_destroy(int, int)(&blendthrough_attr_entries);
-    map_destroy(ColorKey, ColorItem)(&ns_hl);
+    map_destroy(ColorKey, ColorItem)(&ns_hls);
   }
 }
 
@@ -852,7 +932,6 @@ HlAttrs dict2hlattrs(Dict(highlight) *dict, bool use_rgb, int *link_id, Error *e
   CHECK_FLAG(dict, mask, strikethrough, , HL_STRIKETHROUGH);
   CHECK_FLAG(dict, mask, nocombine, , HL_NOCOMBINE);
   CHECK_FLAG(dict, mask, default, _, HL_DEFAULT);
-  CHECK_FLAG(dict, mask, global, , HL_GLOBAL);
 
   if (HAS_KEY(dict->fg)) {
     fg = object_to_color(dict->fg, "fg", true, err);
@@ -895,14 +974,21 @@ HlAttrs dict2hlattrs(Dict(highlight) *dict, bool use_rgb, int *link_id, Error *e
     return hlattrs;
   }
 
-  if (HAS_KEY(dict->link)) {
+  if (HAS_KEY(dict->link) || HAS_KEY(dict->global_link)) {
     if (link_id) {
-      *link_id = object_to_hl_id(dict->link, "link", err);
+      if (HAS_KEY(dict->global_link)) {
+        *link_id = object_to_hl_id(dict->global_link, "link", err);
+        mask |= HL_GLOBAL;
+      } else {
+        *link_id = object_to_hl_id(dict->link, "link", err);
+      }
+
       if (ERROR_SET(err)) {
         return hlattrs;
       }
     } else {
-      api_set_error(err, kErrorTypeValidation, "Invalid Key: 'link'");
+      api_set_error(err, kErrorTypeValidation, "Invalid Key: '%s'",
+                    HAS_KEY(dict->global_link) ? "global_link" : "link");
     }
   }
 
diff --git a/src/nvim/highlight.h b/src/nvim/highlight.h
index ae63f83d65..50299bb91c 100644
--- a/src/nvim/highlight.h
+++ b/src/nvim/highlight.h
@@ -4,6 +4,7 @@
 #include <stdbool.h>
 
 #include "nvim/api/private/defs.h"
+#include "nvim/buffer_defs.h"
 #include "nvim/highlight_defs.h"
 #include "nvim/ui.h"
 
@@ -11,6 +12,13 @@
 # include "highlight.h.generated.h"
 #endif
 
+static inline int win_hl_attr(win_T *wp, int hlf)
+{
+  // wp->w_ns_hl_attr might be null if we check highlights
+  // prior to entering redraw
+  return ((wp->w_ns_hl_attr && ns_hl_fast < 0) ? wp->w_ns_hl_attr : hl_attr_active)[hlf];
+}
+
 #define HL_SET_DEFAULT_COLORS(rgb_fg, rgb_bg, rgb_sp) \
   do { \
     bool dark_ = (*p_bg == 'd'); \
diff --git a/src/nvim/highlight_defs.h b/src/nvim/highlight_defs.h
index da6d5284c2..ffcb0f3f22 100644
--- a/src/nvim/highlight_defs.h
+++ b/src/nvim/highlight_defs.h
@@ -180,7 +180,7 @@ EXTERN const char *hlf_names[] INIT(= {
   [HLF_CU] = "Cursor",
 });
 
-EXTERN int highlight_attr[HLF_COUNT];       // Highl. attr for each context.
+EXTERN int highlight_attr[HLF_COUNT + 1];     // Highl. attr for each context.
 EXTERN int highlight_attr_last[HLF_COUNT];  // copy for detecting changed groups
 EXTERN int highlight_user[9];                   // User[1-9] attributes
 EXTERN int highlight_stlnc[9];                  // On top of user
@@ -190,6 +190,13 @@ EXTERN RgbValue normal_fg INIT(= -1);
 EXTERN RgbValue normal_bg INIT(= -1);
 EXTERN RgbValue normal_sp INIT(= -1);
 
+EXTERN NS ns_hl_global INIT(= 0);  // global highlight namespace
+EXTERN NS ns_hl_win INIT(= -1);    // highlight namespace for the current window
+EXTERN NS ns_hl_fast INIT(= -1);   // highlight namespace specified in a fast callback
+EXTERN NS ns_hl_active INIT(= 0);  // currently active/cached namespace
+
+EXTERN int *hl_attr_active INIT(= highlight_attr);
+
 typedef enum {
   kHlUnknown,
   kHlUI,
@@ -219,9 +226,10 @@ typedef struct {
   int link_id;
   int version;
   bool is_default;
+  bool link_global;
 } ColorItem;
-#define COLOR_ITEM_INITIALIZER { .attr_id = -1, .link_id = -1, \
-                                 .version = -1, .is_default = false }
+#define COLOR_ITEM_INITIALIZER { .attr_id = -1, .link_id = -1, .version = -1, \
+                                 .is_default = false, .link_global = false }
 
 /// highlight attributes with associated priorities
 typedef struct {
diff --git a/src/nvim/highlight_group.c b/src/nvim/highlight_group.c
index d79283d3e3..616c2a670d 100644
--- a/src/nvim/highlight_group.c
+++ b/src/nvim/highlight_group.c
@@ -1790,11 +1790,18 @@ static int syn_add_group(const char *name, size_t len)
 /// @see syn_attr2entry
 int syn_id2attr(int hl_id)
 {
-  hl_id = syn_get_final_id(hl_id);
+  return syn_ns_id2attr(-1, hl_id, false);
+}
+
+int syn_ns_id2attr(int ns_id, int hl_id, bool optional)
+{
+  hl_id = syn_ns_get_final_id(&ns_id, hl_id);
   HlGroup *sgp = &hl_table[hl_id - 1];  // index is ID minus one
 
-  int attr = ns_get_hl(-1, hl_id, false, sgp->sg_set);
-  if (attr >= 0) {
+  int attr = ns_get_hl(&ns_id, hl_id, false, sgp->sg_set);
+
+  // if a highlight group is optional, don't use the global value
+  if (attr >= 0 || (optional && ns_id > 0)) {
     return attr;
   }
   return sgp->sg_attr;
@@ -1803,10 +1810,16 @@ int syn_id2attr(int hl_id)
 /// Translate a group ID to the final group ID (following links).
 int syn_get_final_id(int hl_id)
 {
+  int id = curwin->w_ns_hl_active;
+  return syn_ns_get_final_id(&id, hl_id);
+}
+
+int syn_ns_get_final_id(int *ns_id, int hl_id)
+{
   int count;
 
   if (hl_id > highlight_ga.ga_len || hl_id < 1) {
-    return 0;                           // Can be called from eval!!
+    return 0;  // Can be called from eval!!
   }
 
   // Follow links until there is no more.
@@ -1814,10 +1827,10 @@ int syn_get_final_id(int hl_id)
   for (count = 100; --count >= 0;) {
     HlGroup *sgp = &hl_table[hl_id - 1];  // index is ID minus one
 
-    // ACHTUNG: when using "tmp" attribute (no link) the function might be
+    // TODO(bfredl): when using "tmp" attribute (no link) the function might be
     // called twice. it needs be smart enough to remember attr only to
     // syn_id2attr time
-    int check = ns_get_hl(-1, hl_id, true, sgp->sg_set);
+    int check = ns_get_hl(ns_id, hl_id, true, sgp->sg_set);
     if (check == 0) {
       return hl_id;  // how dare! it broke the link!
     } else if (check > 0) {
@@ -1915,14 +1928,15 @@ void highlight_changed(void)
     if (id == 0) {
       abort();
     }
-    int final_id = syn_get_final_id(id);
+    int ns_id = -1;
+    int final_id = syn_ns_get_final_id(&ns_id, id);
     if (hlf == HLF_SNC) {
       id_SNC = final_id;
     } else if (hlf == HLF_S) {
       id_S = final_id;
     }
 
-    highlight_attr[hlf] = hl_get_ui_attr(hlf, final_id,
+    highlight_attr[hlf] = hl_get_ui_attr(ns_id, hlf, final_id,
                                          (hlf == HLF_INACTIVE || hlf == HLF_LC));
 
     if (highlight_attr[hlf] != highlight_attr_last[hlf]) {
@@ -1935,6 +1949,9 @@ void highlight_changed(void)
     }
   }
 
+  // sentinel value. used when no hightlight namespace is active
+  highlight_attr[HLF_COUNT] = 0;
+
   //
   // Setup the user highlights
   //
diff --git a/src/nvim/lua/executor.h b/src/nvim/lua/executor.h
index 2afbbebfe7..78346fd81f 100644
--- a/src/nvim/lua/executor.h
+++ b/src/nvim/lua/executor.h
@@ -24,6 +24,8 @@ typedef struct {
 #endif
 } nlua_ref_state_t;
 
+#define NLUA_EXEC_STATIC(cstr, arg, err) nlua_exec(STATIC_CSTR_AS_STRING(cstr), arg, err)
+
 #define NLUA_CLEAR_REF(x) \
   do { \
     /* Take the address to avoid double evaluation. #1375 */ \
diff --git a/src/nvim/match.c b/src/nvim/match.c
index f085d7cdb4..6cfc7d1f24 100644
--- a/src/nvim/match.c
+++ b/src/nvim/match.c
@@ -11,6 +11,7 @@
 #include "nvim/eval/funcs.h"
 #include "nvim/ex_docmd.h"
 #include "nvim/fold.h"
+#include "nvim/highlight.h"
 #include "nvim/highlight_group.h"
 #include "nv