api: implement object namespaces

Namespaces is a lightweight concept that should be used to group
objects for purposes of bulk operations and introspection. This is
initially used for highlights and virtual text in buffers, and is
planned to also be used for extended marks. There is no plan use them
for privileges or isolation, neither to introduce nanespace-level
options.

10 files changed, 124 insertions, 34 deletions

diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c
index 487a912882..4559f37191 100644
--- a/src/nvim/api/buffer.c
+++ b/src/nvim/api/buffer.c
@@ -905,34 +905,34 @@ ArrayOf(Integer, 2) nvim_buf_get_mark(Buffer buffer, String name, Error *err)
 ///
 /// Useful for plugins that dynamically generate highlights to a buffer
 /// (like a semantic highlighter or linter). The function adds a single
-/// highlight to a buffer. Unlike matchaddpos() highlights follow changes to
+/// highlight to a buffer. Unlike |matchaddpos()| highlights follow changes to
 /// line numbering (as lines are inserted/removed above the highlighted line),
 /// like signs and marks do.
 ///
-/// `src_id` is useful for batch deletion/updating of a set of highlights. When
-/// called with `src_id = 0`, an unique source id is generated and returned.
-/// Successive calls can pass that `src_id` to associate new highlights with
-/// the same source group. All highlights in the same group can be cleared
-/// with `nvim_buf_clear_highlight`. If the highlight never will be manually
-/// deleted, pass `src_id = -1`.
+/// Namespaces are used for batch deletion/updating of a set of highlights. To
+/// create a namespace, use |nvim_create_namespace| which returns a namespace
+/// id. Pass it in to this function as `ns_id` to add highlights to the
+/// namespace. All highlights in the same namespace can then be cleared with
+/// single call to |nvim_buf_clear_highlight|. If the highlight never will be
+/// deleted by an API call, pass `ns_id = -1`.
 ///
-/// If `hl_group` is the empty string no highlight is added, but a new `src_id`
-/// is still returned. This is useful for an external plugin to synchrounously
-/// request an unique `src_id` at initialization, and later asynchronously add
-/// and clear highlights in response to buffer changes.
+/// As a shorthand, `ns_id = 0` can be used to create a new namespace for the
+/// highlight, the allocated id is then returned. If `hl_group` is the empty
+/// string no highlight is added, but a new `ns_id` is still returned. This is
+/// supported for backwards compatibility, new code should use
+/// |nvim_create_namespace| to create a new empty namespace.
 ///
 /// @param buffer     Buffer handle
-/// @param src_id     Source group to use or 0 to use a new group,
-///                   or -1 for ungrouped highlight
+/// @param ns_id      namespace to use or -1 for ungrouped highlight
 /// @param hl_group   Name of the highlight group to use
 /// @param line       Line to highlight (zero-indexed)
 /// @param col_start  Start of (byte-indexed) column range to highlight
 /// @param col_end    End of (byte-indexed) column range to highlight,
 ///                   or -1 to highlight to end of line
 /// @param[out] err   Error details, if any
-/// @return The src_id that was used
+/// @return The ns_id that was used
 Integer nvim_buf_add_highlight(Buffer buffer,
-                               Integer src_id,
+                               Integer ns_id,
                                String hl_group,
                                Integer line,
                                Integer col_start,
@@ -962,9 +962,9 @@ Integer nvim_buf_add_highlight(Buffer buffer,
     hlg_id = syn_check_group((char_u *)hl_group.data, (int)hl_group.size);
   }
 
-  src_id = bufhl_add_hl(buf, (int)src_id, hlg_id, (linenr_T)line+1,
-                        (colnr_T)col_start+1, (colnr_T)col_end);
-  return src_id;
+  ns_id = bufhl_add_hl(buf, (int)ns_id, hlg_id, (linenr_T)line+1,
+                       (colnr_T)col_start+1, (colnr_T)col_end);
+  return ns_id;
 }
 
 /// Clears highlights and virtual text from a given source id and range of lines
@@ -973,13 +973,13 @@ Integer nvim_buf_add_highlight(Buffer buffer,
 /// line_start and line_end respectively.
 ///
 /// @param buffer     Buffer handle
-/// @param src_id     Highlight source group to clear, or -1 to clear all.
+/// @param ns_id      Namespace to clear, or -1 to clear all.
 /// @param line_start Start of range of lines to clear
 /// @param line_end   End of range of lines to clear (exclusive) or -1 to clear
 ///                   to end of file.
 /// @param[out] err   Error details, if any
 void nvim_buf_clear_highlight(Buffer buffer,
-                              Integer src_id,
+                              Integer ns_id,
                               Integer line_start,
                               Integer line_end,
                               Error *err)
@@ -998,7 +998,7 @@ void nvim_buf_clear_highlight(Buffer buffer,
     line_end = MAXLNUM;
   }
 
-  bufhl_clear_line_range(buf, (int)src_id, (int)line_start+1, (int)line_end);
+  bufhl_clear_line_range(buf, (int)ns_id, (int)line_start+1, (int)line_end);
 }
 
 
@@ -1010,12 +1010,18 @@ void nvim_buf_clear_highlight(Buffer buffer,
 /// begin after one cell to the right of the ordinary text, this will contain
 /// the |lcs-eol| char if set, otherwise just be a space.
 ///
-/// The same src_id can be used for both virtual text and highlights added by
-/// nvim_buf_add_highlight. Virtual text is cleared using
-/// nvim_buf_clear_highlight.
+/// Namespaces are used to support batch deletion/updating of virtual text.
+/// To create a namespace, use |nvim_create_namespace|. Virtual text is
+/// cleared using |nvim_buf_clear_highlight|. The same `ns_id` can be used for
+/// both virtual text and highlights added by |nvim_buf_add_highlight|, both
+/// can then be cleared with a single call to |nvim_buf_clear_highlight|. If the
+/// virtual text never will be cleared by an API call, pass `src_id = -1`.
+///
+/// As a shorthand, `ns_id = 0` can be used to create a new namespace for the
+/// virtual text, the allocated id is then returned.
 ///
 /// @param buffer     Buffer handle
-/// @param src_id     Source group to use or 0 to use a new group,
+/// @param ns_id      Namespace to use or 0 to create a namespace,
 ///                   or -1 for a ungrouped annotation
 /// @param line       Line to annotate with virtual text (zero-indexed)
 /// @param chunks     A list of [text, hl_group] arrays, each representing a
@@ -1023,9 +1029,9 @@ void nvim_buf_clear_highlight(Buffer buffer,
 ///                   can be omitted for no highlight.
 /// @param opts       Optional parameters. Currently not used.
 /// @param[out] err   Error details, if any
-/// @return The src_id that was used
+/// @return The ns_id that was used
 Integer nvim_buf_set_virtual_text(Buffer buffer,
-                                  Integer src_id,
+                                  Integer ns_id,
                                   Integer line,
                                   Array chunks,
                                   Dictionary opts,
@@ -1075,9 +1081,9 @@ Integer nvim_buf_set_virtual_text(Buffer buffer,
     kv_push(virt_text, ((VirtTextChunk){ .text = text, .hl_id = hl_id }));
   }
 
-  src_id = bufhl_add_virt_text(buf, (int)src_id, (linenr_T)line+1,
-                               virt_text);
-  return src_id;
+  ns_id = bufhl_add_virt_text(buf, (int)ns_id, (linenr_T)line+1,
+                              virt_text);
+  return ns_id;
 
 free_exit:
   kv_destroy(virt_text);
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 7fcccfd988..54b27477e0 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -46,6 +46,24 @@
 # include "api/vim.c.generated.h"
 #endif
 
+void api_vim_init(void)
+  FUNC_API_NOEXPORT
+{
+  namespace_ids = map_new(String, handle_T)();
+}
+
+void api_vim_free_all_mem(void)
+  FUNC_API_NOEXPORT
+{
+  String name;
+  handle_T id;
+  map_foreach(namespace_ids, name, id, {
+    (void)id;
+    xfree(name.data);
+  })
+  map_free(String, handle_T)(namespace_ids);
+}
+
 /// Executes an ex-command.
 ///
 /// On execution error: fails with VimL error, does not update v:errmsg.
@@ -884,6 +902,49 @@ void nvim_set_current_tabpage(Tabpage tabpage, Error *err)
   }
 }
 
+/// create a new namespace, or get one with an exisiting name
+///
+/// Namespaces are currently used for buffer highlighting and virtual text, see
+/// |nvim_buf_add_highlight| and |nvim_buf_set_virtual_text|.
+///
+/// Namespaces can have a name of be anonymous. If `name` is a non-empty string,
+/// and a namespace already exists with that name,the existing namespace id is
+/// returned. If an empty string is used, a new anonymous namespace is returned.
+///
+/// @param name Name of the namespace or empty string
+/// @return the namespace id
+Integer nvim_create_namespace(String name)
+  FUNC_API_SINCE(5)
+{
+  handle_T id = map_get(String, handle_T)(namespace_ids, name);
+  if (id > 0) {
+    return id;
+  }
+  id = next_namespace_id++;
+  if (name.size > 0) {
+    String name_alloc = copy_string(name);
+    map_put(String, handle_T)(namespace_ids, name_alloc, id);
+  }
+  return (Integer)id;
+}
+
+/// Get existing named namespaces
+///
+/// @return dict that maps from names to namespace ids.
+Dictionary nvim_get_namespaces(void)
+  FUNC_API_SINCE(5)
+{
+  Dictionary retval = ARRAY_DICT_INIT;
+  String name;
+  handle_T id;
+
+  map_foreach(namespace_ids, name, id, {
+    PUT(retval, name.data, INTEGER_OBJ(id));
+  })
+
+  return retval;
+}
+
 /// Subscribes to event broadcasts
 ///
 /// @param channel_id Channel id (passed automatically by the dispatcher)
diff --git a/src/nvim/api/vim.h b/src/nvim/api/vim.h
index 5e0b35b562..d6873da6d2 100644
--- a/src/nvim/api/vim.h
+++ b/src/nvim/api/vim.h
@@ -4,6 +4,10 @@
 #include <stdint.h>
 
 #include "nvim/api/private/defs.h"
+#include "nvim/map.h"
+
+EXTERN Map(String, handle_T) *namespace_ids INIT(= NULL);
+EXTERN handle_T next_namespace_id INIT(= 1);
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "api/vim.h.generated.h"
diff --git a/src/nvim/buffer.c b/src/nvim/buffer.c
index cd77f48320..634e257d40 100644
--- a/src/nvim/buffer.c
+++ b/src/nvim/buffer.c
@@ -26,6 +26,7 @@
 
 #include "nvim/api/private/handle.h"
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/vim.h"
 #include "nvim/ascii.h"
 #include "nvim/assert.h"
 #include "nvim/vim.h"
@@ -5327,10 +5328,10 @@ int bufhl_add_hl(buf_T *buf,
                  int hl_id,
                  linenr_T lnum,
                  colnr_T col_start,
-                 colnr_T col_end) {
-  static int next_src_id = 1;
+                 colnr_T col_end)
+{
   if (src_id == 0) {
-    src_id = next_src_id++;
+    src_id = (int)nvim_create_namespace((String)STRING_INIT);
   }
   if (hl_id <= 0) {
       // no highlight group or invalid line, just return src_id
diff --git a/src/nvim/channel.c b/src/nvim/channel.c
index e191f838e0..8b8d27affd 100644
--- a/src/nvim/channel.c
+++ b/src/nvim/channel.c
@@ -161,7 +161,6 @@ void channel_init(void)
   channels = pmap_new(uint64_t)();
   channel_alloc(kChannelStreamStderr);
   rpc_init();
-  remote_ui_init();
 }
 
 /// Allocates a channel.
diff --git a/src/nvim/main.c b/src/nvim/main.c
index af54e62393..911e51407d 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -73,6 +73,7 @@
 #ifndef WIN32
 # include "nvim/os/pty_process_unix.h"
 #endif
+#include "nvim/api/vim.h"
 
 // Maximum number of commands from + or -c arguments.
 #define MAX_ARG_CMDS 10
@@ -150,6 +151,8 @@ void event_init(void)
   signal_init();
   // finish mspgack-rpc initialization
   channel_init();
+  remote_ui_init();
+  api_vim_init();
   terminal_init();
 }
 
diff --git a/src/nvim/map.c b/src/nvim/map.c
index cc264f3729..53ab734802 100644
--- a/src/nvim/map.c
+++ b/src/nvim/map.c
@@ -166,3 +166,4 @@ MAP_IMPL(handle_T, ptr_t, DEFAULT_INITIALIZER)
 MAP_IMPL(String, MsgpackRpcRequestHandler, MSGPACK_HANDLER_INITIALIZER)
 #define KVEC_INITIALIZER { .size = 0, .capacity = 0, .items = NULL }
 MAP_IMPL(HlEntry, int, DEFAULT_INITIALIZER)
+MAP_IMPL(String, handle_T, 0)
diff --git a/src/nvim/map.h b/src/nvim/map.h
index 65204a798b..0e4308b953 100644
--- a/src/nvim/map.h
+++ b/src/nvim/map.h
@@ -37,6 +37,7 @@ MAP_DECLS(uint64_t, ptr_t)
 MAP_DECLS(handle_T, ptr_t)
 MAP_DECLS(String, MsgpackRpcRequestHandler)
 MAP_DECLS(HlEntry, int)
+MAP_DECLS(String, handle_T)
 
 #define map_new(T, U) map_##T##_##U##_new
 #define map_free(T, U) map_##T##_##U##_free
diff --git a/src/nvim/memory.c b/src/nvim/memory.c
index 8789075c44..6b96a3b070 100644
--- a/src/nvim/memory.c
+++ b/src/nvim/memory.c
@@ -16,6 +16,7 @@
 #include "nvim/message.h"
 #include "nvim/misc1.h"
 #include "nvim/ui.h"
+#include "nvim/api/vim.h"
 
 #ifdef HAVE_JEMALLOC
 // Force je_ prefix on jemalloc functions.
@@ -681,6 +682,7 @@ void free_all_mem(void)
       break;
 
   eval_clear();
+  api_vim_free_all_mem();
 
   // Free all buffers.  Reset 'autochdir' to avoid accessing things that
   // were freed already.
diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua
index a9d137391e..dde4dab7ac 100644
--- a/test/functional/api/vim_spec.lua
+++ b/test/functional/api/vim_spec.lua
@@ -1269,4 +1269,16 @@ describe('API', function()
     end)
   end)
 
+  describe('nvim_create_namespace', function()
+    it('works', function()
+      eq({}, meths.get_namespaces())
+      eq(1, meths.create_namespace("ns-1"))
+      eq(2, meths.create_namespace("ns-2"))
+      eq(1, meths.create_namespace("ns-1"))
+      eq({["ns-1"]=1, ["ns-2"]=2}, meths.get_namespaces())
+      eq(3, meths.create_namespace(""))
+      eq(4, meths.create_namespace(""))
+      eq({["ns-1"]=1, ["ns-2"]=2}, meths.get_namespaces())
+    end)
+  end)
 end)