feat(extmarks): extend nvim_buf_get_extmarks()

Problem:    Can not get all extmarks in a buffer. Properties are missing
            from the details array.
Solution:   Allow getting all extmarks in a buffer by supplying a -1
            "ns_id". Add missing properties to the details array.

7 files changed, 200 insertions, 48 deletions

diff --git a/runtime/doc/api.txt b/runtime/doc/api.txt
index ea57db22e0..cc887ad024 100644
--- a/runtime/doc/api.txt
+++ b/runtime/doc/api.txt
@@ -2527,6 +2527,8 @@ nvim_buf_get_extmark_by_id({buffer}, {ns_id}, {id}, {opts})
       • {id}      Extmark id
       • {opts}    Optional parameters. Keys:
                   • details: Whether to include the details dict
+                  • hl_name: Whether to include highlight group name instead
+                    of id, true if omitted
 
     Return: ~
         0-indexed (row, col) tuple or empty list () if extmark id was absent
@@ -2563,7 +2565,8 @@ nvim_buf_get_extmarks({buffer}, {ns_id}, {start}, {end}, {opts})
 
     Parameters: ~
       • {buffer}  Buffer handle, or 0 for current buffer
-      • {ns_id}   Namespace id from |nvim_create_namespace()|
+      • {ns_id}   Namespace id from |nvim_create_namespace()| or -1 for all
+                  namespaces
       • {start}   Start of range: a 0-indexed (row, col) or valid extmark id
                   (whose position defines the bound). |api-indexing|
       • {end}     End of range (inclusive): a 0-indexed (row, col) or valid
@@ -2571,7 +2574,11 @@ nvim_buf_get_extmarks({buffer}, {ns_id}, {start}, {end}, {opts})
                   |api-indexing|
       • {opts}    Optional parameters. Keys:
                   • limit: Maximum number of marks to return
-                  • details Whether to include the details dict
+                  • details: Whether to include the details dict
+                  • hl_name: Whether to include highlight group name instead
+                    of id, true if omitted
+                  • type: Filter marks by type: "highlight", "sign",
+                    "virt_text" and "virt_lines"
 
     Return: ~
         List of [extmark_id, row, col] tuples in "traversal order".
diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt
index 890a033268..7efb0ab36f 100644
--- a/runtime/doc/news.txt
+++ b/runtime/doc/news.txt
@@ -67,6 +67,11 @@ NEW FEATURES                                                    *news-features*
 
 The following new APIs or features were added.
 
+• |nvim_buf_get_extmarks()| now accepts a -1 `ns_id` to request extmarks from
+  all namespaces and adds the namespace id to the details array.
+  Other missing properties have been added to the details array and marks can
+  be filtered by type.
+
 • Added a new experimental |lua-loader| that byte-compiles and caches lua files.
   To enable the new loader, add the following at the top of your |init.lua|: >lua
     vim.loader.enable()
diff --git a/src/nvim/api/extmark.c b/src/nvim/api/extmark.c
index adc2925b7e..4afbdfa487 100644
--- a/src/nvim/api/extmark.c
+++ b/src/nvim/api/extmark.c
@@ -105,7 +105,16 @@ bool ns_initialized(uint32_t ns)
   return ns < (uint32_t)next_namespace_id;
 }
 
-static Array extmark_to_array(const ExtmarkInfo *extmark, bool id, bool add_dict)
+static Object hl_group_name(int hl_id, bool hl_name)
+{
+  if (hl_name) {
+    return STRING_OBJ(cstr_to_string(syn_id2name(hl_id)));
+  } else {
+    return INTEGER_OBJ(hl_id);
+  }
+}
+
+static Array extmark_to_array(const ExtmarkInfo *extmark, bool id, bool add_dict, bool hl_name)
 {
   Array rv = ARRAY_DICT_INIT;
   if (id) {
@@ -117,6 +126,8 @@ static Array extmark_to_array(const ExtmarkInfo *extmark, bool id, bool add_dict
   if (add_dict) {
     Dictionary dict = ARRAY_DICT_INIT;
 
+    PUT(dict, "ns_id", INTEGER_OBJ((Integer)extmark->ns_id));
+
     PUT(dict, "right_gravity", BOOLEAN_OBJ(extmark->right_gravity));
 
     if (extmark->end_row >= 0) {
@@ -127,8 +138,7 @@ static Array extmark_to_array(const ExtmarkInfo *extmark, bool id, bool add_dict
 
     const Decoration *decor = &extmark->decor;
     if (decor->hl_id) {
-      String name = cstr_to_string((const char *)syn_id2name(decor->hl_id));
-      PUT(dict, "hl_group", STRING_OBJ(name));
+      PUT(dict, "hl_group", hl_group_name(decor->hl_id, hl_name));
       PUT(dict, "hl_eol", BOOLEAN_OBJ(decor->hl_eol));
     }
     if (decor->hl_mode) {
@@ -142,8 +152,7 @@ static Array extmark_to_array(const ExtmarkInfo *extmark, bool id, bool add_dict
         VirtTextChunk *vtc = &decor->virt_text.items[i];
         ADD(chunk, STRING_OBJ(cstr_to_string(vtc->text)));
         if (vtc->hl_id > 0) {
-          ADD(chunk,
-              STRING_OBJ(cstr_to_string((const char *)syn_id2name(vtc->hl_id))));
+          ADD(chunk, hl_group_name(vtc->hl_id, hl_name));
         }
         ADD(chunks, ARRAY_OBJ(chunk));
       }
@@ -172,8 +181,7 @@ static Array extmark_to_array(const ExtmarkInfo *extmark, bool id, bool add_dict
           VirtTextChunk *vtc = &vt->items[j];
           ADD(chunk, STRING_OBJ(cstr_to_string(vtc->text)));
           if (vtc->hl_id > 0) {
-            ADD(chunk,
-                STRING_OBJ(cstr_to_string((const char *)syn_id2name(vtc->hl_id))));
+            ADD(chunk, hl_group_name(vtc->hl_id, hl_name));
           }
           ADD(chunks, ARRAY_OBJ(chunk));
         }
@@ -184,10 +192,44 @@ static Array extmark_to_array(const ExtmarkInfo *extmark, bool id, bool add_dict
       PUT(dict, "virt_lines_leftcol", BOOLEAN_OBJ(virt_lines_leftcol));
     }
 
-    if (decor->hl_id || kv_size(decor->virt_text) || decor->ui_watched) {
+    if (decor->sign_text) {
+      PUT(dict, "sign_text", STRING_OBJ(cstr_to_string(decor->sign_text)));
+    }
+
+    // uncrustify:off
+
+    struct { char *name; const int val; } hls[] = {
+      { "sign_hl_group"      , decor->sign_hl_id       },
+      { "number_hl_group"    , decor->number_hl_id     },
+      { "line_hl_group"      , decor->line_hl_id       },
+      { "cursorline_hl_group", decor->cursorline_hl_id },
+      { NULL, 0 },
+    };
+
+    // uncrustify:on
+
+    for (int j = 0; hls[j].name && hls[j].val; j++) {
+      if (hls[j].val) {
+        PUT(dict, hls[j].name, hl_group_name(hls[j].val, hl_name));
+      }
+    }
+
+    if (decor->sign_text
+        || decor->hl_id
+        || kv_size(decor->virt_text)
+        || decor->ui_watched) {
       PUT(dict, "priority", INTEGER_OBJ(decor->priority));
     }
 
+    if (decor->conceal) {
+      String name = cstr_to_string((char *)&decor->conceal_char);
+      PUT(dict, "conceal", STRING_OBJ(name));
+    }
+
+    if (decor->spell != kNone) {
+      PUT(dict, "spell", BOOLEAN_OBJ(decor->spell == kTrue));
+    }
+
     if (dict.size) {
       ADD(rv, DICTIONARY_OBJ(dict));
     }
@@ -203,6 +245,7 @@ static Array extmark_to_array(const ExtmarkInfo *extmark, bool id, bool add_dict
 /// @param id  Extmark id
 /// @param opts  Optional parameters. Keys:
 ///          - details: Whether to include the details dict
+///          - hl_name: Whether to include highlight group name instead of id, true if omitted
 /// @param[out] err   Error details, if any
 /// @return 0-indexed (row, col) tuple or empty list () if extmark id was
 /// absent
@@ -224,18 +267,19 @@ ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
   });
 
   bool details = false;
+  bool hl_name = true;
   for (size_t i = 0; i < opts.size; i++) {
     String k = opts.items[i].key;
     Object *v = &opts.items[i].value;
     if (strequal("details", k.data)) {
-      if (v->type == kObjectTypeBoolean) {
-        details = v->data.boolean;
-      } else if (v->type == kObjectTypeInteger) {
-        details = v->data.integer;
-      } else {
-        VALIDATE_EXP(false, "details", "Boolean or Integer", api_typename(v->type), {
-          return rv;
-        });
+      details = api_object_to_bool(*v, "details", false, err);
+      if (ERROR_SET(err)) {
+        return rv;
+      }
+    } else if (strequal("hl_name", k.data)) {
+      hl_name = api_object_to_bool(*v, "hl_name", false, err);
+      if (ERROR_SET(err)) {
+        return rv;
       }
     } else {
       VALIDATE_S(false, "'opts' key", k.data, {
@@ -248,7 +292,7 @@ ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
   if (extmark.row < 0) {
     return rv;
   }
-  return extmark_to_array(&extmark, false, details);
+  return extmark_to_array(&extmark, false, details, hl_name);
 }
 
 /// Gets |extmarks| in "traversal order" from a |charwise| region defined by
@@ -282,14 +326,16 @@ ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
 /// </pre>
 ///
 /// @param buffer  Buffer handle, or 0 for current buffer
-/// @param ns_id  Namespace id from |nvim_create_namespace()|
+/// @param ns_id  Namespace id from |nvim_create_namespace()| or -1 for all namespaces
 /// @param start  Start of range: a 0-indexed (row, col) or valid extmark id
 /// (whose position defines the bound). |api-indexing|
 /// @param end  End of range (inclusive): a 0-indexed (row, col) or valid
 /// extmark id (whose position defines the bound). |api-indexing|
 /// @param opts  Optional parameters. Keys:
 ///          - limit:  Maximum number of marks to return
-///          - details Whether to include the details dict
+///          - details: Whether to include the details dict
+///          - hl_name: Whether to include highlight group name instead of id, true if omitted
+///          - type: Filter marks by type: "highlight", "sign", "virt_text" and "virt_lines"
 /// @param[out] err   Error details, if any
 /// @return List of [extmark_id, row, col] tuples in "traversal order".
 Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start, Object end, Dictionary opts,
@@ -303,12 +349,20 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start, Object e
     return rv;
   }
 
-  VALIDATE_INT(ns_initialized((uint32_t)ns_id), "ns_id", ns_id, {
-    return rv;
-  });
+  bool all_ns;
+  if (ns_id == -1) {
+    all_ns = true;
+  } else {
+    VALIDATE_INT(ns_initialized((uint32_t)ns_id), "ns_id", ns_id, {
+      return rv;
+    });
+    all_ns = false;
+  }
 
   Integer limit = -1;
   bool details = false;
+  bool hl_name = true;
+  ExtmarkType type = kExtmarkNone;
 
   for (size_t i = 0; i < opts.size; i++) {
     String k = opts.items[i].key;
@@ -319,12 +373,29 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start, Object e
       });
       limit = v->data.integer;
     } else if (strequal("details", k.data)) {
-      if (v->type == kObjectTypeBoolean) {
-        details = v->data.boolean;
-      } else if (v->type == kObjectTypeInteger) {
-        details = v->data.integer;
+      details = api_object_to_bool(*v, "details", false, err);
+      if (ERROR_SET(err)) {
+        return rv;
+      }
+    } else if (strequal("hl_name", k.data)) {
+      hl_name = api_object_to_bool(*v, "hl_name", false, err);
+      if (ERROR_SET(err)) {
+        return rv;
+      }
+    } else if (strequal("type", k.data)) {
+      VALIDATE_EXP(v->type == kObjectTypeString, "type", "String", api_typename(v->type), {
+        return rv;
+      });
+      if (strequal(v->data.string.data, "sign")) {
+        type = kExtmarkSign;
+      } else if (strequal(v->data.string.data, "virt_text")) {
+        type = kExtmarkVirtText;
+      } else if (strequal(v->data.string.data, "virt_lines")) {
+        type = kExtmarkVirtLines;
+      } else if (strequal(v->data.string.data, "highlight")) {
+        type = kExtmarkHighlight;
       } else {
-        VALIDATE_EXP(false, "details", "Boolean or Integer", api_typename(v->type), {
+        VALIDATE_EXP(false, "type", "sign, virt_text, virt_lines or highlight", v->data.string.data, {
           return rv;
         });
       }
@@ -359,11 +430,11 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start, Object e
     reverse = true;
   }
 
-  ExtmarkInfoArray marks = extmark_get(buf, (uint32_t)ns_id, l_row, l_col,
-                                       u_row, u_col, (int64_t)limit, reverse);
+  ExtmarkInfoArray marks = extmark_get(buf, (uint32_t)ns_id, l_row, l_col, u_row,
+                                       u_col, (int64_t)limit, reverse, all_ns, type);
 
   for (size_t i = 0; i < kv_size(marks); i++) {
-    ADD(rv, ARRAY_OBJ(extmark_to_array(&kv_A(marks, i), true, (bool)details)));
+    ADD(rv, ARRAY_OBJ(extmark_to_array(&kv_A(marks, i), true, (bool)details, hl_name)));
   }
 
   kv_destroy(marks);
diff --git a/src/nvim/extmark.c b/src/nvim/extmark.c
index d385a2018f..32050c5b41 100644
--- a/src/nvim/extmark.c
+++ b/src/nvim/extmark.c
@@ -301,7 +301,8 @@ bool extmark_clear(buf_T *buf, uint32_t ns_id, int l_row, colnr_T l_col, int u_r
 /// dir can be set to control the order of the array
 /// amount = amount of marks to find or -1 for all
 ExtmarkInfoArray extmark_get(buf_T *buf, uint32_t ns_id, int l_row, colnr_T l_col, int u_row,
-                             colnr_T u_col, int64_t amount, bool reverse)
+                             colnr_T u_col, int64_t amount, bool reverse, bool all_ns,
+                             ExtmarkType type_filter)
 {
   ExtmarkInfoArray array = KV_INITIAL_VALUE;
   MarkTreeIter itr[1];
@@ -320,7 +321,25 @@ ExtmarkInfoArray extmark_get(buf_T *buf, uint32_t ns_id, int l_row, colnr_T l_co
       goto next_mark;
     }
 
-    if (mark.ns == ns_id) {
+    uint16_t type_flags = kExtmarkNone;
+    if (type_filter != kExtmarkNone) {
+      Decoration *decor = mark.decor_full;
+      if (decor && (decor->sign_text || decor->number_hl_id)) {
+        type_flags |= kExtmarkSign;
+      }
+      if (decor && decor->virt_text.size) {
+        type_flags |= kExtmarkVirtText;
+      }
+      if (decor && decor->virt_lines.size) {
+        type_flags |= kExtmarkVirtLines;
+      }
+      if ((decor && (decor->line_hl_id || decor->cursorline_hl_id))
+          || mark.hl_id) {
+        type_flags |= kExtmarkHighlight;
+      }
+    }
+
+    if ((all_ns || mark.ns == ns_id) && type_flags & type_filter) {
       mtkey_t end = marktree_get_alt(buf->b_marktree, mark, NULL);
       kv_push(array, ((ExtmarkInfo) { .ns_id = mark.ns,
                                       .mark_id = mark.id,
diff --git a/src/nvim/extmark.h b/src/nvim/extmark.h
index 657e99a938..0b1f22aa84 100644
--- a/src/nvim/extmark.h
+++ b/src/nvim/extmark.h
@@ -76,6 +76,14 @@ typedef enum {
   kExtmarkClear,
 } UndoObjectType;
 
+typedef enum {
+  kExtmarkNone = 0x1,
+  kExtmarkSign = 0x2,
+  kExtmarkVirtText = 0x4,
+  kExtmarkVirtLines = 0x8,
+  kExtmarkHighlight = 0x10,
+} ExtmarkType;
+
 // TODO(bfredl): reduce the number of undo action types
 struct undo_object {
   UndoObjectType type;
diff --git a/test/functional/api/extmark_spec.lua b/test/functional/api/extmark_spec.lua
index 3882fc90ca..4bd8f51904 100644
--- a/test/functional/api/extmark_spec.lua
+++ b/test/functional/api/extmark_spec.lua
@@ -1463,6 +1463,7 @@ describe('API/extmarks', function()
       end_line = 1
     })
     eq({ {1, 0, 0, {
+      ns_id = 1,
       end_col = 0,
       end_row = 1,
       right_gravity = true,
@@ -1480,20 +1481,27 @@ describe('API/extmarks', function()
 
   it('can get details', function()
     set_extmark(ns, marks[1], 0, 0, {
+      conceal = "c",
+      cursorline_hl_group = "Statement",
       end_col = 0,
-      end_row = 1,
-      right_gravity = false,
       end_right_gravity = true,
-      priority = 0,
+      end_row = 1,
       hl_eol = true,
-      hl_mode = "blend",
       hl_group = "String",
-      virt_text = { { "text", "Statement" } },
-      virt_text_pos = "right_align",
-      virt_text_hide = true,
+      hl_mode = "blend",
+      line_hl_group = "Statement",
+      number_hl_group = "Statement",
+      priority = 0,
+      right_gravity = false,
+      sign_hl_group = "Statement",
+      sign_text = ">>",
+      spell = true,
       virt_lines = { { { "lines", "Statement" } }},
       virt_lines_above = true,
       virt_lines_leftcol = true,
+      virt_text = { { "text", "Statement" } },
+      virt_text_hide = true,
+      virt_text_pos = "right_align",
     })
     set_extmark(ns, marks[2], 0, 0, {
       priority = 0,
@@ -1501,22 +1509,31 @@ describe('API/extmarks', function()
       virt_text_win_col = 1,
     })
     eq({0, 0, {
+      conceal = "c",
+      cursorline_hl_group = "Statement",
       end_col = 0,
-      end_row = 1,
-      right_gravity = false,
       end_right_gravity = true,
-      priority = 0,
+      end_row = 1,
       hl_eol = true,
-      hl_mode = "blend",
       hl_group = "String",
-      virt_text = { { "text", "Statement" } },
-      virt_text_pos = "right_align",
-      virt_text_hide = true,
+      hl_mode = "blend",
+      line_hl_group = "Statement",
+      ns_id = 1,
+      number_hl_group = "Statement",
+      priority = 0,
+      right_gravity = false,
+      sign_hl_group = "Statement",
+      sign_text = ">>",
+      spell = true,
       virt_lines = { { { "lines", "Statement" } }},
       virt_lines_above = true,
       virt_lines_leftcol = true,
+      virt_text = { { "text", "Statement" } },
+      virt_text_hide = true,
+      virt_text_pos = "right_align",
     } }, get_extmark_by_id(ns, marks[1], { details = true }))
     eq({0, 0, {
+      ns_id = 1,
       right_gravity = true,
       priority = 0,
       virt_text = { { "text", "Statement" } },
@@ -1525,6 +1542,29 @@ describe('API/extmarks', function()
       virt_text_win_col = 1,
     } }, get_extmark_by_id(ns, marks[2], { details = true }))
   end)
+
+  it('can get marks from anonymous namespaces', function()
+    ns = request('nvim_create_namespace', "")
+    ns2 = request('nvim_create_namespace', "")
+    set_extmark(ns, 1, 0, 0, {})
+    set_extmark(ns2, 2, 1, 0, {})
+    eq({{ 1, 0, 0, { ns_id = ns, right_gravity = true }},
+        { 2, 1, 0, { ns_id = ns2, right_gravity = true }}},
+        get_extmarks(-1, 0, -1, { details = true }))
+  end)
+
+  it('can filter by extmark properties', function()
+    set_extmark(ns, 1, 0, 0, {})
+    set_extmark(ns, 2, 0, 0, { hl_group = 'Normal' })
+    set_extmark(ns, 3, 0, 0, { sign_text = '>>' })
+    set_extmark(ns, 4, 0, 0, { virt_text = {{'text', 'Normal'}}})
+    set_extmark(ns, 5, 0, 0, { virt_lines = {{{ 'line', 'Normal' }}}})
+    eq(5, #get_extmarks(-1, 0, -1, { details = true }))
+    eq({{ 2, 0, 0 }}, get_extmarks(-1, 0, -1, { type = 'highlight' }))
+    eq({{ 3, 0, 0 }}, get_extmarks(-1, 0, -1, { type = 'sign' }))
+    eq({{ 4, 0, 0 }}, get_extmarks(-1, 0, -1, { type = 'virt_text' }))
+    eq({{ 5, 0, 0 }}, get_extmarks(-1, 0, -1, { type = 'virt_lines' }))
+  end)
 end)
 
 describe('Extmarks buffer api with many marks', function()
diff --git a/test/functional/ui/bufhl_spec.lua b/test/functional/ui/bufhl_spec.lua
index adec0770de..5263fb4c24 100644
--- a/test/functional/ui/bufhl_spec.lua
+++ b/test/functional/ui/bufhl_spec.lua
@@ -766,6 +766,7 @@ describe('Buffer highlighting', function()
       -- an existing virtual text. We might add a prioritation system.
       set_virtual_text(id1, 0, s1, {})
       eq({{1, 0, 0, {
+        ns_id = 1,
         priority = 0,
         virt_text = s1,
         -- other details
@@ -778,6 +779,7 @@ describe('Buffer highlighting', function()
       local lastline = line_count()
       set_virtual_text(id1, line_count(), s2, {})
       eq({{3, lastline, 0, {
+        ns_id = 1,
         priority = 0,
         virt_text = s2,
         -- other details