api/msgpack-rpc: Refactor metadata object construction

Instead of building all metadata from msgpack-gen.lua, we now merge the
generated part with manual information(such as types and features). The metadata
is accessible through the api method `vim_get_api_info`.

This was done to simplify the generator while also increasing flexibility(by
being able to add more metadata)

8 files changed, 94 insertions, 34 deletions

diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index 405cd20486..83de3347bd 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -3,7 +3,6 @@ include(CheckLibraryExists)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
 set(DISPATCH_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/msgpack-gen.lua)
 file(GLOB API_HEADERS api/*.h)
-file(GLOB API_DEFS api/private/defs.h)
 set(MSGPACK_RPC_HEADER ${PROJECT_SOURCE_DIR}/src/nvim/os/msgpack_rpc.h)
 set(MSGPACK_DISPATCH ${GENERATED_DIR}/msgpack_dispatch.c)
 set(HEADER_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendeclarations.lua)
@@ -124,10 +123,9 @@ foreach(sfile ${NEOVIM_SOURCES}
 endforeach()
 
 add_custom_command(OUTPUT ${MSGPACK_DISPATCH}
-  COMMAND ${LUA_PRG} ${DISPATCH_GENERATOR} ${API_DEFS} ${API_HEADERS} ${MSGPACK_DISPATCH}
+  COMMAND ${LUA_PRG} ${DISPATCH_GENERATOR} ${API_HEADERS} ${MSGPACK_DISPATCH}
   DEPENDS
     ${API_HEADERS}
-    ${API_DEFS}
     ${MSGPACK_RPC_HEADER}
     ${DISPATCH_GENERATOR}
 )
diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index bae1819172..cf559a372e 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -44,13 +44,9 @@ typedef struct {
 } Dictionary;
 
 typedef enum {
-// The following comments are markers that msgpack-gen.lua uses to extract
-// types, don't remove!
-// start custom types
   kObjectTypeBuffer,
   kObjectTypeWindow,
   kObjectTypeTabpage,
-// end custom types
   kObjectTypeNil,
   kObjectTypeBoolean,
   kObjectTypeInteger,
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index de23481813..14a820aa1b 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -6,6 +6,7 @@
 #include "nvim/api/private/helpers.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/handle.h"
+#include "nvim/os/provider.h"
 #include "nvim/ascii.h"
 #include "nvim/vim.h"
 #include "nvim/buffer.h"
@@ -506,6 +507,72 @@ void api_free_dictionary(Dictionary value)
   free(value.items);
 }
 
+Dictionary api_metadata(void)
+{
+  static Dictionary metadata = ARRAY_DICT_INIT;
+
+  if (!metadata.size) {
+    msgpack_rpc_init_function_metadata(&metadata);
+    init_type_metadata(&metadata);
+    provider_init_feature_metadata(&metadata);
+  }
+
+  return copy_object(DICTIONARY_OBJ(metadata)).data.dictionary;
+}
+
+static void init_type_metadata(Dictionary *metadata)
+{
+  Dictionary types = ARRAY_DICT_INIT;
+
+  Dictionary buffer_metadata = ARRAY_DICT_INIT;
+  PUT(buffer_metadata, "id", INTEGER_OBJ(kObjectTypeBuffer));
+
+  Dictionary window_metadata = ARRAY_DICT_INIT;
+  PUT(window_metadata, "id", INTEGER_OBJ(kObjectTypeWindow));
+
+  Dictionary tabpage_metadata = ARRAY_DICT_INIT;
+  PUT(tabpage_metadata, "id", INTEGER_OBJ(kObjectTypeTabpage));
+
+  PUT(types, "Buffer", DICTIONARY_OBJ(buffer_metadata));
+  PUT(types, "Window", DICTIONARY_OBJ(window_metadata));
+  PUT(types, "Tabpage", DICTIONARY_OBJ(tabpage_metadata));
+
+  PUT(*metadata, "types", DICTIONARY_OBJ(types));
+}
+
+/// Creates a deep clone of an object
+static Object copy_object(Object obj)
+{
+  switch (obj.type) {
+    case kObjectTypeNil:
+    case kObjectTypeBoolean:
+    case kObjectTypeInteger:
+    case kObjectTypeFloat:
+      return obj;
+
+    case kObjectTypeString:
+      return STRING_OBJ(cstr_to_string(obj.data.string.data));
+
+    case kObjectTypeArray: {
+      Array rv = ARRAY_DICT_INIT;
+      for (size_t i = 0; i < obj.data.array.size; i++) {
+        ADD(rv, copy_object(obj.data.array.items[i]));
+      }
+      return ARRAY_OBJ(rv);
+    }
+
+    case kObjectTypeDictionary: {
+      Dictionary rv = ARRAY_DICT_INIT;
+      for (size_t i = 0; i < obj.data.dictionary.size; i++) {
+        KeyValuePair item = obj.data.dictionary.items[i];
+        PUT(rv, item.key.data, copy_object(item.value));
+      }
+      return DICTIONARY_OBJ(rv);
+    }
+    default:
+      abort();
+  }
+}
 
 /// Recursion helper for the `vim_to_object`. This uses a pointer table
 /// to avoid infinite recursion due to cyclic references
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 25454761ea..c0a9fe3410 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -528,10 +528,15 @@ void vim_register_provider(uint64_t channel_id, String feature, Error *err)
   }
 }
 
-/// Returns a feature->method list dictionary for all pluggable features
-Dictionary vim_discover_features(void)
+Array vim_get_api_info(uint64_t channel_id)
 {
-  return provider_get_all();
+  Array rv = ARRAY_DICT_INIT;
+
+  assert(channel_id <= INT64_MAX);
+  ADD(rv, INTEGER_OBJ((int64_t)channel_id));
+  ADD(rv, DICTIONARY_OBJ(api_metadata()));
+
+  return rv;
 }
 
 /// Writes a message to vim output or error buffer. The string is split
diff --git a/src/nvim/main.c b/src/nvim/main.c
index 4b440ba92a..fc1826975a 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -12,6 +12,8 @@
 #include <string.h>
 #include <stdbool.h>
 
+#include <msgpack.h>
+
 #include "nvim/ascii.h"
 #include "nvim/vim.h"
 #include "nvim/main.h"
@@ -57,6 +59,9 @@
 #include "nvim/os/input.h"
 #include "nvim/os/os.h"
 #include "nvim/os/signal.h"
+#include "nvim/os/msgpack_rpc_helpers.h"
+#include "nvim/api/private/defs.h"
+#include "nvim/api/private/helpers.h"
 
 /* Maximum number of commands from + or -c arguments. */
 #define MAX_ARG_CMDS 10
@@ -116,9 +121,6 @@ static void init_locale(void);
 # endif
 #endif /* NO_VIM_MAIN */
 
-extern const uint8_t msgpack_metadata[];
-extern const unsigned int msgpack_metadata_size;
-
 /*
  * Different types of error messages.
  */
@@ -1027,9 +1029,15 @@ static void command_line_scan(mparm_T *parmp)
             msg_didout = FALSE;
             mch_exit(0);
           } else if (STRICMP(argv[0] + argv_idx, "api-info") == 0) {
-            for (unsigned int i = 0; i<msgpack_metadata_size; i++) {
-              putchar(msgpack_metadata[i]);
+            msgpack_sbuffer* b = msgpack_sbuffer_new();
+            msgpack_packer* p = msgpack_packer_new(b, msgpack_sbuffer_write);
+            Object md = DICTIONARY_OBJ(api_metadata());
+            msgpack_rpc_from_object(md, p);
+
+            for (size_t i = 0; i < b->size; i++) {
+              putchar(b->data[i]);
             }
+
             mch_exit(0);
           } else if (STRICMP(argv[0] + argv_idx, "embed") == 0) {
             embedded_mode = true;
diff --git a/src/nvim/os/msgpack_rpc.c b/src/nvim/os/msgpack_rpc.c
index 2f347d9b15..d7e3d33c4b 100644
--- a/src/nvim/os/msgpack_rpc.c
+++ b/src/nvim/os/msgpack_rpc.c
@@ -17,8 +17,6 @@
 # include "os/msgpack_rpc.c.generated.h"
 #endif
 
-extern msgpack_unpacked msgpack_unpacked_metadata;
-
 /// Validates the basic structure of the msgpack-rpc call and fills `res`
 /// with the basic response structure.
 ///
@@ -83,19 +81,6 @@ Object msgpack_rpc_handle_missing_method(uint64_t channel_id,
   return NIL;
 }
 
-/// Handler for retrieving API metadata through a msgpack-rpc call
-Object msgpack_rpc_handle_get_api_metadata(uint64_t channel_id,
-                                           msgpack_object *req,
-                                           Error *error)
-{
-  Array rv = ARRAY_DICT_INIT;
-  Object metadata;
-  msgpack_rpc_to_object(&msgpack_unpacked_metadata.data, &metadata);
-  ADD(rv, INTEGER_OBJ((int64_t)channel_id));
-  ADD(rv, metadata);
-  return ARRAY_OBJ(rv);
-}
-
 /// Serializes a msgpack-rpc request or notification(id == 0)
 WBuffer *serialize_request(uint64_t request_id,
                            String method,
diff --git a/src/nvim/os/msgpack_rpc.h b/src/nvim/os/msgpack_rpc.h
index 35f175d2a0..3476d791ea 100644
--- a/src/nvim/os/msgpack_rpc.h
+++ b/src/nvim/os/msgpack_rpc.h
@@ -25,6 +25,7 @@ typedef Object (*rpc_method_handler_fn)(uint64_t channel_id,
 /// Initializes the msgpack-rpc method table
 void msgpack_rpc_init(void);
 
+void msgpack_rpc_init_function_metadata(Dictionary *metadata);
 
 /// Dispatches to the actual API function after basic payload validation by
 /// `msgpack_rpc_call`. It is responsible for validating/converting arguments
diff --git a/src/nvim/os/provider.c b/src/nvim/os/provider.c
index 9d8f6f297c..2e7a677793 100644
--- a/src/nvim/os/provider.c
+++ b/src/nvim/os/provider.c
@@ -120,9 +120,9 @@ Object provider_call(char *method, Array args)
   return result;
 }
 
-Dictionary provider_get_all(void)
+void provider_init_feature_metadata(Dictionary *metadata)
 {
-  Dictionary rv = ARRAY_DICT_INIT;
+  Dictionary md = ARRAY_DICT_INIT;
 
   for (size_t i = 0; i < FEATURE_COUNT; i++) {
     Array methods = ARRAY_DICT_INIT;
@@ -134,10 +134,10 @@ Dictionary provider_get_all(void)
       ADD(methods, STRING_OBJ(cstr_to_string(method)));
     }
 
-    PUT(rv, f->name, ARRAY_OBJ(methods));
+    PUT(md, f->name, ARRAY_OBJ(methods));
   }
 
-  return rv;
+  PUT(*metadata, "features", DICTIONARY_OBJ(md));
 }
 
 static Feature * find_feature(char *name)