api/msgpack-rpc: Refactor metadata object construction

Instead of building all metadata from msgpack-gen.lua, we now merge the
generated part with manual information(such as types and features). The metadata
is accessible through the api method `vim_get_api_info`.

This was done to simplify the generator while also increasing flexibility(by
being able to add more metadata)

9 files changed, 120 insertions, 86 deletions

diff --git a/scripts/msgpack-gen.lua b/scripts/msgpack-gen.lua
index 5ef61267f2..8fbf64ebc0 100644
--- a/scripts/msgpack-gen.lua
+++ b/scripts/msgpack-gen.lua
@@ -35,30 +35,7 @@ grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1)
 
 -- we need at least 2 arguments since the last one is the output file
 assert(#arg >= 1)
--- api metadata
-api = {
-  functions = {},
-  types = {}
-}
-
--- Extract type codes from api/private/defs.h. The codes are values between
--- comment markers in the ObjectType enum
-local typedefs_header = arg[1]
-local input = io.open(typedefs_header, 'rb')
-local reading_types = false
-while true do
-  local line = input:read('*l'):gsub("^%s*(.-)%s*$", "%1")
-  if reading_types then
-    if line == '// end custom types' then
-      break
-    end
-    local type_name = line:gsub("^kObjectType(.-),$", "%1")
-    api.types[#api.types + 1] = type_name
-  else
-    reading_types = line == '// start custom types'
-  end
-end
-input:close()
+functions = {}
 
 -- names of all headers relative to the source root(for inclusion in the
 -- generated file)
@@ -67,7 +44,7 @@ headers = {}
 outputf = arg[#arg]
 
 -- read each input file, parse and append to the api metadata
-for i = 2, #arg - 1 do
+for i = 1, #arg - 1 do
   local full_path = arg[i]
   local parts = {}
   for part in string.gmatch(full_path, '[^/]+') do
@@ -78,7 +55,7 @@ for i = 2, #arg - 1 do
   local input = io.open(full_path, 'rb')
   local tmp = grammar:match(input:read('*all'))
   for i = 1, #tmp do
-    api.functions[#api.functions + 1] = tmp[i]
+    functions[#functions + 1] = tmp[i]
     local fn = tmp[i]
     if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
       -- this function should receive the channel id
@@ -124,12 +101,12 @@ end
 output:write([[
 
 
-const uint8_t msgpack_metadata[] = {
+static const uint8_t msgpack_metadata[] = {
 
 ]])
 -- serialize the API metadata using msgpack and embed into the resulting
 -- binary for easy querying by clients
-packed = msgpack.pack(api)
+packed = msgpack.pack(functions)
 for i = 1, #packed do
   output:write(string.byte(packed, i)..', ')
   if i % 10 == 0 then
@@ -138,17 +115,28 @@ for i = 1, #packed do
 end
 output:write([[
 };
-const unsigned int msgpack_metadata_size = sizeof(msgpack_metadata);
-msgpack_unpacked msgpack_unpacked_metadata;
+
+void msgpack_rpc_init_function_metadata(Dictionary *metadata)
+{
+  msgpack_unpacked unpacked;
+  msgpack_unpacked_init(&unpacked);
+  assert(msgpack_unpack_next(&unpacked,
+                             (const char *)msgpack_metadata,
+                             sizeof(msgpack_metadata),
+                             NULL) == MSGPACK_UNPACK_SUCCESS);
+  Object functions;
+  msgpack_rpc_to_object(&unpacked.data, &functions);
+  msgpack_unpacked_destroy(&unpacked);
+  PUT(*metadata, "functions", functions);
+}
 
 ]])
 
--- start the handler functions. First handler (method_id=0) is reserved for
--- querying the metadata, usually it is the first function called by clients.
--- Visit each function metadata to build the handler function with code
--- generated for validating arguments and calling to the real API.
-for i = 1, #api.functions do
-  local fn = api.functions[i]
+-- start the handler functions. Visit each function metadata to build the
+-- handler function with code generated for validating arguments and calling to
+-- the real API.
+for i = 1, #functions do
+  local fn = functions[i]
   local args = {}
 
   output:write('static Object handle_'..fn.name..'(uint64_t channel_id, msgpack_object *req, Error *error)')
@@ -243,14 +231,6 @@ static Map(String, rpc_method_handler_fn) *methods = NULL;
 
 void msgpack_rpc_init(void)
 {
-  msgpack_unpacked_init(&msgpack_unpacked_metadata);
-  if (msgpack_unpack_next(&msgpack_unpacked_metadata,
-                          (const char *)msgpack_metadata,
-                          msgpack_metadata_size,
-                          NULL) != MSGPACK_UNPACK_SUCCESS) {
-    abort();
-  }
-
   methods = map_new(String, rpc_method_handler_fn)();
 
 ]])
@@ -258,8 +238,8 @@ void msgpack_rpc_init(void)
 -- Keep track of the maximum method name length in order to avoid walking
 -- strings longer than that when searching for a method handler
 local max_fname_len = 0
-for i = 1, #api.functions do
-  local fn = api.functions[i]
+for i = 1, #functions do
+  local fn = functions[i]
   output:write('  map_put(String, rpc_method_handler_fn)(methods, '..
                '(String) {.data = "'..fn.name..'", '..
                '.size = sizeof("'..fn.name..'") - 1}, handle_'..
@@ -270,12 +250,6 @@ for i = 1, #api.functions do
   end
 end
 
-local metadata_fn = 'get_api_metadata'
-output:write('  map_put(String, rpc_method_handler_fn)(methods, '..
-             '(String) {.data = "'..metadata_fn..'", '..
-             '.size = sizeof("'..metadata_fn..'") - 1}, msgpack_rpc_handle_'..
-             metadata_fn..');\n')
-
 output:write('\n}\n\n')
 
 output:write([[
diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
index 405cd20486..83de3347bd 100644
--- a/src/nvim/CMakeLists.txt
+++ b/src/nvim/CMakeLists.txt
@@ -3,7 +3,6 @@ include(CheckLibraryExists)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
 set(DISPATCH_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/msgpack-gen.lua)
 file(GLOB API_HEADERS api/*.h)
-file(GLOB API_DEFS api/private/defs.h)
 set(MSGPACK_RPC_HEADER ${PROJECT_SOURCE_DIR}/src/nvim/os/msgpack_rpc.h)
 set(MSGPACK_DISPATCH ${GENERATED_DIR}/msgpack_dispatch.c)
 set(HEADER_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendeclarations.lua)
@@ -124,10 +123,9 @@ foreach(sfile ${NEOVIM_SOURCES}
 endforeach()
 
 add_custom_command(OUTPUT ${MSGPACK_DISPATCH}
-  COMMAND ${LUA_PRG} ${DISPATCH_GENERATOR} ${API_DEFS} ${API_HEADERS} ${MSGPACK_DISPATCH}
+  COMMAND ${LUA_PRG} ${DISPATCH_GENERATOR} ${API_HEADERS} ${MSGPACK_DISPATCH}
   DEPENDS
     ${API_HEADERS}
-    ${API_DEFS}
     ${MSGPACK_RPC_HEADER}
     ${DISPATCH_GENERATOR}
 )
diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index bae1819172..cf559a372e 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -44,13 +44,9 @@ typedef struct {
 } Dictionary;
 
 typedef enum {
-// The following comments are markers that msgpack-gen.lua uses to extract
-// types, don't remove!
-// start custom types
   kObjectTypeBuffer,
   kObjectTypeWindow,
   kObjectTypeTabpage,
-// end custom types
   kObjectTypeNil,
   kObjectTypeBoolean,
   kObjectTypeInteger,
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index de23481813..14a820aa1b 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -6,6 +6,7 @@
 #include "nvim/api/private/helpers.h"
 #include "nvim/api/private/defs.h"
 #include "nvim/api/private/handle.h"
+#include "nvim/os/provider.h"
 #include "nvim/ascii.h"
 #include "nvim/vim.h"
 #include "nvim/buffer.h"
@@ -506,6 +507,72 @@ void api_free_dictionary(Dictionary value)
   free(value.items);
 }
 
+Dictionary api_metadata(void)
+{
+  static Dictionary metadata = ARRAY_DICT_INIT;
+
+  if (!metadata.size) {
+    msgpack_rpc_init_function_metadata(&metadata);
+    init_type_metadata(&metadata);
+    provider_init_feature_metadata(&metadata);
+  }
+
+  return copy_object(DICTIONARY_OBJ(metadata)).data.dictionary;
+}
+
+static void init_type_metadata(Dictionary *metadata)
+{
+  Dictionary types = ARRAY_DICT_INIT;
+
+  Dictionary buffer_metadata = ARRAY_DICT_INIT;
+  PUT(buffer_metadata, "id", INTEGER_OBJ(kObjectTypeBuffer));
+
+  Dictionary window_metadata = ARRAY_DICT_INIT;
+  PUT(window_metadata, "id", INTEGER_OBJ(kObjectTypeWindow));
+
+  Dictionary tabpage_metadata = ARRAY_DICT_INIT;
+  PUT(tabpage_metadata, "id", INTEGER_OBJ(kObjectTypeTabpage));
+
+  PUT(types, "Buffer", DICTIONARY_OBJ(buffer_metadata));
+  PUT(types, "Window", DICTIONARY_OBJ(window_metadata));
+  PUT(types, "Tabpage", DICTIONARY_OBJ(tabpage_metadata));
+
+  PUT(*metadata, "types", DICTIONARY_OBJ(types));
+}
+
+/// Creates a deep clone of an object
+static Object copy_object(Object obj)
+{
+  switch (obj.type) {
+    case kObjectTypeNil:
+    case kObjectTypeBoolean:
+    case kObjectTypeInteger:
+    case kObjectTypeFloat:
+      return obj;
+
+    case kObjectTypeString:
+      return STRING_OBJ(cstr_to_string(obj.data.string.data));
+
+    case kObjectTypeArray: {
+      Array rv = ARRAY_DICT_INIT;
+      for (size_t i = 0; i < obj.data.array.size; i++) {
+        ADD(rv, copy_object(obj.data.array.items[i]));
+      }
+      return ARRAY_OBJ(rv);
+    }
+
+    case kObjectTypeDictionary: {
+      Dictionary rv = ARRAY_DICT_INIT;
+      for (size_t i = 0; i < obj.data.dictionary.size; i++) {
+        KeyValuePair item = obj.data.dictionary.items[i];
+        PUT(rv, item.key.data, copy_object(item.value));
+      }
+      return DICTIONARY_OBJ(rv);
+    }
+    default:
+      abort();
+  }
+}
 
 /// Recursion helper for the `vim_to_object`. This uses a pointer table
 /// to avoid infinite recursion due to cyclic references
diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c
index 25454761ea..c0a9fe3410 100644
--- a/src/nvim/api/vim.c
+++ b/src/nvim/api/vim.c
@@ -528,10 +528,15 @@ void vim_register_provider(uint64_t channel_id, String feature, Error *err)
   }
 }
 
-/// Returns a feature->method list dictionary for all pluggable features
-Dictionary vim_discover_features(void)
+Array vim_get_api_info(uint64_t channel_id)
 {
-  return provider_get_all();
+  Array rv = ARRAY_DICT_INIT;
+
+  assert(channel_id <= INT64_MAX);
+  ADD(rv, INTEGER_OBJ((int64_t)channel_id));
+  ADD(rv, DICTIONARY_OBJ(api_metadata()));
+
+  return rv;
 }
 
 /// Writes a message to vim output or error buffer. The string is split
diff --git a/src/nvim/main.c b/src/nvim/main.c
index 4b440ba92a..fc1826975a 100644
--- a/src/nvim/main.c
+++ b/src/nvim/main.c
@@ -12,6 +12,8 @@
 #include <string.h>
 #include <stdbool.h>
 
+#include <msgpack.h>
+
 #include "nvim/ascii.h"
 #include "nvim/vim.h"
 #include "nvim/main.h"
@@ -57,6 +59,9 @@
 #include "nvim/os/input.h"
 #include "nvim/os/os.h"
 #include "nvim/os/signal.h"
+#include "nvim/os/msgpack_rpc_helpers.h"
+#include "nvim/api/private/defs.h"
+#include "nvim/api/private/helpers.h"
 
 /* Maximum number of commands from + or -c arguments. */
 #define MAX_ARG_CMDS 10
@@ -116,9 +121,6 @@ static void init_locale(void);
 # endif
 #endif /* NO_VIM_MAIN */
 
-extern const uint8_t msgpack_metadata[];
-extern const unsigned int msgpack_metadata_size;
-
 /*
  * Different types of error messages.
  */
@@ -1027,9 +1029,15 @@ static void command_line_scan(mparm_T *parmp)
             msg_didout = FALSE;
             mch_exit(0);
           } else if (STRICMP(argv[0] + argv_idx, "api-info") == 0) {
-            for (unsigned int i = 0; i<msgpack_metadata_size; i++) {
-              putchar(msgpack_metadata[i]);
+            msgpack_sbuffer* b = msgpack_sbuffer_new();
+            msgpack_packer* p = msgpack_packer_new(b, msgpack_sbuffer_write);
+            Object md = DICTIONARY_OBJ(api_metadata());
+            msgpack_rpc_from_object(md, p);
+
+            for (size_t i = 0; i < b->size; i++) {
+              putchar(b->data[i]);
             }
+
             mch_exit(0);
           } else if (STRICMP(argv[0] + argv_idx, "embed") == 0) {
             embedded_mode = true;
diff --git a/src/nvim/os/msgpack_rpc.c b/src/nvim/os/msgpack_rpc.c
index 2f347d9b15..d7e3d33c4b 100644
--- a/src/nvim/os/msgpack_rpc.c
+++ b/src/nvim/os/msgpack_rpc.c
@@ -17,8 +17,6 @@
 # include "os/msgpack_rpc.c.generated.h"
 #endif
 
-extern msgpack_unpacked msgpack_unpacked_metadata;
-
 /// Validates the basic structure of the msgpack-rpc call and fills `res`
 /// with the basic response structure.
 ///
@@ -83,19 +81,6 @@ Object msgpack_rpc_handle_missing_method(uint64_t channel_id,
   return NIL;
 }
 
-/// Handler for retrieving API metadata through a msgpack-rpc call
-Object msgpack_rpc_handle_get_api_metadata(uint64_t channel_id,
-                                           msgpack_object *req,
-                                           Error *error)
-{
-  Array rv = ARRAY_DICT_INIT;
-  Object metadata;
-  msgpack_rpc_to_object(&msgpack_unpacked_metadata.data, &metadata);
-  ADD(rv, INTEGER_OBJ((int64_t)channel_id));
-  ADD(rv, metadata);
-  return ARRAY_OBJ(rv);
-}
-
 /// Serializes a msgpack-rpc request or notification(id == 0)
 WBuffer *serialize_request(uint64_t request_id,
                            String method,
diff --git a/src/nvim/os/msgpack_rpc.h b/src/nvim/os/msgpack_rpc.h
index 35f175d2a0..3476d791ea 100644
--- a/src/nvim/os/msgpack_rpc.h
+++ b/src/nvim/os/msgpack_rpc.h
@@ -25,6 +25,7 @@ typedef Object (*rpc_method_handler_fn)(uint64_t channel_id,
 /// Initializes the msgpack-rpc method table
 void msgpack_rpc_init(void);
 
+void msgpack_rpc_init_function_metadata(Dictionary *metadata);
 
 /// Dispatches to the actual API function after basic payload validation by
 /// `msgpack_rpc_call`. It is responsible for validating/converting arguments
diff --git a/src/nvim/os/provider.c b/src/nvim/os/provider.c
index 9d8f6f297c..2e7a677793 100644
--- a/src/nvim/os/provider.c
+++ b/src/nvim/os/provider.c
@@ -120,9 +120,9 @@ Object provider_call(char *method, Array args)
   return result;
 }
 
-Dictionary provider_get_all(void)
+void provider_init_feature_metadata(Dictionary *metadata)
 {
-  Dictionary rv = ARRAY_DICT_INIT;
+  Dictionary md = ARRAY_DICT_INIT;
 
   for (size_t i = 0; i < FEATURE_COUNT; i++) {
     Array methods = ARRAY_DICT_INIT;
@@ -134,10 +134,10 @@ Dictionary provider_get_all(void)
       ADD(methods, STRING_OBJ(cstr_to_string(method)));
     }
 
-    PUT(rv, f->name, ARRAY_OBJ(methods));
+    PUT(md, f->name, ARRAY_OBJ(methods));
   }
 
-  return rv;
+  PUT(*metadata, "features", DICTIONARY_OBJ(md));
 }
 
 static Feature * find_feature(char *name)