channel/msgpack_rpc: Refactor API dispatching

This is how API dispatching worked before this commit:

- The generated `msgpack_rpc_dispatch` function receives a the `msgpack_packer`
  argument.
- The response is incrementally built while validating/calling the API.
- Return values/errors are also packed into the `msgpack_packer` while the
  final response is being calculated.

Now the `msgpack_packer` argument is no longer provided, and the
`msgpack_rpc_dispatch` function returns `Object`/`Error` values to
`msgpack_rpc_call`, which will use those values to build the response in a
single pass.

This was done because the new `channel_send_call` function created the
possibility of having recursive API invocations, and this wasn't possible when
sharing a single `msgpack_sbuffer` across call frames(it was shared implicitly
through the `msgpack_packer` instance).

Since we only start to build the response when the necessary information has
been computed, it's now safe to share a single `msgpack_sbuffer` instance
across all channels and API invocations.

Some other changes also had to be performed:

- Handling of the metadata discover was moved to `msgpack_rpc_call`
- Expose more types as subtypes of `Object`, this was required to forward the
  return value from `msgpack_rpc_dispatch` to `msgpack_rpc_call`
- Added more helper macros for casting API types to `Object`
  any

1 files changed, 19 insertions, 30 deletions

diff --git a/scripts/msgpack-gen.lua b/scripts/msgpack-gen.lua
index 91bdf3df21..e2cc267191 100644
--- a/scripts/msgpack-gen.lua
+++ b/scripts/msgpack-gen.lua
@@ -92,6 +92,7 @@ output:write([[
 
 #include "nvim/os/msgpack_rpc.h"
 #include "nvim/os/msgpack_rpc_helpers.h"
+#include "nvim/api/private/helpers.h"
 ]])
 
 for i = 1, #headers do
@@ -121,20 +122,13 @@ output:write([[
 };
 const unsigned int msgpack_metadata_size = sizeof(msgpack_metadata);
 
-void msgpack_rpc_dispatch(uint64_t channel_id, msgpack_object *req, msgpack_packer *res)
+Object msgpack_rpc_dispatch(uint64_t channel_id,
+                            uint64_t method_id,
+                            msgpack_object *req,
+                            Error *error)
 {
-  Error error = { .set = false };
-  uint64_t method_id = (uint32_t)req->via.array.ptr[2].via.u64;
-
+  Object ret = NIL;
   switch (method_id) {
-    case 0:
-      msgpack_pack_nil(res);
-      // The result is the [channel_id, metadata] array
-      msgpack_pack_array(res, 2);
-      msgpack_pack_uint64(res, channel_id);
-      msgpack_pack_raw(res, sizeof(msgpack_metadata));
-      msgpack_pack_raw_body(res, msgpack_metadata, sizeof(msgpack_metadata));
-      return;
 ]])
 
 -- Visit each function metadata to build the case label with code generated
@@ -146,8 +140,7 @@ for i = 1, #api.functions do
   output:write('\n    case '..fn.id..': {')
 
   output:write('\n      if (req->via.array.ptr[3].via.array.size != '..#fn.parameters..') {')
-  output:write('\n        snprintf(error.msg, sizeof(error.msg), "Wrong number of arguments: expecting '..#fn.parameters..' but got %u", req->via.array.ptr[3].via.array.size);')
-  output:write('\n        msgpack_rpc_error(error.msg, res);')
+  output:write('\n        snprintf(error->msg, sizeof(error->msg), "Wrong number of arguments: expecting '..#fn.parameters..' but got %u", req->via.array.ptr[3].via.array.size);')
   output:write('\n        goto '..cleanup_label..';')
   output:write('\n      }\n')
   -- Declare/initialize variables that will hold converted arguments
@@ -165,7 +158,9 @@ for i = 1, #api.functions do
     converted = 'arg_'..j
     convert_arg = 'msgpack_rpc_to_'..string.lower(param[1])
     output:write('\n      if (!'..convert_arg..'('..arg..', &'..converted..')) {')
-    output:write('\n        msgpack_rpc_error("Wrong type for argument '..j..', expecting '..param[1]..'", res);')
+    output:write('\n        snprintf(error->msg, sizeof(error->msg), "Wrong type for argument '..j..', expecting '..param[1]..'");')
+        
+    output:write('\n        error->set = true;')
     output:write('\n        goto '..cleanup_label..';')
     output:write('\n      }\n')
     args[#args + 1] = converted
@@ -196,28 +191,20 @@ for i = 1, #api.functions do
   if fn.can_fail then
     -- if the function can fail, also pass a pointer to the local error object
     if #args > 0 then
-      output:write(', &error);\n')
+      output:write(', error);\n')
     else
-      output:write('&error);\n')
+      output:write('error);\n')
     end
     -- and check for the error
-    output:write('\n      if (error.set) {')
-    output:write('\n        msgpack_rpc_error(error.msg, res);')
+    output:write('\n      if (error->set) {')
     output:write('\n        goto '..cleanup_label..';')
     output:write('\n      }\n')
   else
     output:write(');\n')
   end
 
-  -- nil error
-  output:write('\n      msgpack_pack_nil(res);');
-
-  if fn.return_type == 'void' then
-    output:write('\n      msgpack_pack_nil(res);');
-  else
-    output:write('\n      msgpack_rpc_from_'..string.lower(fn.return_type)..'(rv, res);')
-    -- free the return value
-    output:write('\n      msgpack_rpc_free_'..string.lower(fn.return_type)..'(rv);')
+  if fn.return_type ~= 'void' then
+    output:write('\n      ret = '..string.upper(fn.return_type)..'_OBJ(rv);')
   end
   -- Now generate the cleanup label for freeing memory allocated for the
   -- arguments
@@ -227,7 +214,7 @@ for i = 1, #api.functions do
     local param = fn.parameters[j]
     output:write('\n      msgpack_rpc_free_'..string.lower(param[1])..'(arg_'..j..');')
   end
-  output:write('\n      return;');
+  output:write('\n      break;');
   output:write('\n    };\n');
 
 end
@@ -236,8 +223,10 @@ output:write([[
 
 
     default:
-      msgpack_rpc_error("Invalid function id", res);
+      snprintf(error->msg, sizeof(error->msg), "Invalid function id");
+      error->set = true;
   }
+  return ret;
 }
 ]])
 output:close()