msgpack: Allow notifications to execute commands.

Consider: `let vim = rpcstart('nvim', ['--embed'])`

Allows `rpcnotify(vim, ...)` to work like an asynchronous
`rpcrequest(nvim, ...)`.

Helped-by: Michael Reed <m.reed@mykolab.com>
Helped-by: Justin M. Keyes <>

3 files changed, 79 insertions, 29 deletions

diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index 76ac23a521..6c8e324649 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -22,6 +22,15 @@ typedef enum {
   kErrorTypeValidation
 } ErrorType;
 
+typedef enum {
+  kMessageTypeRequest,
+  kMessageTypeResponse,
+  kMessageTypeNotification
+} MessageType;
+
+/// Used as the message ID of notifications.
+#define NO_RESPONSE UINT64_MAX
+
 typedef struct {
   ErrorType type;
   char msg[1024];
diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index fc0409e137..9c812ced82 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -435,18 +435,18 @@ static void handle_request(Channel *channel, msgpack_object *request)
 
   // Retrieve the request handler
   MsgpackRpcRequestHandler handler;
-  msgpack_object method = request->via.array.ptr[2];
+  msgpack_object *method = msgpack_rpc_method(request);
 
-  if (method.type == MSGPACK_OBJECT_BIN || method.type == MSGPACK_OBJECT_STR) {
-    handler = msgpack_rpc_get_handler_for(method.via.bin.ptr,
-                                          method.via.bin.size);
+  if (method) {
+    handler = msgpack_rpc_get_handler_for(method->via.bin.ptr,
+                                          method->via.bin.size);
   } else {
     handler.fn = msgpack_rpc_handle_missing_method;
     handler.defer = false;
   }
 
   Array args = ARRAY_DICT_INIT;
-  msgpack_rpc_to_array(request->via.array.ptr + 3, &args);
+  msgpack_rpc_to_array(msgpack_rpc_args(request), &args);
   bool defer = (!kv_size(channel->call_stack) && handler.defer);
   RequestEvent *event_data = xmalloc(sizeof(RequestEvent));
   event_data->channel = channel;
@@ -469,14 +469,18 @@ static void on_request_event(Event event)
   uint64_t request_id = e->request_id;
   Error error = ERROR_INIT;
   Object result = handler.fn(channel->id, request_id, args, &error);
-  // send the response
-  msgpack_packer response;
-  msgpack_packer_init(&response, &out_buffer, msgpack_sbuffer_write);
-  channel_write(channel, serialize_response(channel->id,
-                                            request_id,
-                                            &error,
-                                            result,
-                                            &out_buffer));
+  if (request_id != NO_RESPONSE) {
+    // send the response
+    msgpack_packer response;
+    msgpack_packer_init(&response, &out_buffer, msgpack_sbuffer_write);
+    channel_write(channel, serialize_response(channel->id,
+                                              request_id,
+                                              &error,
+                                              result,
+                                              &out_buffer));
+  } else {
+    api_free_object(result);
+  }
   // All arguments were freed already, but we still need to free the array
   xfree(args.items);
   decref(channel);
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index 355176aa5f..7d0db9a9b8 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -351,49 +351,86 @@ void msgpack_rpc_serialize_response(uint64_t response_id,
   }
 }
 
+static bool msgpack_rpc_is_notification(msgpack_object *req)
+{
+  return req->via.array.ptr[0].via.u64 == 2;
+}
+
+msgpack_object *msgpack_rpc_method(msgpack_object *req)
+{
+  msgpack_object *obj = req->via.array.ptr
+    + (msgpack_rpc_is_notification(req) ? 1 : 2);
+  return obj->type == MSGPACK_OBJECT_STR || obj->type == MSGPACK_OBJECT_BIN ?
+    obj : NULL;
+}
+
+msgpack_object *msgpack_rpc_args(msgpack_object *req)
+{
+  msgpack_object *obj = req->via.array.ptr
+    + (msgpack_rpc_is_notification(req) ? 2 : 3);
+  return obj->type == MSGPACK_OBJECT_ARRAY ? obj : NULL;
+}
+
+static msgpack_object *msgpack_rpc_msg_id(msgpack_object *req)
+{
+  if (msgpack_rpc_is_notification(req)) {
+    return NULL;
+  }
+  msgpack_object *obj = &req->via.array.ptr[1];
+  return obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER ? obj : NULL;
+}
+
 void msgpack_rpc_validate(uint64_t *response_id,
                           msgpack_object *req,
                           Error *err)
 {
   // response id not known yet
 
-  *response_id = 0;
+  *response_id = NO_RESPONSE;
   // Validate the basic structure of the msgpack-rpc payload
   if (req->type != MSGPACK_OBJECT_ARRAY) {
-    api_set_error(err, Validation, _("Request is not an array"));
+    api_set_error(err, Validation, _("Message is not an array"));
     return;
   }
 
-  if (req->via.array.size != 4) {
-    api_set_error(err, Validation, _("Request array size should be 4"));
+  if (req->via.array.size == 0) {
+    api_set_error(err, Validation, _("Message is empty"));
     return;
   }
 
-  if (req->via.array.ptr[1].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-    api_set_error(err, Validation, _("Id must be a positive integer"));
+  if (req->via.array.ptr[0].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
+    api_set_error(err, Validation, _("Message type must be an integer"));
     return;
   }
 
-  // Set the response id, which is the same as the request
-  *response_id = req->via.array.ptr[1].via.u64;
-
-  if (req->via.array.ptr[0].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-    api_set_error(err, Validation, _("Message type must be an integer"));
+  uint64_t type = req->via.array.ptr[0].via.u64;
+  if (type != kMessageTypeRequest && type != kMessageTypeNotification) {
+    api_set_error(err, Validation, _("Unknown message type"));
     return;
   }
 
-  if (req->via.array.ptr[0].via.u64 != 0) {
-    api_set_error(err, Validation, _("Message type must be 0"));
+  if ((type == kMessageTypeRequest && req->via.array.size != 4) ||
+      (type == kMessageTypeNotification && req->via.array.size != 3)) {
+    api_set_error(err, Validation, _("Request array size should be 4 (request) "
+                                     "or 3 (notification)"));
     return;
   }
 
-  if (req->via.array.ptr[2].type != MSGPACK_OBJECT_BIN
-    && req->via.array.ptr[2].type != MSGPACK_OBJECT_STR) {
+  if (type == kMessageTypeRequest) {
+    msgpack_object *id_obj = msgpack_rpc_msg_id(req);
+    if (!id_obj) {
+      api_set_error(err, Validation, _("ID must be a positive integer"));
+      return;
+    }
+    *response_id = id_obj->via.u64;
+  }
+
+  if (!msgpack_rpc_method(req)) {
     api_set_error(err, Validation, _("Method must be a string"));
     return;
   }
 
-  if (req->via.array.ptr[3].type != MSGPACK_OBJECT_ARRAY) {
+  if (!msgpack_rpc_args(req)) {
     api_set_error(err, Validation, _("Parameters must be an array"));
     return;
   }