msgpack: Allow notifications to execute commands.

Consider: `let vim = rpcstart('nvim', ['--embed'])`

Allows `rpcnotify(vim, ...)` to work like an asynchronous
`rpcrequest(nvim, ...)`.

Helped-by: Michael Reed <m.reed@mykolab.com>
Helped-by: Justin M. Keyes <>

1 files changed, 53 insertions, 16 deletions

diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index 355176aa5f..7d0db9a9b8 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -351,49 +351,86 @@ void msgpack_rpc_serialize_response(uint64_t response_id,
   }
 }
 
+static bool msgpack_rpc_is_notification(msgpack_object *req)
+{
+  return req->via.array.ptr[0].via.u64 == 2;
+}
+
+msgpack_object *msgpack_rpc_method(msgpack_object *req)
+{
+  msgpack_object *obj = req->via.array.ptr
+    + (msgpack_rpc_is_notification(req) ? 1 : 2);
+  return obj->type == MSGPACK_OBJECT_STR || obj->type == MSGPACK_OBJECT_BIN ?
+    obj : NULL;
+}
+
+msgpack_object *msgpack_rpc_args(msgpack_object *req)
+{
+  msgpack_object *obj = req->via.array.ptr
+    + (msgpack_rpc_is_notification(req) ? 2 : 3);
+  return obj->type == MSGPACK_OBJECT_ARRAY ? obj : NULL;
+}
+
+static msgpack_object *msgpack_rpc_msg_id(msgpack_object *req)
+{
+  if (msgpack_rpc_is_notification(req)) {
+    return NULL;
+  }
+  msgpack_object *obj = &req->via.array.ptr[1];
+  return obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER ? obj : NULL;
+}
+
 void msgpack_rpc_validate(uint64_t *response_id,
                           msgpack_object *req,
                           Error *err)
 {
   // response id not known yet
 
-  *response_id = 0;
+  *response_id = NO_RESPONSE;
   // Validate the basic structure of the msgpack-rpc payload
   if (req->type != MSGPACK_OBJECT_ARRAY) {
-    api_set_error(err, Validation, _("Request is not an array"));
+    api_set_error(err, Validation, _("Message is not an array"));
     return;
   }
 
-  if (req->via.array.size != 4) {
-    api_set_error(err, Validation, _("Request array size should be 4"));
+  if (req->via.array.size == 0) {
+    api_set_error(err, Validation, _("Message is empty"));
     return;
   }
 
-  if (req->via.array.ptr[1].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-    api_set_error(err, Validation, _("Id must be a positive integer"));
+  if (req->via.array.ptr[0].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
+    api_set_error(err, Validation, _("Message type must be an integer"));
     return;
   }
 
-  // Set the response id, which is the same as the request
-  *response_id = req->via.array.ptr[1].via.u64;
-
-  if (req->via.array.ptr[0].type != MSGPACK_OBJECT_POSITIVE_INTEGER) {
-    api_set_error(err, Validation, _("Message type must be an integer"));
+  uint64_t type = req->via.array.ptr[0].via.u64;
+  if (type != kMessageTypeRequest && type != kMessageTypeNotification) {
+    api_set_error(err, Validation, _("Unknown message type"));
     return;
   }
 
-  if (req->via.array.ptr[0].via.u64 != 0) {
-    api_set_error(err, Validation, _("Message type must be 0"));
+  if ((type == kMessageTypeRequest && req->via.array.size != 4) ||
+      (type == kMessageTypeNotification && req->via.array.size != 3)) {
+    api_set_error(err, Validation, _("Request array size should be 4 (request) "
+                                     "or 3 (notification)"));
     return;
   }
 
-  if (req->via.array.ptr[2].type != MSGPACK_OBJECT_BIN
-    && req->via.array.ptr[2].type != MSGPACK_OBJECT_STR) {
+  if (type == kMessageTypeRequest) {
+    msgpack_object *id_obj = msgpack_rpc_msg_id(req);
+    if (!id_obj) {
+      api_set_error(err, Validation, _("ID must be a positive integer"));
+      return;
+    }
+    *response_id = id_obj->via.u64;
+  }
+
+  if (!msgpack_rpc_method(req)) {
     api_set_error(err, Validation, _("Method must be a string"));
     return;
   }
 
-  if (req->via.array.ptr[3].type != MSGPACK_OBJECT_ARRAY) {
+  if (!msgpack_rpc_args(req)) {
     api_set_error(err, Validation, _("Parameters must be an array"));
     return;
   }