msgpack_rpc: Fix crash in log_server_msg

It appears that used msgpack library is not able to parse back message created 
by msgpack_rpc_from_object() if nesting level is too high, so log_server_msg now 
cares about msgpack_unpack_next() return value. Also error message from 
server_notifications_spec.lua is not readable if something is wrong (though at 
least now it does not crash when parsing deeply nested structures).

log_server_msg() in the test reports

    [msgpack-rpc] nvim -> client(1) [error]        "parse error"

1 files changed, 42 insertions, 7 deletions

diff --git a/src/nvim/msgpack_rpc/channel.c b/src/nvim/msgpack_rpc/channel.c
index 0d7d5a247e..b14278a554 100644
--- a/src/nvim/msgpack_rpc/channel.c
+++ b/src/nvim/msgpack_rpc/channel.c
@@ -816,20 +816,55 @@ static void decref(Channel *channel)
 #define REQ "[request]      "
 #define RES "[response]     "
 #define NOT "[notification] "
+#define ERR "[error]        "
+
+// Cannot define array with negative offsets, so this one is needed to be added
+// to MSGPACK_UNPACK_\* values.
+#define MUR_OFF 2
+
+static const char *const msgpack_error_messages[] = {
+  [MSGPACK_UNPACK_EXTRA_BYTES + MUR_OFF] = "extra bytes found",
+  [MSGPACK_UNPACK_CONTINUE + MUR_OFF] = "incomplete string",
+  [MSGPACK_UNPACK_PARSE_ERROR + MUR_OFF] = "parse error",
+  [MSGPACK_UNPACK_NOMEM_ERROR + MUR_OFF] = "not enough memory",
+};
 
 static void log_server_msg(uint64_t channel_id,
                            msgpack_sbuffer *packed)
 {
   msgpack_unpacked unpacked;
   msgpack_unpacked_init(&unpacked);
-  msgpack_unpack_next(&unpacked, packed->data, packed->size, NULL);
-  uint64_t type = unpacked.data.via.array.ptr[0].via.u64;
   DLOGN("[msgpack-rpc] nvim -> client(%" PRIu64 ") ", channel_id);
-  log_lock();
-  FILE *f = open_log_file();
-  fprintf(f, type ? (type == 1 ? RES : NOT) : REQ);
-  log_msg_close(f, unpacked.data);
-  msgpack_unpacked_destroy(&unpacked);
+  const msgpack_unpack_return result =
+      msgpack_unpack_next(&unpacked, packed->data, packed->size, NULL);
+  switch (result) {
+    case MSGPACK_UNPACK_SUCCESS: {
+      uint64_t type = unpacked.data.via.array.ptr[0].via.u64;
+      log_lock();
+      FILE *f = open_log_file();
+      fprintf(f, type ? (type == 1 ? RES : NOT) : REQ);
+      log_msg_close(f, unpacked.data);
+      msgpack_unpacked_destroy(&unpacked);
+      break;
+    }
+    case MSGPACK_UNPACK_EXTRA_BYTES:
+    case MSGPACK_UNPACK_CONTINUE:
+    case MSGPACK_UNPACK_PARSE_ERROR:
+    case MSGPACK_UNPACK_NOMEM_ERROR: {
+      log_lock();
+      FILE *f = open_log_file();
+      fprintf(f, ERR);
+      log_msg_close(f, (msgpack_object) {
+          .type = MSGPACK_OBJECT_STR,
+          .via.str = {
+              .ptr = (char *) msgpack_error_messages[result + MUR_OFF],
+              .size = (uint32_t) strlen(
+                  msgpack_error_messages[result + MUR_OFF]),
+          },
+      });
+      break;
+    }
+  }
 }
 
 static void log_client_msg(uint64_t channel_id,