object_to_vim: Fix buffer/window/tabpage conversion on BE systems

Since data.integer is a different (larger) integer type than
data.{buffer,window,tabpage}, we cannot abuse the union by using
data.integer to access the value for all 4 types.  Instead, remove the
{buffer,window,tabpage} members and always use the integer member.

In order to accomodate this, perform distinct validation and coercion
between the Integer type and Buffer/Window/Tabpage types in
object_to_vim, msgpack_rpc helpers, and gendispatch.lua.

4 files changed, 14 insertions, 17 deletions

diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index 1d5ecd3071..223aab09dc 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -91,9 +91,6 @@ typedef enum {
 struct object {
   ObjectType type;
   union {
-    Buffer buffer;
-    Window window;
-    Tabpage tabpage;
     Boolean boolean;
     Integer integer;
     Float floating;
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index 208c3b53c8..8c6eb4c0df 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -616,13 +616,13 @@ bool object_to_vim(Object obj, typval_T *tv, Error *err)
     case kObjectTypeWindow:
     case kObjectTypeTabpage:
     case kObjectTypeInteger:
-      if (obj.data.integer > INT_MAX || obj.data.integer < INT_MIN) {
+      if (obj.data.integer > VARNUMBER_MAX || obj.data.integer < VARNUMBER_MIN) {
         api_set_error(err, Validation, _("Integer value outside range"));
         return false;
       }
 
       tv->v_type = VAR_NUMBER;
-      tv->vval.v_number = (int)obj.data.integer;
+      tv->vval.v_number = (varnumber_T)obj.data.integer;
       break;
 
     case kObjectTypeFloat:
diff --git a/src/nvim/api/private/helpers.h b/src/nvim/api/private/helpers.h
index a946e35149..9fe8c351cf 100644
--- a/src/nvim/api/private/helpers.h
+++ b/src/nvim/api/private/helpers.h
@@ -37,15 +37,15 @@
 
 #define BUFFER_OBJ(s) ((Object) { \
     .type = kObjectTypeBuffer, \
-    .data.buffer = s })
+    .data.integer = s })
 
 #define WINDOW_OBJ(s) ((Object) { \
     .type = kObjectTypeWindow, \
-    .data.window = s })
+    .data.integer = s })
 
 #define TABPAGE_OBJ(s) ((Object) { \
     .type = kObjectTypeTabpage, \
-    .data.tabpage = s })
+    .data.integer = s })
 
 #define ARRAY_OBJ(a) ((Object) { \
     .type = kObjectTypeArray, \
diff --git a/src/nvim/msgpack_rpc/helpers.c b/src/nvim/msgpack_rpc/helpers.c
index 14e1c2d978..8c1f101deb 100644
--- a/src/nvim/msgpack_rpc/helpers.c
+++ b/src/nvim/msgpack_rpc/helpers.c
@@ -21,7 +21,7 @@ static msgpack_zone zone;
 static msgpack_sbuffer sbuffer;
 
 #define HANDLE_TYPE_CONVERSION_IMPL(t, lt) \
-  bool msgpack_rpc_to_##lt(const msgpack_object *const obj, t *const arg) \
+  bool msgpack_rpc_to_##lt(const msgpack_object *const obj, Integer *const arg) \
     FUNC_ATTR_NONNULL_ALL \
   { \
     if (obj->type != MSGPACK_OBJECT_EXT \
@@ -44,12 +44,12 @@ static msgpack_sbuffer sbuffer;
     return true; \
   } \
   \
-  void msgpack_rpc_from_##lt(t o, msgpack_packer *res) \
+  void msgpack_rpc_from_##lt(Integer o, msgpack_packer *res) \
     FUNC_ATTR_NONNULL_ARG(2) \
   { \
     msgpack_packer pac; \
     msgpack_packer_init(&pac, &sbuffer, msgpack_sbuffer_write); \
-    msgpack_pack_int64(&pac, o); \
+    msgpack_pack_int64(&pac, (handle_T)o); \
     msgpack_pack_ext(res, sbuffer.size, kObjectType##t); \
     msgpack_pack_ext_body(res, sbuffer.data, sbuffer.size); \
     msgpack_sbuffer_clear(&sbuffer); \
@@ -213,17 +213,17 @@ bool msgpack_rpc_to_object(const msgpack_object *const obj, Object *const arg)
         switch (cur.mobj->via.ext.type) {
           case kObjectTypeBuffer: {
             cur.aobj->type = kObjectTypeBuffer;
-            ret = msgpack_rpc_to_buffer(cur.mobj, &cur.aobj->data.buffer);
+            ret = msgpack_rpc_to_buffer(cur.mobj, &cur.aobj->data.integer);
             break;
           }
           case kObjectTypeWindow: {
             cur.aobj->type = kObjectTypeWindow;
-            ret = msgpack_rpc_to_window(cur.mobj, &cur.aobj->data.window);
+            ret = msgpack_rpc_to_window(cur.mobj, &cur.aobj->data.integer);
             break;
           }
           case kObjectTypeTabpage: {
             cur.aobj->type = kObjectTypeTabpage;
-            ret = msgpack_rpc_to_tabpage(cur.mobj, &cur.aobj->data.tabpage);
+            ret = msgpack_rpc_to_tabpage(cur.mobj, &cur.aobj->data.integer);
             break;
           }
         }
@@ -369,15 +369,15 @@ void msgpack_rpc_from_object(const Object result, msgpack_packer *const res)
         break;
       }
       case kObjectTypeBuffer: {
-        msgpack_rpc_from_buffer(cur.aobj->data.buffer, res);
+        msgpack_rpc_from_buffer(cur.aobj->data.integer, res);
         break;
       }
       case kObjectTypeWindow: {
-        msgpack_rpc_from_window(cur.aobj->data.window, res);
+        msgpack_rpc_from_window(cur.aobj->data.integer, res);
         break;
       }
       case kObjectTypeTabpage: {
-        msgpack_rpc_from_tabpage(cur.aobj->data.tabpage, res);
+        msgpack_rpc_from_tabpage(cur.aobj->data.integer, res);
         break;
       }
       case kObjectTypeArray: {