object_to_vim: Fix buffer/window/tabpage conversion on BE systems

Since data.integer is a different (larger) integer type than
data.{buffer,window,tabpage}, we cannot abuse the union by using
data.integer to access the value for all 4 types.  Instead, remove the
{buffer,window,tabpage} members and always use the integer member.

In order to accomodate this, perform distinct validation and coercion
between the Integer type and Buffer/Window/Tabpage types in
object_to_vim, msgpack_rpc helpers, and gendispatch.lua.

3 files changed, 5 insertions, 8 deletions

diff --git a/src/nvim/api/private/defs.h b/src/nvim/api/private/defs.h
index 1d5ecd3071..223aab09dc 100644
--- a/src/nvim/api/private/defs.h
+++ b/src/nvim/api/private/defs.h
@@ -91,9 +91,6 @@ typedef enum {
 struct object {
   ObjectType type;
   union {
-    Buffer buffer;
-    Window window;
-    Tabpage tabpage;
     Boolean boolean;
     Integer integer;
     Float floating;
diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c
index 208c3b53c8..8c6eb4c0df 100644
--- a/src/nvim/api/private/helpers.c
+++ b/src/nvim/api/private/helpers.c
@@ -616,13 +616,13 @@ bool object_to_vim(Object obj, typval_T *tv, Error *err)
     case kObjectTypeWindow:
     case kObjectTypeTabpage:
     case kObjectTypeInteger:
-      if (obj.data.integer > INT_MAX || obj.data.integer < INT_MIN) {
+      if (obj.data.integer > VARNUMBER_MAX || obj.data.integer < VARNUMBER_MIN) {
         api_set_error(err, Validation, _("Integer value outside range"));
         return false;
       }
 
       tv->v_type = VAR_NUMBER;
-      tv->vval.v_number = (int)obj.data.integer;
+      tv->vval.v_number = (varnumber_T)obj.data.integer;
       break;
 
     case kObjectTypeFloat:
diff --git a/src/nvim/api/private/helpers.h b/src/nvim/api/private/helpers.h
index a946e35149..9fe8c351cf 100644
--- a/src/nvim/api/private/helpers.h
+++ b/src/nvim/api/private/helpers.h
@@ -37,15 +37,15 @@
 
 #define BUFFER_OBJ(s) ((Object) { \
     .type = kObjectTypeBuffer, \
-    .data.buffer = s })
+    .data.integer = s })
 
 #define WINDOW_OBJ(s) ((Object) { \
     .type = kObjectTypeWindow, \
-    .data.window = s })
+    .data.integer = s })
 
 #define TABPAGE_OBJ(s) ((Object) { \
     .type = kObjectTypeTabpage, \
-    .data.tabpage = s })
+    .data.integer = s })
 
 #define ARRAY_OBJ(a) ((Object) { \
     .type = kObjectTypeArray, \