config: Check order and endianess even when cross-compiling

2 files changed, 41 insertions, 18 deletions

diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt
index 9334691e2a..b780291264 100644
--- a/config/CMakeLists.txt
+++ b/config/CMakeLists.txt
@@ -3,6 +3,7 @@ include(CheckSymbolExists)
 include(CheckFunctionExists)
 include(CheckIncludeFiles)
 include(CheckCSourceRuns)
+include(CheckCSourceCompiles)
 
 check_type_size("int" SIZEOF_INT)
 check_type_size("long" SIZEOF_LONG)
@@ -75,31 +76,50 @@ endif()
 set(SI "#include <stdint.h>\n")
 set(MS "int main(int argc,char**argv)\n{\n  uint64_t i=0x0102030405060708ULL;")
 set(ME "}")
-check_c_source_runs("
-  ${SI}
-  ${MS}
-    char *s = (char *) &i;
-    return (
-      s[0] == 0x01
-      && s[1] == 0x02
-      && s[2] == 0x03
-      && s[3] == 0x04
-      && s[4] == 0x05
-      && s[5] == 0x06
-      && s[6] == 0x07
-      && s[7] == 0x08) ? 0 : 1;
-  ${ME}"
-  ORDER_BIG_ENDIAN)
-check_c_source_runs("
+check_c_source_compiles("
   #define _BSD_SOURCE 1
   #define _DEFAULT_SOURCE 1
   ${SI}
   #include <endian.h>
+  #ifndef be64toh
+  # error No be64toh macros
+  #endif
   ${MS}
     uint64_t j = be64toh(i);
-    return (j == 0);  // Must not be zero
+    return (j == 0);  // j must not be zero
   ${ME}"
-  HAVE_BE64TOH)
+  HAVE_BE64TOH_MACROS)
+if(NOT "${HAVE_BE64TOH_MACROS}")
+  check_function_exists(be64toh HAVE_BE64TOH_FUNC)
+endif()
+if("${HAVE_BE64TOH_MACROS}" OR "${HAVE_BE64TOH_FUNC}")
+  set(HAVE_BE64TOH 1)
+endif()
+if (NOT "${HAVE_BE64TOH}")
+  if (NOT "${CMAKE_CROSSCOMPILING}")
+    # It is safe to make ORDER_BIG_ENDIAN not defined if
+    # - HAVE_BE64TOH is true. In this case be64toh will be used unconditionally in 
+    #   any case and ORDER_BIG_ENDIAN will not be examined.
+    # - CMAKE_CROSSCOMPILING *and* HAVE_BE64TOH are both false. In this case 
+    #   be64toh function which uses cycle and arithmetic operations is used which 
+    #   will work regardless of endianess. Function is sub-optimal though.
+    check_c_source_runs("
+      ${SI}
+      ${MS}
+        char *s = (char *) &i;
+        return (
+          s[0] == 0x01
+          && s[1] == 0x02
+          && s[2] == 0x03
+          && s[3] == 0x04
+          && s[4] == 0x05
+          && s[5] == 0x06
+          && s[6] == 0x07
+          && s[7] == 0x08) ? 0 : 1;
+      ${ME}"
+      ORDER_BIG_ENDIAN)
+  endif()
+endif()
 
 # generate configuration header and update include directories
 configure_file (
diff --git a/src/nvim/shada.c b/src/nvim/shada.c
index 6de3d8b20e..1d4b486823 100644
--- a/src/nvim/shada.c
+++ b/src/nvim/shada.c
@@ -3141,6 +3141,9 @@ static inline uint64_t be64toh(uint64_t big_endian_64_bits)
 #ifdef ORDER_BIG_ENDIAN
   return big_endian_64_bits;
 #else
+  // It may appear that when !defined(ORDER_BIG_ENDIAN) actual order is big
+  // endian. This variant is suboptimal, but it works regardless of actual
+  // order.
   uint8_t *buf = (uint8_t *) &big_endian_64_bits;
   uint64_t ret = 0;
   for (size_t i = 8; i; i--) {