Windows: Improve the check for endianness when using Visual Studio.

Visual studio does not define __BYTE_ORDER__ so all architectures
were detected as LITTLE_ENDIAN since both __BYTE_ORDER__ and
__ORDER_LITTLE_ENDIAN__ would evaluate to 0 and compare equal. This
updates the checks to use CMakes detection of endianness, with a hard
error, if this also fails.

2 files changed, 25 insertions, 1 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index fdd2e14..477e51f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -140,6 +140,15 @@ endif()
 if(MSVC)
         add_definitions(-Dinline=__inline)
         message(STATUS "Using [${CMAKE_C_COMPILER_ID}] compiler")
+
+        include(TestBigEndian)
+        TEST_BIG_ENDIAN(HAVE_BIG_ENDIAN)
+        if(HAVE_BIG_ENDIAN)
+                add_definitions(-DHAVE_BIG_ENDIAN)
+        else()
+                add_definitions(-DHAVE_LITTLE_ENDIAN)
+        endif()
+
         if(CMAKE_C_COMPILER_ID MATCHES "MSVC" OR CMAKE_C_COMPILER_ID MATCHES "Clang")
                 set(MSVC_DISABLED_WARNINGS_LIST
                         "C4018" # 'expression' : signed/unsigned mismatch
diff --git a/include/compat/endian.h b/include/compat/endian.h
index d0dcfe3..5376c1a 100644
--- a/include/compat/endian.h
+++ b/include/compat/endian.h
@@ -13,13 +13,28 @@
 #define PDP_ENDIAN 3412
 
 /*
- * Use GCC and Visual Studio compiler defines to determine endian.
+ * Use GCC compiler defines to determine endianness.
  */
+#if defined(__BYTE_ORDER__)
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 #define BYTE_ORDER LITTLE_ENDIAN
 #else
 #define BYTE_ORDER BIG_ENDIAN
 #endif
+#endif
+
+/*
+ * Use build system defines to determine endianness.
+ */
+#if !defined(BYTE_ORDER)
+#if defined(HAVE_LITTLE_ENDIAN)
+#define BYTE_ORDER LITTLE_ENDIAN
+#elif defined(HAVE_BIG_ENDIAN)
+#define BYTE_ORDER BIG_ENDIAN
+#else
+#error "Could not detect endianness."
+#endif
+#endif
 
 #elif defined(HAVE_ENDIAN_H)
 #include_next <endian.h>