Windows: Improve the check for endianness when using Visual Studio.

Visual studio does not define __BYTE_ORDER__ so all architectures
were detected as LITTLE_ENDIAN since both __BYTE_ORDER__ and
__ORDER_LITTLE_ENDIAN__ would evaluate to 0 and compare equal. This
updates the checks to use CMakes detection of endianness, with a hard
error, if this also fails.

1 files changed, 16 insertions, 1 deletions

diff --git a/include/compat/endian.h b/include/compat/endian.h
index d0dcfe3..5376c1a 100644
--- a/include/compat/endian.h
+++ b/include/compat/endian.h
@@ -13,13 +13,28 @@
 #define PDP_ENDIAN 3412
 
 /*
- * Use GCC and Visual Studio compiler defines to determine endian.
+ * Use GCC compiler defines to determine endianness.
  */
+#if defined(__BYTE_ORDER__)
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 #define BYTE_ORDER LITTLE_ENDIAN
 #else
 #define BYTE_ORDER BIG_ENDIAN
 #endif
+#endif
+
+/*
+ * Use build system defines to determine endianness.
+ */
+#if !defined(BYTE_ORDER)
+#if defined(HAVE_LITTLE_ENDIAN)
+#define BYTE_ORDER LITTLE_ENDIAN
+#elif defined(HAVE_BIG_ENDIAN)
+#define BYTE_ORDER BIG_ENDIAN
+#else
+#error "Could not detect endianness."
+#endif
+#endif
 
 #elif defined(HAVE_ENDIAN_H)
 #include_next <endian.h>