Figure out endianness at compile-time, using _BYTE_ORDER from

<machine/endian.h>, rather than writing 1 to a 32-bit variable and checking
whether the first byte is nonzero.

tweaks and ok matthew@; ok beck@ tedu@

2 files changed, 6 insertions, 4 deletions

diff --git a/src/lib/libcrypto/sha/sha256.c b/src/lib/libcrypto/sha/sha256.c
index 4eae074849..e767afde5a 100644
--- a/src/lib/libcrypto/sha/sha256.c
+++ b/src/lib/libcrypto/sha/sha256.c
@@ -9,6 +9,7 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <machine/endian.h>
 
 #include <openssl/crypto.h>
 #include <openssl/sha.h>
@@ -206,14 +207,14 @@ static void sha256_block_data_order (SHA256_CTX *ctx, const void *in, size_t num
         SHA_LONG        X[16];
         int i;
         const unsigned char *data=in;
-        const union { long one; char little; } is_endian = {1};
 
                         while (num--) {
 
         a = ctx->h[0];  b = ctx->h[1];  c = ctx->h[2];  d = ctx->h[3];
         e = ctx->h[4];  f = ctx->h[5];  g = ctx->h[6];  h = ctx->h[7];
 
-        if (!is_endian.little && sizeof(SHA_LONG)==4 && ((size_t)in%4)==0)
+        if (_BYTE_ORDER != _LITTLE_ENDIAN &&
+            sizeof(SHA_LONG)==4 && ((size_t)in%4)==0)
                 {
                 const SHA_LONG *W=(const SHA_LONG *)data;
 
diff --git a/src/lib/libcrypto/sha/sha_locl.h b/src/lib/libcrypto/sha/sha_locl.h
index 6c6cd64282..1210176dda 100644
--- a/src/lib/libcrypto/sha/sha_locl.h
+++ b/src/lib/libcrypto/sha/sha_locl.h
@@ -202,6 +202,7 @@ fips_md_init_ctx(SHA1, SHA)
 #endif
 
 #if !defined(SHA_1) || !defined(SHA1_ASM)
+#include <machine/endian.h>
 static void HASH_BLOCK_DATA_ORDER (SHA_CTX *c, const void *p, size_t num)
         {
         const unsigned char *data=p;
@@ -221,9 +222,9 @@ static void HASH_BLOCK_DATA_ORDER (SHA_CTX *c, const void *p, size_t num)
 
         for (;;)
                         {
-        const union { long one; char little; } is_endian = {1};
 
-        if (!is_endian.little && sizeof(SHA_LONG)==4 && ((size_t)p%4)==0)
+        if (_BYTE_ORDER != _LITTLE_ENDIAN &&
+            sizeof(SHA_LONG)==4 && ((size_t)p%4)==0)
                 {
                 const SHA_LONG *W=(const SHA_LONG *)data;