Figure out endianness at compile-time, using _BYTE_ORDER from

<machine/endian.h>, rather than writing 1 to a 32-bit variable and checking
whether the first byte is nonzero.

tweaks and ok matthew@; ok beck@ tedu@

1 files changed, 31 insertions, 44 deletions

diff --git a/src/lib/libcrypto/modes/gcm128.c b/src/lib/libcrypto/modes/gcm128.c
index a495db110f..92b7f4f3c8 100644
--- a/src/lib/libcrypto/modes/gcm128.c
+++ b/src/lib/libcrypto/modes/gcm128.c
@@ -147,7 +147,6 @@ static void gcm_gmult_8bit(u64 Xi[2], const u128 Htable[256])
         u128 Z = { 0, 0};
         const u8 *xi = (const u8 *)Xi+15;
         size_t rem, n = *xi;
-        const union { long one; char little; } is_endian = {1};
         static const size_t rem_8bit[256] = {
                 PACK(0x0000), PACK(0x01C2), PACK(0x0384), PACK(0x0246),
                 PACK(0x0708), PACK(0x06CA), PACK(0x048C), PACK(0x054E),
@@ -231,7 +230,7 @@ static void gcm_gmult_8bit(u64 Xi[2], const u128 Htable[256])
                         Z.hi ^= (u64)rem_8bit[rem]<<32;
         }
 
-        if (is_endian.little) {
+        if (_BYTE_ORDER == _LITTLE_ENDIAN) {
 #ifdef BSWAP8
                 Xi[0] = BSWAP8(Z.hi);
                 Xi[1] = BSWAP8(Z.lo);
@@ -307,9 +306,8 @@ static void gcm_init_4bit(u128 Htable[16], u64 H[2])
          */
         {
         int j;
-        const union { long one; char little; } is_endian = {1};
 
-        if (is_endian.little)
+        if (_BYTE_ORDER == _LITTLE_ENDIAN)
                 for (j=0;j<16;++j) {
                         V = Htable[j];
                         Htable[j].hi = V.lo;
@@ -337,7 +335,6 @@ static void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16])
         u128 Z;
         int cnt = 15;
         size_t rem, nlo, nhi;
-        const union { long one; char little; } is_endian = {1};
 
         nlo  = ((const u8 *)Xi)[15];
         nhi  = nlo>>4;
@@ -376,7 +373,7 @@ static void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16])
                 Z.lo ^= Htable[nlo].lo;
         }
 
-        if (is_endian.little) {
+        if (_BYTE_ORDER == _LITTLE_ENDIAN) {
 #ifdef BSWAP8
                 Xi[0] = BSWAP8(Z.hi);
                 Xi[1] = BSWAP8(Z.lo);
@@ -409,7 +406,6 @@ static void gcm_ghash_4bit(u64 Xi[2],const u128 Htable[16],
     u128 Z;
     int cnt;
     size_t rem, nlo, nhi;
-    const union { long one; char little; } is_endian = {1};
 
 #if 1
     do {
@@ -546,7 +542,7 @@ static void gcm_ghash_4bit(u64 Xi[2],const u128 Htable[16],
         Z.hi ^= ((u64)rem_8bit[rem<<4])<<48;
 #endif
 
-        if (is_endian.little) {
+        if (_BYTE_ORDER == _LITTLE_ENDIAN) {
 #ifdef BSWAP8
                 Xi[0] = BSWAP8(Z.hi);
                 Xi[1] = BSWAP8(Z.lo);
@@ -588,13 +584,12 @@ static void gcm_gmult_1bit(u64 Xi[2],const u64 H[2])
         long X;
         int  i,j;
         const long *xi = (const long *)Xi;
-        const union { long one; char little; } is_endian = {1};
 
         V.hi = H[0];    /* H is in host byte order, no byte swapping */
         V.lo = H[1];
 
         for (j=0; j<16/sizeof(long); ++j) {
-                if (is_endian.little) {
+                if (_BYTE_ORDER == _LITTLE_ENDIAN) {
                         if (sizeof(long)==8) {
 #ifdef BSWAP8
                                 X = (long)(BSWAP8(xi[j]));
@@ -620,7 +615,7 @@ static void gcm_gmult_1bit(u64 Xi[2],const u64 H[2])
                 }
         }
 
-        if (is_endian.little) {
+        if (_BYTE_ORDER == _LITTLE_ENDIAN) {
 #ifdef BSWAP8
                 Xi[0] = BSWAP8(Z.hi);
                 Xi[1] = BSWAP8(Z.lo);
@@ -685,15 +680,13 @@ void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
 
 void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
 {
-        const union { long one; char little; } is_endian = {1};
-
         memset(ctx,0,sizeof(*ctx));
         ctx->block = block;
         ctx->key   = key;
 
         (*block)(ctx->H.c,ctx->H.c,key);
 
-        if (is_endian.little) {
+        if (_BYTE_ORDER == _LITTLE_ENDIAN) {
                 /* H is stored in host byte order */
 #ifdef BSWAP8
                 ctx->H.u[0] = BSWAP8(ctx->H.u[0]);
@@ -755,7 +748,6 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
 
 void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx,const unsigned char *iv,size_t len)
 {
-        const union { long one; char little; } is_endian = {1};
         unsigned int ctr;
 #ifdef GCM_FUNCREF_4BIT
         void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])    = ctx->gmult;
@@ -790,7 +782,7 @@ void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx,const unsigned char *iv,size_t len)
                         GCM_MUL(ctx,Yi);
                 }
                 len0 <<= 3;
-                if (is_endian.little) {
+                if (_BYTE_ORDER == _LITTLE_ENDIAN) {
 #ifdef BSWAP8
                         ctx->Yi.u[1]  ^= BSWAP8(len0);
 #else
@@ -809,7 +801,7 @@ void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx,const unsigned char *iv,size_t len)
 
                 GCM_MUL(ctx,Yi);
 
-                if (is_endian.little)
+                if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                         ctr = BSWAP4(ctx->Yi.d[3]);
 #else
@@ -821,7 +813,7 @@ void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx,const unsigned char *iv,size_t len)
 
         (*ctx->block)(ctx->Yi.c,ctx->EK0.c,ctx->key);
         ++ctr;
-        if (is_endian.little)
+        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -892,7 +884,6 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                 const unsigned char *in, unsigned char *out,
                 size_t len)
 {
-        const union { long one; char little; } is_endian = {1};
         unsigned int n, ctr;
         size_t i;
         u64        mlen  = ctx->len.u[1];
@@ -920,7 +911,7 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                 ctx->ares = 0;
         }
 
-        if (is_endian.little)
+        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                 ctr = BSWAP4(ctx->Yi.d[3]);
 #else
@@ -958,7 +949,7 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
 
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -984,7 +975,7 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
 
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1007,7 +998,7 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
 
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1027,7 +1018,7 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                 if (len) {
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1049,7 +1040,7 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                 if (n==0) {
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1072,7 +1063,6 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                 const unsigned char *in, unsigned char *out,
                 size_t len)
 {
-        const union { long one; char little; } is_endian = {1};
         unsigned int n, ctr;
         size_t i;
         u64        mlen  = ctx->len.u[1];
@@ -1097,7 +1087,7 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                 ctx->ares = 0;
         }
 
-        if (is_endian.little)
+        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                 ctr = BSWAP4(ctx->Yi.d[3]);
 #else
@@ -1138,7 +1128,7 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
 
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1162,7 +1152,7 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
 
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1184,7 +1174,7 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
 
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1206,7 +1196,7 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                 if (len) {
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1231,7 +1221,7 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                 if (n==0) {
                         (*block)(ctx->Yi.c,ctx->EKi.c,key);
                         ++ctr;
-                        if (is_endian.little)
+                        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                                 ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1256,7 +1246,6 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
                 const unsigned char *in, unsigned char *out,
                 size_t len, ctr128_f stream)
 {
-        const union { long one; char little; } is_endian = {1};
         unsigned int n, ctr;
         size_t i;
         u64   mlen = ctx->len.u[1];
@@ -1280,7 +1269,7 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
                 ctx->ares = 0;
         }
 
-        if (is_endian.little)
+        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                 ctr = BSWAP4(ctx->Yi.d[3]);
 #else
@@ -1306,7 +1295,7 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
         while (len>=GHASH_CHUNK) {
                 (*stream)(in,out,GHASH_CHUNK/16,key,ctx->Yi.c);
                 ctr += GHASH_CHUNK/16;
-                if (is_endian.little)
+                if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                         ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1325,7 +1314,7 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
 
                 (*stream)(in,out,j,key,ctx->Yi.c);
                 ctr += (unsigned int)j;
-                if (is_endian.little)
+                if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                         ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1349,7 +1338,7 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
         if (len) {
                 (*ctx->block)(ctx->Yi.c,ctx->EKi.c,key);
                 ++ctr;
-                if (is_endian.little)
+                if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                         ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1371,7 +1360,6 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
                 const unsigned char *in, unsigned char *out,
                 size_t len,ctr128_f stream)
 {
-        const union { long one; char little; } is_endian = {1};
         unsigned int n, ctr;
         size_t i;
         u64   mlen = ctx->len.u[1];
@@ -1395,7 +1383,7 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
                 ctx->ares = 0;
         }
 
-        if (is_endian.little)
+        if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                 ctr = BSWAP4(ctx->Yi.d[3]);
 #else
@@ -1424,7 +1412,7 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
                 GHASH(ctx,in,GHASH_CHUNK);
                 (*stream)(in,out,GHASH_CHUNK/16,key,ctx->Yi.c);
                 ctr += GHASH_CHUNK/16;
-                if (is_endian.little)
+                if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                         ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1454,7 +1442,7 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
 #endif
                 (*stream)(in,out,j,key,ctx->Yi.c);
                 ctr += (unsigned int)j;
-                if (is_endian.little)
+                if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                         ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1469,7 +1457,7 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
         if (len) {
                 (*ctx->block)(ctx->Yi.c,ctx->EKi.c,key);
                 ++ctr;
-                if (is_endian.little)
+                if (_BYTE_ORDER == _LITTLE_ENDIAN)
 #ifdef BSWAP4
                         ctx->Yi.d[3] = BSWAP4(ctr);
 #else
@@ -1492,7 +1480,6 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
 int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx,const unsigned char *tag,
                         size_t len)
 {
-        const union { long one; char little; } is_endian = {1};
         u64 alen = ctx->len.u[0]<<3;
         u64 clen = ctx->len.u[1]<<3;
 #ifdef GCM_FUNCREF_4BIT
@@ -1502,7 +1489,7 @@ int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx,const unsigned char *tag,
         if (ctx->mres || ctx->ares)
                 GCM_MUL(ctx,Xi);
 
-        if (is_endian.little) {
+        if (_BYTE_ORDER == _LITTLE_ENDIAN) {
 #ifdef BSWAP8
                 alen = BSWAP8(alen);
                 clen = BSWAP8(clen);