6 files changed, 60 insertions, 37 deletions

diff --git a/src/lib/libcrypto/bn/bn.h b/src/lib/libcrypto/bn/bn.h
index e484b7fc11..a0bc47837d 100644
--- a/src/lib/libcrypto/bn/bn.h
+++ b/src/lib/libcrypto/bn/bn.h
@@ -253,6 +253,24 @@ extern "C" {
 #define BN_HEX_FMT2     "%08X"
 #endif
 
+/* 2011-02-22 SMS.
+ * In various places, a size_t variable or a type cast to size_t was
+ * used to perform integer-only operations on pointers.  This failed on
+ * VMS with 64-bit pointers (CC /POINTER_SIZE = 64) because size_t is
+ * still only 32 bits.  What's needed in these cases is an integer type
+ * with the same size as a pointer, which size_t is not certain to be. 
+ * The only fix here is VMS-specific.
+ */
+#if defined(OPENSSL_SYS_VMS)
+# if __INITIAL_POINTER_SIZE == 64
+#  define PTR_SIZE_INT long long
+# else /* __INITIAL_POINTER_SIZE == 64 */
+#  define PTR_SIZE_INT int
+# endif /* __INITIAL_POINTER_SIZE == 64 [else] */
+#else /* defined(OPENSSL_SYS_VMS) */
+# define PTR_SIZE_INT size_t
+#endif /* defined(OPENSSL_SYS_VMS) [else] */
+
 #define BN_DEFAULT_BITS 1280
 
 #define BN_FLG_MALLOCED         0x01
diff --git a/src/lib/libcrypto/bn/bn_exp2.c b/src/lib/libcrypto/bn/bn_exp2.c
index b3f43cec8c..bd0c34b91b 100644
--- a/src/lib/libcrypto/bn/bn_exp2.c
+++ b/src/lib/libcrypto/bn/bn_exp2.c
@@ -301,7 +301,8 @@ int BN_mod_exp2_mont(BIGNUM *rr, const BIGNUM *a1, const BIGNUM *p1,
                         r_is_one = 0;
                         }
                 }
-        BN_from_montgomery(rr,r,mont,ctx);
+        if (!BN_from_montgomery(rr,r,mont,ctx))
+                goto err;
         ret=1;
 err:
         if ((in_mont == NULL) && (mont != NULL)) BN_MONT_CTX_free(mont);
diff --git a/src/lib/libcrypto/bn/bn_gf2m.c b/src/lib/libcrypto/bn/bn_gf2m.c
index 527b0fa15b..432a3aa338 100644
--- a/src/lib/libcrypto/bn/bn_gf2m.c
+++ b/src/lib/libcrypto/bn/bn_gf2m.c
@@ -545,6 +545,7 @@ int BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
                 {
                 while (!BN_is_odd(u))
                         {
+                        if (BN_is_zero(u)) goto err;
                         if (!BN_rshift1(u, u)) goto err;
                         if (BN_is_odd(b))
                                 {
diff --git a/src/lib/libcrypto/bn/bn_mont.c b/src/lib/libcrypto/bn/bn_mont.c
index 7224637ab3..1a866880f5 100644
--- a/src/lib/libcrypto/bn/bn_mont.c
+++ b/src/lib/libcrypto/bn/bn_mont.c
@@ -277,7 +277,7 @@ static int BN_from_montgomery_word(BIGNUM *ret, BIGNUM *r, BN_MONT_CTX *mont)
         m1|=m2;                 /* (al!=ri) */
         m1|=(0-(size_t)v);      /* (al!=ri || v) */
         m1&=~m2;                /* (al!=ri || v) && !al>ri */
-        nrp=(BN_ULONG *)(((size_t)rp&~m1)|((size_t)ap&m1));
+        nrp=(BN_ULONG *)(((PTR_SIZE_INT)rp&~m1)|((PTR_SIZE_INT)ap&m1));
         }
 
         /* 'i<ri' is chosen to eliminate dependency on input data, even
diff --git a/src/lib/libcrypto/bn/bn_mul.c b/src/lib/libcrypto/bn/bn_mul.c
index a0e9ec3b46..12e5be80eb 100644
--- a/src/lib/libcrypto/bn/bn_mul.c
+++ b/src/lib/libcrypto/bn/bn_mul.c
@@ -551,7 +551,7 @@ void bn_mul_part_recursive(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n,
              int tna, int tnb, BN_ULONG *t)
         {
         int i,j,n2=n*2;
-        int c1,c2,neg,zero;
+        int c1,c2,neg;
         BN_ULONG ln,lo,*p;
 
 # ifdef BN_COUNT
@@ -567,7 +567,7 @@ void bn_mul_part_recursive(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n,
         /* r=(a[0]-a[1])*(b[1]-b[0]) */
         c1=bn_cmp_part_words(a,&(a[n]),tna,n-tna);
         c2=bn_cmp_part_words(&(b[n]),b,tnb,tnb-n);
-        zero=neg=0;
+        neg=0;
         switch (c1*3+c2)
                 {
         case -4:
@@ -575,7 +575,6 @@ void bn_mul_part_recursive(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n,
                 bn_sub_part_words(&(t[n]),b,      &(b[n]),tnb,n-tnb); /* - */
                 break;
         case -3:
-                zero=1;
                 /* break; */
         case -2:
                 bn_sub_part_words(t,      &(a[n]),a,      tna,tna-n); /* - */
@@ -585,7 +584,6 @@ void bn_mul_part_recursive(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n,
         case -1:
         case 0:
         case 1:
-                zero=1;
                 /* break; */
         case 2:
                 bn_sub_part_words(t,      a,      &(a[n]),tna,n-tna); /* + */
@@ -593,7 +591,6 @@ void bn_mul_part_recursive(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n,
                 neg=1;
                 break;
         case 3:
-                zero=1;
                 /* break; */
         case 4:
                 bn_sub_part_words(t,      a,      &(a[n]),tna,n-tna);
@@ -1012,7 +1009,6 @@ int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
                 {
                 if (i >= -1 && i <= 1)
                         {
-                        int sav_j =0;
                         /* Find out the power of two lower or equal
                            to the longest of the two numbers */
                         if (i >= 0)
@@ -1023,7 +1019,6 @@ int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
                                 {
                                 j = BN_num_bits_word((BN_ULONG)bl);
                                 }
-                        sav_j = j;
                         j = 1<<(j-1);
                         assert(j <= al || j <= bl);
                         k = j+j;
diff --git a/src/lib/libcrypto/bn/bn_nist.c b/src/lib/libcrypto/bn/bn_nist.c
index 2ca5b01391..c6de032696 100644
--- a/src/lib/libcrypto/bn/bn_nist.c
+++ b/src/lib/libcrypto/bn/bn_nist.c
@@ -354,7 +354,7 @@ int BN_nist_mod_192(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
                  buf[BN_NIST_192_TOP],
                  c_d[BN_NIST_192_TOP],
                 *res;
-        size_t   mask;
+        PTR_SIZE_INT mask;
         static const BIGNUM _bignum_nist_p_192_sqr = {
                 (BN_ULONG *)_nist_p_192_sqr,
                 sizeof(_nist_p_192_sqr)/sizeof(_nist_p_192_sqr[0]),
@@ -405,9 +405,10 @@ int BN_nist_mod_192(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
          * 'tmp=result-modulus; if (!carry || !borrow) result=tmp;'
          * this is what happens below, but without explicit if:-) a.
          */
-        mask  = 0-(size_t)bn_sub_words(c_d,r_d,_nist_p_192[0],BN_NIST_192_TOP);
-        mask &= 0-(size_t)carry;
-        res   = (BN_ULONG *)(((size_t)c_d&~mask) | ((size_t)r_d&mask));
+        mask  = 0-(PTR_SIZE_INT)bn_sub_words(c_d,r_d,_nist_p_192[0],BN_NIST_192_TOP);
+        mask &= 0-(PTR_SIZE_INT)carry;
+        res   = (BN_ULONG *)
+         (((PTR_SIZE_INT)c_d&~mask) | ((PTR_SIZE_INT)r_d&mask));
         nist_cp_bn(r_d, res, BN_NIST_192_TOP);
         r->top = BN_NIST_192_TOP;
         bn_correct_top(r);
@@ -438,8 +439,8 @@ int BN_nist_mod_224(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
                  buf[BN_NIST_224_TOP],
                  c_d[BN_NIST_224_TOP],
                 *res;
-        size_t   mask;
-        union { bn_addsub_f f; size_t p; } u;
+        PTR_SIZE_INT mask;
+        union { bn_addsub_f f; PTR_SIZE_INT p; } u;
         static const BIGNUM _bignum_nist_p_224_sqr = {
                 (BN_ULONG *)_nist_p_224_sqr,
                 sizeof(_nist_p_224_sqr)/sizeof(_nist_p_224_sqr[0]),
@@ -510,16 +511,18 @@ int BN_nist_mod_224(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
                  * to be compared to the modulus and conditionally
                  * adjusted by *subtracting* the latter. */
                 carry = (int)bn_add_words(r_d,r_d,_nist_p_224[-carry-1],BN_NIST_224_TOP);
-                mask = 0-(size_t)carry;
-                u.p = ((size_t)bn_sub_words&mask) | ((size_t)bn_add_words&~mask);
+                mask = 0-(PTR_SIZE_INT)carry;
+                u.p = ((PTR_SIZE_INT)bn_sub_words&mask) |
+                 ((PTR_SIZE_INT)bn_add_words&~mask);
                 }
         else
                 carry = 1;
 
         /* otherwise it's effectively same as in BN_nist_mod_192... */
-        mask  = 0-(size_t)(*u.f)(c_d,r_d,_nist_p_224[0],BN_NIST_224_TOP);
-        mask &= 0-(size_t)carry;
-        res   = (BN_ULONG *)(((size_t)c_d&~mask) | ((size_t)r_d&mask));
+        mask  = 0-(PTR_SIZE_INT)(*u.f)(c_d,r_d,_nist_p_224[0],BN_NIST_224_TOP);
+        mask &= 0-(PTR_SIZE_INT)carry;
+        res   = (BN_ULONG *)(((PTR_SIZE_INT)c_d&~mask) |
+         ((PTR_SIZE_INT)r_d&mask));
         nist_cp_bn(r_d, res, BN_NIST_224_TOP);
         r->top = BN_NIST_224_TOP;
         bn_correct_top(r);
@@ -549,8 +552,8 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
                  buf[BN_NIST_256_TOP],
                  c_d[BN_NIST_256_TOP],
                 *res;
-        size_t   mask;
-        union { bn_addsub_f f; size_t p; } u;
+        PTR_SIZE_INT mask;
+        union { bn_addsub_f f; PTR_SIZE_INT p; } u;
         static const BIGNUM _bignum_nist_p_256_sqr = {
                 (BN_ULONG *)_nist_p_256_sqr,
                 sizeof(_nist_p_256_sqr)/sizeof(_nist_p_256_sqr[0]),
@@ -629,15 +632,17 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
         else if (carry < 0)
                 {
                 carry = (int)bn_add_words(r_d,r_d,_nist_p_256[-carry-1],BN_NIST_256_TOP);
-                mask = 0-(size_t)carry;
-                u.p = ((size_t)bn_sub_words&mask) | ((size_t)bn_add_words&~mask);
+                mask = 0-(PTR_SIZE_INT)carry;
+                u.p = ((PTR_SIZE_INT)bn_sub_words&mask) |
+                 ((PTR_SIZE_INT)bn_add_words&~mask);
                 }
         else
                 carry = 1;
 
-        mask  = 0-(size_t)(*u.f)(c_d,r_d,_nist_p_256[0],BN_NIST_256_TOP);
-        mask &= 0-(size_t)carry;
-        res   = (BN_ULONG *)(((size_t)c_d&~mask) | ((size_t)r_d&mask));
+        mask  = 0-(PTR_SIZE_INT)(*u.f)(c_d,r_d,_nist_p_256[0],BN_NIST_256_TOP);
+        mask &= 0-(PTR_SIZE_INT)carry;
+        res   = (BN_ULONG *)(((PTR_SIZE_INT)c_d&~mask) |
+         ((PTR_SIZE_INT)r_d&mask));
         nist_cp_bn(r_d, res, BN_NIST_256_TOP);
         r->top = BN_NIST_256_TOP;
         bn_correct_top(r);
@@ -671,8 +676,8 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
                  buf[BN_NIST_384_TOP],
                  c_d[BN_NIST_384_TOP],
                 *res;
-        size_t   mask;
-        union { bn_addsub_f f; size_t p; } u;
+        PTR_SIZE_INT mask;
+        union { bn_addsub_f f; PTR_SIZE_INT p; } u;
         static const BIGNUM _bignum_nist_p_384_sqr = {
                 (BN_ULONG *)_nist_p_384_sqr,
                 sizeof(_nist_p_384_sqr)/sizeof(_nist_p_384_sqr[0]),
@@ -754,15 +759,17 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
         else if (carry < 0)
                 {
                 carry = (int)bn_add_words(r_d,r_d,_nist_p_384[-carry-1],BN_NIST_384_TOP);
-                mask = 0-(size_t)carry;
-                u.p = ((size_t)bn_sub_words&mask) | ((size_t)bn_add_words&~mask);
+                mask = 0-(PTR_SIZE_INT)carry;
+                u.p = ((PTR_SIZE_INT)bn_sub_words&mask) |
+                 ((PTR_SIZE_INT)bn_add_words&~mask);
                 }
         else
                 carry = 1;
 
-        mask  = 0-(size_t)(*u.f)(c_d,r_d,_nist_p_384[0],BN_NIST_384_TOP);
-        mask &= 0-(size_t)carry;
-        res   = (BN_ULONG *)(((size_t)c_d&~mask) | ((size_t)r_d&mask));
+        mask  = 0-(PTR_SIZE_INT)(*u.f)(c_d,r_d,_nist_p_384[0],BN_NIST_384_TOP);
+        mask &= 0-(PTR_SIZE_INT)carry;
+        res   = (BN_ULONG *)(((PTR_SIZE_INT)c_d&~mask) |
+         ((PTR_SIZE_INT)r_d&mask));
         nist_cp_bn(r_d, res, BN_NIST_384_TOP);
         r->top = BN_NIST_384_TOP;
         bn_correct_top(r);
@@ -781,7 +788,7 @@ int BN_nist_mod_521(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
         BN_ULONG *r_d, *a_d = a->d,
                  t_d[BN_NIST_521_TOP],
                  val,tmp,*res;
-        size_t  mask;
+        PTR_SIZE_INT mask;
         static const BIGNUM _bignum_nist_p_521_sqr = {
                 (BN_ULONG *)_nist_p_521_sqr,
                 sizeof(_nist_p_521_sqr)/sizeof(_nist_p_521_sqr[0]),
@@ -826,8 +833,9 @@ int BN_nist_mod_521(BIGNUM *r, const BIGNUM *a, const BIGNUM *field,
         r_d[i] &= BN_NIST_521_TOP_MASK;
 
         bn_add_words(r_d,r_d,t_d,BN_NIST_521_TOP);
-        mask = 0-(size_t)bn_sub_words(t_d,r_d,_nist_p_521,BN_NIST_521_TOP);
-        res  = (BN_ULONG *)(((size_t)t_d&~mask) | ((size_t)r_d&mask));
+        mask = 0-(PTR_SIZE_INT)bn_sub_words(t_d,r_d,_nist_p_521,BN_NIST_521_TOP);
+        res  = (BN_ULONG *)(((PTR_SIZE_INT)t_d&~mask) |
+         ((PTR_SIZE_INT)r_d&mask));
         nist_cp_bn(r_d,res,BN_NIST_521_TOP);
         r->top = BN_NIST_521_TOP;
         bn_correct_top(r);