1 files changed, 29 insertions, 19 deletions

diff --git a/src/lib/libcrypto/bn/bn_recp.c b/src/lib/libcrypto/bn/bn_recp.c
index d019941d6b..ef5fdd4708 100644
--- a/src/lib/libcrypto/bn/bn_recp.c
+++ b/src/lib/libcrypto/bn/bn_recp.c
@@ -93,18 +93,19 @@ void BN_RECP_CTX_free(BN_RECP_CTX *recp)
 
 int BN_RECP_CTX_set(BN_RECP_CTX *recp, const BIGNUM *d, BN_CTX *ctx)
         {
-        BN_copy(&(recp->N),d);
-        BN_zero(&(recp->Nr));
+        if (!BN_copy(&(recp->N),d)) return 0;
+        if (!BN_zero(&(recp->Nr))) return 0;
         recp->num_bits=BN_num_bits(d);
         recp->shift=0;
         return(1);
         }
 
-int BN_mod_mul_reciprocal(BIGNUM *r, BIGNUM *x, BIGNUM *y, BN_RECP_CTX *recp,
-             BN_CTX *ctx)
+int BN_mod_mul_reciprocal(BIGNUM *r, const BIGNUM *x, const BIGNUM *y,
+        BN_RECP_CTX *recp, BN_CTX *ctx)
         {
         int ret=0;
         BIGNUM *a;
+        const BIGNUM *ca;
 
         BN_CTX_start(ctx);
         if ((a = BN_CTX_get(ctx)) == NULL) goto err;
@@ -114,19 +115,19 @@ int BN_mod_mul_reciprocal(BIGNUM *r, BIGNUM *x, BIGNUM *y, BN_RECP_CTX *recp,
                         { if (!BN_sqr(a,x,ctx)) goto err; }
                 else
                         { if (!BN_mul(a,x,y,ctx)) goto err; }
+                ca = a;
                 }
         else
-                a=x; /* Just do the mod */
+                ca=x; /* Just do the mod */
 
-        BN_div_recp(NULL,r,a,recp,ctx);
-        ret=1;
+        ret = BN_div_recp(NULL,r,ca,recp,ctx);
 err:
         BN_CTX_end(ctx);
         return(ret);
         }
 
-int BN_div_recp(BIGNUM *dv, BIGNUM *rem, BIGNUM *m, BN_RECP_CTX *recp,
-             BN_CTX *ctx)
+int BN_div_recp(BIGNUM *dv, BIGNUM *rem, const BIGNUM *m,
+        BN_RECP_CTX *recp, BN_CTX *ctx)
         {
         int i,j,ret=0;
         BIGNUM *a,*b,*d,*r;
@@ -146,8 +147,8 @@ int BN_div_recp(BIGNUM *dv, BIGNUM *rem, BIGNUM *m, BN_RECP_CTX *recp,
 
         if (BN_ucmp(m,&(recp->N)) < 0)
                 {
-                BN_zero(d);
-                BN_copy(r,m);
+                if (!BN_zero(d)) return 0;
+                if (!BN_copy(r,m)) return 0;
                 BN_CTX_end(ctx);
                 return(1);
                 }
@@ -157,20 +158,28 @@ int BN_div_recp(BIGNUM *dv, BIGNUM *rem, BIGNUM *m, BN_RECP_CTX *recp,
          * we need multiply ABCDEF by 3 digests of the reciprocal of ab
          *
          */
-        i=BN_num_bits(m);
 
+        /* i := max(BN_num_bits(m), 2*BN_num_bits(N)) */
+        i=BN_num_bits(m);
         j=recp->num_bits<<1;
         if (j>i) i=j;
-        j>>=1;
 
+        /* Nr := round(2^i / N) */
         if (i != recp->shift)
                 recp->shift=BN_reciprocal(&(recp->Nr),&(recp->N),
-                        i,ctx);
+                        i,ctx); /* BN_reciprocal returns i, or -1 for an error */
+        if (recp->shift == -1) goto err;
 
-        if (!BN_rshift(a,m,j)) goto err;
+        /* d := |round(round(m / 2^BN_num_bits(N)) * recp->Nr / 2^(i - BN_num_bits(N)))|
+         *    = |round(round(m / 2^BN_num_bits(N)) * round(2^i / N) / 2^(i - BN_num_bits(N)))|
+         *   <= |(m / 2^BN_num_bits(N)) * (2^i / N) * (2^BN_num_bits(N) / 2^i)|
+         *    = |m/N|
+         */
+        if (!BN_rshift(a,m,recp->num_bits)) goto err;
         if (!BN_mul(b,a,&(recp->Nr),ctx)) goto err;
-        if (!BN_rshift(d,b,i-j)) goto err;
+        if (!BN_rshift(d,b,i-recp->num_bits)) goto err;
         d->neg=0;
+
         if (!BN_mul(b,&(recp->N),d,ctx)) goto err;
         if (!BN_usub(r,m,b)) goto err;
         r->neg=0;
@@ -201,20 +210,21 @@ err:
  * We actually calculate with an extra word of precision, so
  * we can do faster division if the remainder is not required.
  */
-int BN_reciprocal(BIGNUM *r, BIGNUM *m, int len, BN_CTX *ctx)
+/* r := 2^len / m */
+int BN_reciprocal(BIGNUM *r, const BIGNUM *m, int len, BN_CTX *ctx)
         {
         int ret= -1;
         BIGNUM t;
 
         BN_init(&t);
 
-        BN_zero(&t);
+        if (!BN_zero(&t)) goto err;
         if (!BN_set_bit(&t,len)) goto err;
 
         if (!BN_div(r,NULL,&t,m,ctx)) goto err;
+
         ret=len;
 err:
         BN_free(&t);
         return(ret);
         }
-