Factor out a bit of ugly code that truncates the digest to the order_bits

leftmost bits of a longer digest, according to FIPS 183-6, 6.4. Eliminate
a microoptimization that only converts the relevant part of the digest to
a bignum.

ok beck, jsing

1 files changed, 32 insertions, 32 deletions

diff --git a/src/lib/libcrypto/ecdsa/ecs_ossl.c b/src/lib/libcrypto/ecdsa/ecs_ossl.c
index 720fa1f741..87d80642df 100644
--- a/src/lib/libcrypto/ecdsa/ecs_ossl.c
+++ b/src/lib/libcrypto/ecdsa/ecs_ossl.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ecs_ossl.c,v 1.15 2018/06/16 08:11:33 tb Exp $ */
+/* $OpenBSD: ecs_ossl.c,v 1.16 2018/07/10 21:36:02 tb Exp $ */
 /*
  * Written by Nils Larsch for the OpenSSL project
  */
@@ -65,7 +65,9 @@
 #include "bn_lcl.h"
 #include "ecs_locl.h"
 
-static ECDSA_SIG *ecdsa_do_sign(const unsigned char *dgst, int dlen,
+static int ecdsa_prepare_digest(const unsigned char *dgst, int dgst_len,
+    BIGNUM *order, BIGNUM *ret);
+static ECDSA_SIG *ecdsa_do_sign(const unsigned char *dgst, int dgst_len,
     const BIGNUM *, const BIGNUM *, EC_KEY *eckey);
 static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp,
     BIGNUM **rp);
@@ -86,6 +88,30 @@ ECDSA_OpenSSL(void)
 }
 
 static int
+ecdsa_prepare_digest(const unsigned char *dgst, int dgst_len, BIGNUM *order,
+    BIGNUM *ret)
+{
+        int dgst_bits, order_bits;
+
+        if (!BN_bin2bn(dgst, dgst_len, ret)) {
+                ECDSAerror(ERR_R_BN_LIB);
+                return 0;
+        }
+
+        /* FIPS 186-3 6.4: Use order_bits leftmost bits if digest is too long */
+        dgst_bits = 8 * dgst_len;
+        order_bits = BN_num_bits(order);
+        if (dgst_bits > order_bits) {
+                if (!BN_rshift(ret, ret, dgst_bits - order_bits)) {
+                        ECDSAerror(ERR_R_BN_LIB);
+                        return 0;
+                }
+        }
+
+        return 1;
+}
+
+static int
 ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp)
 {
         BN_CTX *ctx = ctx_in;
@@ -220,7 +246,7 @@ ecdsa_do_sign(const unsigned char *dgst, int dgst_len,
         const EC_GROUP *group;
         ECDSA_SIG  *ret;
         ECDSA_DATA *ecdsa;
-        int ok = 0, order_bits;
+        int ok = 0;
 
         ecdsa = ecdsa_check(eckey);
         group = EC_KEY_get0_group(eckey);
@@ -250,21 +276,8 @@ ecdsa_do_sign(const unsigned char *dgst, int dgst_len,
                 goto err;
         }
 
-        /* Truncate digest if it is too long: first truncate whole bytes. */
-        order_bits = BN_num_bits(order);
-        if (8 * dgst_len > order_bits)
-                dgst_len = (order_bits + 7) / 8;
-        if (!BN_bin2bn(dgst, dgst_len, m)) {
-                ECDSAerror(ERR_R_BN_LIB);
+        if (!ecdsa_prepare_digest(dgst, dgst_len, order, m))
                 goto err;
-        }
-        /* If it is still too long, truncate the remaining bits with a shift. */
-        if (8 * dgst_len > order_bits) {
-                if (!BN_rshift(m, m, 8 - (order_bits & 0x7))) {
-                        ECDSAerror(ERR_R_BN_LIB);
-                        goto err;
-                }
-        }
 
         do {
                 if (in_kinv == NULL || in_r == NULL) {
@@ -380,7 +393,7 @@ ecdsa_do_verify(const unsigned char *dgst, int dgst_len, const ECDSA_SIG *sig,
         EC_POINT *point = NULL;
         const EC_GROUP *group;
         const EC_POINT *pub_key;
-        int order_bits, ret = -1;
+        int ret = -1;
 
         if (eckey == NULL || (group = EC_KEY_get0_group(eckey)) == NULL ||
             (pub_key = EC_KEY_get0_public_key(eckey)) == NULL || sig == NULL) {
@@ -418,21 +431,8 @@ ecdsa_do_verify(const unsigned char *dgst, int dgst_len, const ECDSA_SIG *sig,
                 goto err;
         }
 
-        /* Truncate digest if it is too long: first truncate whole bytes. */
-        order_bits = BN_num_bits(order);
-        if (8 * dgst_len > order_bits)
-                dgst_len = (order_bits + 7) / 8;
-        if (!BN_bin2bn(dgst, dgst_len, m)) {
-                ECDSAerror(ERR_R_BN_LIB);
+        if (!ecdsa_prepare_digest(dgst, dgst_len, order, m))
                 goto err;
-        }
-        /* If it is still too long, truncate the remaining bits with a shift. */
-        if (8 * dgst_len > order_bits) {
-                if (!BN_rshift(m, m, 8 - (order_bits & 0x7))) {
-                        ECDSAerror(ERR_R_BN_LIB);
-                        goto err;
-                }
-        }
 
         if (!BN_mod_inverse_ct(u2, sig->s, order, ctx)) {       /* w = inv(s) */
                 ECDSAerror(ERR_R_BN_LIB);