replace atoi() calls with strtol(). Follow the idiomatic pattern in our

manual page strictly.  Return -2 if the strings are not strict numbers.

The numbers remain in the range of "int".  Range checking for these parameters
is done later in the pkey_*_ctl() functions, or sometimes in functions much
further downstream... but not always!!!
ok millert miod mikeb

6 files changed, 194 insertions, 80 deletions

diff --git a/src/lib/libcrypto/dh/dh_pmeth.c b/src/lib/libcrypto/dh/dh_pmeth.c
index b51e0794b7..cb424ac149 100644
--- a/src/lib/libcrypto/dh/dh_pmeth.c
+++ b/src/lib/libcrypto/dh/dh_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_pmeth.c,v 1.5 2014/06/12 15:49:28 deraadt Exp $ */
+/* $OpenBSD: dh_pmeth.c,v 1.6 2014/06/12 20:40:57 deraadt Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -57,6 +57,7 @@
  */
 
 #include <stdio.h>
+#include <limits.h>
 #include "cryptlib.h"
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
@@ -143,21 +144,38 @@ static int pkey_dh_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 static int pkey_dh_ctrl_str(EVP_PKEY_CTX *ctx,
                         const char *type, const char *value)
         {
-        if (!strcmp(type, "dh_paramgen_prime_len"))
-                {
-                int len;
-                len = atoi(value);
+        long lval;
+        char *ep;
+        int len;
+
+        if (!strcmp(type, "dh_paramgen_prime_len")) {
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                len = lval;
                 return EVP_PKEY_CTX_set_dh_paramgen_prime_len(ctx, len);
-                }
-        if (!strcmp(type, "dh_paramgen_generator"))
-                {
-                int len;
-                len = atoi(value);
+        }
+        if (!strcmp(type, "dh_paramgen_generator")) {
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                len = lval;
                 return EVP_PKEY_CTX_set_dh_paramgen_generator(ctx, len);
-                }
-        return -2;
         }
 
+not_a_number:
+out_of_range:
+        return -2;
+}
+
 static int pkey_dh_paramgen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
         {
         DH *dh = NULL;
diff --git a/src/lib/libcrypto/dsa/dsa_pmeth.c b/src/lib/libcrypto/dsa/dsa_pmeth.c
index 438fa59af2..e75f0153de 100644
--- a/src/lib/libcrypto/dsa/dsa_pmeth.c
+++ b/src/lib/libcrypto/dsa/dsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dsa_pmeth.c,v 1.5 2014/06/12 15:49:28 deraadt Exp $ */
+/* $OpenBSD: dsa_pmeth.c,v 1.6 2014/06/12 20:40:57 deraadt Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -57,6 +57,7 @@
  */
 
 #include <stdio.h>
+#include <limits.h>
 #include "cryptlib.h"
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
@@ -217,24 +218,43 @@ static int pkey_dsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 static int pkey_dsa_ctrl_str(EVP_PKEY_CTX *ctx,
                         const char *type, const char *value)
         {
-        if (!strcmp(type, "dsa_paramgen_bits"))
-                {
+        long lval;
+        char *ep;
+
+        if (!strcmp(type, "dsa_paramgen_bits")) {
                 int nbits;
-                nbits = atoi(value);
+
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                nbits = lval;
                 return EVP_PKEY_CTX_set_dsa_paramgen_bits(ctx, nbits);
-                }
-        if (!strcmp(type, "dsa_paramgen_q_bits"))
-                {
-                int qbits = atoi(value);
+        }
+        if (!strcmp(type, "dsa_paramgen_q_bits")) {
+                int qbits;
+
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                qbits = lval;
                 return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DSA, EVP_PKEY_OP_PARAMGEN,
-                                         EVP_PKEY_CTRL_DSA_PARAMGEN_Q_BITS, qbits, NULL);
-                }
-        if (!strcmp(type, "dsa_paramgen_md"))
-                {
+                    EVP_PKEY_CTRL_DSA_PARAMGEN_Q_BITS, qbits, NULL);
+        }
+        if (!strcmp(type, "dsa_paramgen_md")){
                 return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DSA, EVP_PKEY_OP_PARAMGEN,
-                                         EVP_PKEY_CTRL_DSA_PARAMGEN_MD, 0, 
-                                         (void *)EVP_get_digestbyname(value));
-                }
+                    EVP_PKEY_CTRL_DSA_PARAMGEN_MD, 0, 
+                    (void *)EVP_get_digestbyname(value));
+        }
+not_a_number:
+out_of_range:
         return -2;
         }
 
diff --git a/src/lib/libcrypto/rsa/rsa_pmeth.c b/src/lib/libcrypto/rsa/rsa_pmeth.c
index a611fc3461..1f9d826014 100644
--- a/src/lib/libcrypto/rsa/rsa_pmeth.c
+++ b/src/lib/libcrypto/rsa/rsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_pmeth.c,v 1.7 2014/06/12 15:49:30 deraadt Exp $ */
+/* $OpenBSD: rsa_pmeth.c,v 1.8 2014/06/12 20:40:57 deraadt Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -57,6 +57,7 @@
  */
 
 #include <stdio.h>
+#include <limits.h>
 #include "cryptlib.h"
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
@@ -518,6 +519,9 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                         const char *type, const char *value)
         {
+        long lval;
+        char *ep;
+
         if (!value)
                 {
                 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
@@ -549,22 +553,35 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
                 }
 
-        if (!strcmp(type, "rsa_pss_saltlen"))
-                {
+        if (!strcmp(type, "rsa_pss_saltlen")) {
                 int saltlen;
-                saltlen = atoi(value);
+
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                saltlen = lval;
                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
-                }
+        }
 
-        if (!strcmp(type, "rsa_keygen_bits"))
-                {
+        if (!strcmp(type, "rsa_keygen_bits")) {
                 int nbits;
-                nbits = atoi(value);
+
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                nbits = lval;
                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
-                }
+        }
 
-        if (!strcmp(type, "rsa_keygen_pubexp"))
-                {
+        if (!strcmp(type, "rsa_keygen_pubexp")) {
                 int ret;
                 BIGNUM *pubexp = NULL;
                 if (!BN_asc2bn(&pubexp, value))
@@ -573,10 +590,12 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                 if (ret <= 0)
                         BN_free(pubexp);
                 return ret;
-                }
+        }
 
+not_a_number:
+out_of_range:
         return -2;
-        }
+}
 
 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
         {
diff --git a/src/lib/libssl/src/crypto/dh/dh_pmeth.c b/src/lib/libssl/src/crypto/dh/dh_pmeth.c
index b51e0794b7..cb424ac149 100644
--- a/src/lib/libssl/src/crypto/dh/dh_pmeth.c
+++ b/src/lib/libssl/src/crypto/dh/dh_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_pmeth.c,v 1.5 2014/06/12 15:49:28 deraadt Exp $ */
+/* $OpenBSD: dh_pmeth.c,v 1.6 2014/06/12 20:40:57 deraadt Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -57,6 +57,7 @@
  */
 
 #include <stdio.h>
+#include <limits.h>
 #include "cryptlib.h"
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
@@ -143,21 +144,38 @@ static int pkey_dh_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 static int pkey_dh_ctrl_str(EVP_PKEY_CTX *ctx,
                         const char *type, const char *value)
         {
-        if (!strcmp(type, "dh_paramgen_prime_len"))
-                {
-                int len;
-                len = atoi(value);
+        long lval;
+        char *ep;
+        int len;
+
+        if (!strcmp(type, "dh_paramgen_prime_len")) {
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                len = lval;
                 return EVP_PKEY_CTX_set_dh_paramgen_prime_len(ctx, len);
-                }
-        if (!strcmp(type, "dh_paramgen_generator"))
-                {
-                int len;
-                len = atoi(value);
+        }
+        if (!strcmp(type, "dh_paramgen_generator")) {
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                len = lval;
                 return EVP_PKEY_CTX_set_dh_paramgen_generator(ctx, len);
-                }
-        return -2;
         }
 
+not_a_number:
+out_of_range:
+        return -2;
+}
+
 static int pkey_dh_paramgen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
         {
         DH *dh = NULL;
diff --git a/src/lib/libssl/src/crypto/dsa/dsa_pmeth.c b/src/lib/libssl/src/crypto/dsa/dsa_pmeth.c
index 438fa59af2..e75f0153de 100644
--- a/src/lib/libssl/src/crypto/dsa/dsa_pmeth.c
+++ b/src/lib/libssl/src/crypto/dsa/dsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dsa_pmeth.c,v 1.5 2014/06/12 15:49:28 deraadt Exp $ */
+/* $OpenBSD: dsa_pmeth.c,v 1.6 2014/06/12 20:40:57 deraadt Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -57,6 +57,7 @@
  */
 
 #include <stdio.h>
+#include <limits.h>
 #include "cryptlib.h"
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
@@ -217,24 +218,43 @@ static int pkey_dsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 static int pkey_dsa_ctrl_str(EVP_PKEY_CTX *ctx,
                         const char *type, const char *value)
         {
-        if (!strcmp(type, "dsa_paramgen_bits"))
-                {
+        long lval;
+        char *ep;
+
+        if (!strcmp(type, "dsa_paramgen_bits")) {
                 int nbits;
-                nbits = atoi(value);
+
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                nbits = lval;
                 return EVP_PKEY_CTX_set_dsa_paramgen_bits(ctx, nbits);
-                }
-        if (!strcmp(type, "dsa_paramgen_q_bits"))
-                {
-                int qbits = atoi(value);
+        }
+        if (!strcmp(type, "dsa_paramgen_q_bits")) {
+                int qbits;
+
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                qbits = lval;
                 return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DSA, EVP_PKEY_OP_PARAMGEN,
-                                         EVP_PKEY_CTRL_DSA_PARAMGEN_Q_BITS, qbits, NULL);
-                }
-        if (!strcmp(type, "dsa_paramgen_md"))
-                {
+                    EVP_PKEY_CTRL_DSA_PARAMGEN_Q_BITS, qbits, NULL);
+        }
+        if (!strcmp(type, "dsa_paramgen_md")){
                 return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_DSA, EVP_PKEY_OP_PARAMGEN,
-                                         EVP_PKEY_CTRL_DSA_PARAMGEN_MD, 0, 
-                                         (void *)EVP_get_digestbyname(value));
-                }
+                    EVP_PKEY_CTRL_DSA_PARAMGEN_MD, 0, 
+                    (void *)EVP_get_digestbyname(value));
+        }
+not_a_number:
+out_of_range:
         return -2;
         }
 
diff --git a/src/lib/libssl/src/crypto/rsa/rsa_pmeth.c b/src/lib/libssl/src/crypto/rsa/rsa_pmeth.c
index a611fc3461..1f9d826014 100644
--- a/src/lib/libssl/src/crypto/rsa/rsa_pmeth.c
+++ b/src/lib/libssl/src/crypto/rsa/rsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_pmeth.c,v 1.7 2014/06/12 15:49:30 deraadt Exp $ */
+/* $OpenBSD: rsa_pmeth.c,v 1.8 2014/06/12 20:40:57 deraadt Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -57,6 +57,7 @@
  */
 
 #include <stdio.h>
+#include <limits.h>
 #include "cryptlib.h"
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
@@ -518,6 +519,9 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                         const char *type, const char *value)
         {
+        long lval;
+        char *ep;
+
         if (!value)
                 {
                 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
@@ -549,22 +553,35 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
                 }
 
-        if (!strcmp(type, "rsa_pss_saltlen"))
-                {
+        if (!strcmp(type, "rsa_pss_saltlen")) {
                 int saltlen;
-                saltlen = atoi(value);
+
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                saltlen = lval;
                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
-                }
+        }
 
-        if (!strcmp(type, "rsa_keygen_bits"))
-                {
+        if (!strcmp(type, "rsa_keygen_bits")) {
                 int nbits;
-                nbits = atoi(value);
+
+                errno = 0;
+                lval = strtol(value, &ep, 10);
+                if (value[0] == '\0' || *ep != '\0')
+                        goto not_a_number;
+                if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) ||
+                    (lval > INT_MAX || lval < INT_MIN))
+                        goto out_of_range;
+                nbits = lval;
                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
-                }
+        }
 
-        if (!strcmp(type, "rsa_keygen_pubexp"))
-                {
+        if (!strcmp(type, "rsa_keygen_pubexp")) {
                 int ret;
                 BIGNUM *pubexp = NULL;
                 if (!BN_asc2bn(&pubexp, value))
@@ -573,10 +590,12 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                 if (ret <= 0)
                         BN_free(pubexp);
                 return ret;
-                }
+        }
 
+not_a_number:
+out_of_range:
         return -2;
-        }
+}
 
 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
         {