Send an illegal_parameter alert if a client sends us invalid DH key

shares.  Previously we would fail and just close the pipe.

Fixes the remaining failing test-dhe-rsa-key-exchange-with-bad-messages.py
tests of tlsfuzzer.

ok beck (earlier version) jsing

1 files changed, 15 insertions, 3 deletions

diff --git a/src/lib/libssl/ssl_srvr.c b/src/lib/libssl/ssl_srvr.c
index fac24f4d00..15dfdc35b1 100644
--- a/src/lib/libssl/ssl_srvr.c
+++ b/src/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.77 2020/05/31 16:36:35 jsing Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.78 2020/06/01 08:04:02 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1778,7 +1778,8 @@ ssl3_get_client_kex_rsa(SSL *s, CBS *cbs)
 static int
 ssl3_get_client_kex_dhe(SSL *s, CBS *cbs)
 {
-        int key_size = 0, key_len, al;
+        int key_size = 0;
+        int key_is_invalid, key_len, al;
         unsigned char *key = NULL;
         BIGNUM *bn = NULL;
         CBS dh_Yc;
@@ -1809,9 +1810,20 @@ ssl3_get_client_kex_dhe(SSL *s, CBS *cbs)
                 SSLerror(s, ERR_R_MALLOC_FAILURE);
                 goto err;
         }
+        if (!DH_check_pub_key(dh, bn, &key_is_invalid)) {
+                al = SSL_AD_INTERNAL_ERROR;
+                SSLerror(s, ERR_R_DH_LIB);
+                goto f_err;
+        }
+        if (key_is_invalid) {
+                al = SSL_AD_ILLEGAL_PARAMETER;
+                SSLerror(s, ERR_R_DH_LIB);
+                goto f_err;
+        }
         if ((key_len = DH_compute_key(key, bn, dh)) <= 0) {
+                al = SSL_AD_INTERNAL_ERROR;
                 SSLerror(s, ERR_R_DH_LIB);
-                goto err;
+                goto f_err;
         }
 
         s->session->master_key_length = tls1_generate_master_secret(s,