Add record processing limit to DTLS code.

This is effectively the same record processing limit that was previously
added to the legacy TLS stack - without this a single session can be made
to spin on a stream of alerts or other similar records.

ok beck@ tb@

2 files changed, 18 insertions, 3 deletions

diff --git a/src/lib/libssl/d1_pkt.c b/src/lib/libssl/d1_pkt.c
index 9601a39e3a..f0f393b0fd 100644
--- a/src/lib/libssl/d1_pkt.c
+++ b/src/lib/libssl/d1_pkt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: d1_pkt.c,v 1.114 2021/10/25 10:09:28 jsing Exp $ */
+/* $OpenBSD: d1_pkt.c,v 1.115 2021/10/25 10:14:48 jsing Exp $ */
 /*
  * DTLS implementation written by Nagendra Modadugu
  * (nagendra@cs.stanford.edu) for the OpenSSL project 2005.
@@ -514,6 +514,7 @@ int
 dtls1_read_bytes(SSL *s, int type, unsigned char *buf, int len, int peek)
 {
         int al, i, ret;
+        int rrcount = 0;
         unsigned int n;
         SSL3_RECORD_INTERNAL *rr;
 
@@ -539,6 +540,19 @@ dtls1_read_bytes(SSL *s, int type, unsigned char *buf, int len, int peek)
         }
 
  start:
+        /*
+         * Do not process more than three consecutive records, otherwise the
+         * peer can cause us to loop indefinitely. Instead, return with an
+         * SSL_ERROR_WANT_READ so the caller can choose when to handle further
+         * processing. In the future, the total number of non-handshake and
+         * non-application data records per connection should probably also be
+         * limited...
+         */
+        if (rrcount++ >= 3) {
+                ssl_force_want_read(s);
+                return -1;
+        }
+
         s->internal->rwstate = SSL_NOTHING;
 
         /* S3I(s)->rrec.type        - is the type of record
diff --git a/src/lib/libssl/ssl_pkt.c b/src/lib/libssl/ssl_pkt.c
index 8a5f97e5c7..e3101eefba 100644
--- a/src/lib/libssl/ssl_pkt.c
+++ b/src/lib/libssl/ssl_pkt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_pkt.c,v 1.51 2021/10/25 10:09:28 jsing Exp $ */
+/* $OpenBSD: ssl_pkt.c,v 1.52 2021/10/25 10:14:48 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -715,7 +715,8 @@ ssl3_write_pending(SSL *s, int type, const unsigned char *buf, unsigned int len)
 int
 ssl3_read_bytes(SSL *s, int type, unsigned char *buf, int len, int peek)
 {
-        int al, i, ret, rrcount = 0;
+        int al, i, ret;
+        int rrcount = 0;
         unsigned int n;
         SSL3_RECORD_INTERNAL *rr;