Provide functions for starting, finishing and writing SSL handshake

messages. This will allow for removal of repeated/duplicated code.

Additionally, DTLS was written by wholesale copying of the SSL/TLS code,
with some DTLS specifics being added to the duplicated code. Since these
SSL handshake message functions know how to handle both SSL/TLS and DTLS,
upon conversion the duplicate versions will become identical (or close to),
at which point the DTLS versions can be removed and the SSL/TLS versions
used for both protocols.

Partially based on similar changes in OpenSSL.

ok miod@

6 files changed, 120 insertions, 8 deletions

diff --git a/src/lib/libssl/d1_both.c b/src/lib/libssl/d1_both.c
index 2dc26e38cb..bff683d06c 100644
--- a/src/lib/libssl/d1_both.c
+++ b/src/lib/libssl/d1_both.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: d1_both.c,v 1.30 2014/11/16 14:12:47 jsing Exp $ */
+/* $OpenBSD: d1_both.c,v 1.31 2014/12/14 16:07:26 jsing Exp $ */
 /*
  * DTLS implementation written by Nagendra Modadugu
  * (nagendra@cs.stanford.edu) for the OpenSSL project 2005.
@@ -1140,6 +1140,8 @@ dtls1_buffer_message(SSL *s, int is_ccs)
         hm_fragment *frag;
         unsigned char seq64be[8];
 
+        /* Buffer the messsage in order to handle DTLS retransmissions. */
+
         /*
          * This function is called immediately after a message has
          * been serialized
diff --git a/src/lib/libssl/s3_lib.c b/src/lib/libssl/s3_lib.c
index e60f004e57..f372b6523c 100644
--- a/src/lib/libssl/s3_lib.c
+++ b/src/lib/libssl/s3_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: s3_lib.c,v 1.89 2014/12/14 15:30:50 jsing Exp $ */
+/* $OpenBSD: s3_lib.c,v 1.90 2014/12/14 16:07:26 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1837,6 +1837,58 @@ ssl3_pending(const SSL *s)
             s->s3->rrec.length : 0;
 }
 
+unsigned char *
+ssl3_handshake_msg_start(SSL *s, uint8_t msg_type)
+{
+        unsigned char *d, *p;
+        int hdr_len;
+
+        d = p = (unsigned char *)s->init_buf->data;
+
+        hdr_len = SSL_IS_DTLS(s) ? DTLS1_HM_HEADER_LENGTH :
+            SSL3_HM_HEADER_LENGTH;
+
+        /* Handshake message type and length. */
+        *(p++) = msg_type;
+        l2n3(0, p);
+
+        return (d + hdr_len);
+}
+
+void
+ssl3_handshake_msg_finish(SSL *s, unsigned int len)
+{
+        unsigned char *d, *p;
+        uint8_t msg_type;
+        int hdr_len;
+
+        d = p = (unsigned char *)s->init_buf->data;
+
+        hdr_len = SSL_IS_DTLS(s) ? DTLS1_HM_HEADER_LENGTH :
+            SSL3_HM_HEADER_LENGTH;
+
+        /* Handshake message length. */
+        msg_type = *(p++);
+        l2n3(len, p);
+
+        s->init_num = hdr_len + (int)len;
+        s->init_off = 0;
+
+        if (SSL_IS_DTLS(s)) {
+                dtls1_set_message_header(s, d, msg_type, len, 0, len);
+                dtls1_buffer_message(s, 0);
+        }
+}
+
+int
+ssl3_handshake_write(SSL *s)
+{
+        if (SSL_IS_DTLS(s))
+                return dtls1_do_write(s, SSL3_RT_HANDSHAKE);
+
+        return ssl3_do_write(s, SSL3_RT_HANDSHAKE);
+}
+
 int
 ssl3_new(SSL *s)
 {
diff --git a/src/lib/libssl/src/ssl/d1_both.c b/src/lib/libssl/src/ssl/d1_both.c
index 2dc26e38cb..bff683d06c 100644
--- a/src/lib/libssl/src/ssl/d1_both.c
+++ b/src/lib/libssl/src/ssl/d1_both.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: d1_both.c,v 1.30 2014/11/16 14:12:47 jsing Exp $ */
+/* $OpenBSD: d1_both.c,v 1.31 2014/12/14 16:07:26 jsing Exp $ */
 /*
  * DTLS implementation written by Nagendra Modadugu
  * (nagendra@cs.stanford.edu) for the OpenSSL project 2005.
@@ -1140,6 +1140,8 @@ dtls1_buffer_message(SSL *s, int is_ccs)
         hm_fragment *frag;
         unsigned char seq64be[8];
 
+        /* Buffer the messsage in order to handle DTLS retransmissions. */
+
         /*
          * This function is called immediately after a message has
          * been serialized
diff --git a/src/lib/libssl/src/ssl/s3_lib.c b/src/lib/libssl/src/ssl/s3_lib.c
index e60f004e57..f372b6523c 100644
--- a/src/lib/libssl/src/ssl/s3_lib.c
+++ b/src/lib/libssl/src/ssl/s3_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: s3_lib.c,v 1.89 2014/12/14 15:30:50 jsing Exp $ */
+/* $OpenBSD: s3_lib.c,v 1.90 2014/12/14 16:07:26 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1837,6 +1837,58 @@ ssl3_pending(const SSL *s)
             s->s3->rrec.length : 0;
 }
 
+unsigned char *
+ssl3_handshake_msg_start(SSL *s, uint8_t msg_type)
+{
+        unsigned char *d, *p;
+        int hdr_len;
+
+        d = p = (unsigned char *)s->init_buf->data;
+
+        hdr_len = SSL_IS_DTLS(s) ? DTLS1_HM_HEADER_LENGTH :
+            SSL3_HM_HEADER_LENGTH;
+
+        /* Handshake message type and length. */
+        *(p++) = msg_type;
+        l2n3(0, p);
+
+        return (d + hdr_len);
+}
+
+void
+ssl3_handshake_msg_finish(SSL *s, unsigned int len)
+{
+        unsigned char *d, *p;
+        uint8_t msg_type;
+        int hdr_len;
+
+        d = p = (unsigned char *)s->init_buf->data;
+
+        hdr_len = SSL_IS_DTLS(s) ? DTLS1_HM_HEADER_LENGTH :
+            SSL3_HM_HEADER_LENGTH;
+
+        /* Handshake message length. */
+        msg_type = *(p++);
+        l2n3(len, p);
+
+        s->init_num = hdr_len + (int)len;
+        s->init_off = 0;
+
+        if (SSL_IS_DTLS(s)) {
+                dtls1_set_message_header(s, d, msg_type, len, 0, len);
+                dtls1_buffer_message(s, 0);
+        }
+}
+
+int
+ssl3_handshake_write(SSL *s)
+{
+        if (SSL_IS_DTLS(s))
+                return dtls1_do_write(s, SSL3_RT_HANDSHAKE);
+
+        return ssl3_do_write(s, SSL3_RT_HANDSHAKE);
+}
+
 int
 ssl3_new(SSL *s)
 {
diff --git a/src/lib/libssl/src/ssl/ssl3.h b/src/lib/libssl/src/ssl/ssl3.h
index 9270ded96f..b5df1056ab 100644
--- a/src/lib/libssl/src/ssl/ssl3.h
+++ b/src/lib/libssl/src/ssl/ssl3.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl3.h,v 1.31 2014/12/14 15:30:50 jsing Exp $ */
+/* $OpenBSD: ssl3.h,v 1.32 2014/12/14 16:07:26 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -231,9 +231,11 @@ extern "C" {
 #define SSL3_RANDOM_SIZE                        32
 #define SSL3_SEQUENCE_SIZE                      8
 #define SSL3_SESSION_ID_SIZE                    32
-#define SSL3_RT_HEADER_LENGTH                   5
 #define SSL3_CIPHER_VALUE_SIZE                  2
 
+#define SSL3_RT_HEADER_LENGTH                   5
+#define SSL3_HM_HEADER_LENGTH                   4
+
 #ifndef SSL3_ALIGN_PAYLOAD
  /* Some will argue that this increases memory footprint, but it's
   * not actually true. Point is that malloc has to return at least
diff --git a/src/lib/libssl/ssl3.h b/src/lib/libssl/ssl3.h
index 9270ded96f..b5df1056ab 100644
--- a/src/lib/libssl/ssl3.h
+++ b/src/lib/libssl/ssl3.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl3.h,v 1.31 2014/12/14 15:30:50 jsing Exp $ */
+/* $OpenBSD: ssl3.h,v 1.32 2014/12/14 16:07:26 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -231,9 +231,11 @@ extern "C" {
 #define SSL3_RANDOM_SIZE                        32
 #define SSL3_SEQUENCE_SIZE                      8
 #define SSL3_SESSION_ID_SIZE                    32
-#define SSL3_RT_HEADER_LENGTH                   5
 #define SSL3_CIPHER_VALUE_SIZE                  2
 
+#define SSL3_RT_HEADER_LENGTH                   5
+#define SSL3_HM_HEADER_LENGTH                   4
+
 #ifndef SSL3_ALIGN_PAYLOAD
  /* Some will argue that this increases memory footprint, but it's
   * not actually true. Point is that malloc has to return at least