4 files changed, 117 insertions, 76 deletions

diff --git a/src/lib/libssl/d1_both.c b/src/lib/libssl/d1_both.c
index 4ce4064cc9..2180c6d4da 100644
--- a/src/lib/libssl/d1_both.c
+++ b/src/lib/libssl/d1_both.c
@@ -153,7 +153,7 @@
 #endif
 
 static unsigned char bitmask_start_values[] = {0xff, 0xfe, 0xfc, 0xf8, 0xf0, 0xe0, 0xc0, 0x80};
-static unsigned char bitmask_end_values[]   = {0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f};
+static unsigned char bitmask_end_values[]   = {0xff, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f};
 
 /* XDTLS:  figure out the right values */
 static unsigned int g_probable_mtu[] = {1500 - 28, 512 - 28, 256 - 28};
@@ -464,20 +464,9 @@ again:
 
         memset(msg_hdr, 0x00, sizeof(struct hm_header_st));
 
-        s->d1->handshake_read_seq++;
-        /* we just read a handshake message from the other side:
-         * this means that we don't need to retransmit of the
-         * buffered messages.  
-         * XDTLS: may be able clear out this
-         * buffer a little sooner (i.e if an out-of-order
-         * handshake message/record is received at the record
-         * layer.  
-         * XDTLS: exception is that the server needs to
-         * know that change cipher spec and finished messages
-         * have been received by the client before clearing this
-         * buffer.  this can simply be done by waiting for the
-         * first data  segment, but is there a better way?  */
-        dtls1_clear_record_buffer(s);
+        /* Don't change sequence numbers while listening */
+        if (!s->d1->listen)
+                s->d1->handshake_read_seq++;
 
         s->init_msg = s->init_buf->data + DTLS1_HM_HEADER_LENGTH;
         return s->init_num;
@@ -813,9 +802,11 @@ dtls1_get_message_fragment(SSL *s, int st1, int stn, long max, int *ok)
 
         /* 
          * if this is a future (or stale) message it gets buffered
-         * (or dropped)--no further processing at this time 
+         * (or dropped)--no further processing at this time
+         * While listening, we accept seq 1 (ClientHello with cookie)
+         * although we're still expecting seq 0 (ClientHello)
          */
-        if ( msg_hdr.seq != s->d1->handshake_read_seq)
+        if (msg_hdr.seq != s->d1->handshake_read_seq && !(s->d1->listen && msg_hdr.seq == 1))
                 return dtls1_process_out_of_seq_message(s, &msg_hdr, ok);
 
         len = msg_hdr.msg_len;
@@ -1322,7 +1313,8 @@ unsigned char *
 dtls1_set_message_header(SSL *s, unsigned char *p, unsigned char mt,
                         unsigned long len, unsigned long frag_off, unsigned long frag_len)
         {
-        if ( frag_off == 0)
+        /* Don't change sequence numbers while listening */
+        if (frag_off == 0 && !s->d1->listen)
                 {
                 s->d1->handshake_write_seq = s->d1->next_handshake_write_seq;
                 s->d1->next_handshake_write_seq++;
diff --git a/src/lib/libssl/d1_enc.c b/src/lib/libssl/d1_enc.c
index 8fa57347a9..becbab91c2 100644
--- a/src/lib/libssl/d1_enc.c
+++ b/src/lib/libssl/d1_enc.c
@@ -231,11 +231,7 @@ int dtls1_enc(SSL *s, int send)
                 if (!send)
                         {
                         if (l == 0 || l%bs != 0)
-                                {
-                                SSLerr(SSL_F_DTLS1_ENC,SSL_R_BLOCK_CIPHER_PAD_IS_WRONG);
-                                ssl3_send_alert(s,SSL3_AL_FATAL,SSL_AD_DECRYPTION_FAILED);
-                                return 0;
-                                }
+                                return -1;
                         }
                 
                 EVP_Cipher(ds,rec->data,rec->input,l);
diff --git a/src/lib/libssl/d1_lib.c b/src/lib/libssl/d1_lib.c
index 96b220e87c..48e8b6ffbb 100644
--- a/src/lib/libssl/d1_lib.c
+++ b/src/lib/libssl/d1_lib.c
@@ -129,26 +129,33 @@ int dtls1_new(SSL *s)
         return(1);
         }
 
-void dtls1_free(SSL *s)
+static void dtls1_clear_queues(SSL *s)
         {
     pitem *item = NULL;
     hm_fragment *frag = NULL;
-
-        ssl3_free(s);
+        DTLS1_RECORD_DATA *rdata;
 
     while( (item = pqueue_pop(s->d1->unprocessed_rcds.q)) != NULL)
         {
+                rdata = (DTLS1_RECORD_DATA *) item->data;
+                if (rdata->rbuf.buf)
+                        {
+                        OPENSSL_free(rdata->rbuf.buf);
+                        }
         OPENSSL_free(item->data);
         pitem_free(item);
         }
-    pqueue_free(s->d1->unprocessed_rcds.q);
 
     while( (item = pqueue_pop(s->d1->processed_rcds.q)) != NULL)
         {
+                rdata = (DTLS1_RECORD_DATA *) item->data;
+                if (rdata->rbuf.buf)
+                        {
+                        OPENSSL_free(rdata->rbuf.buf);
+                        }
         OPENSSL_free(item->data);
         pitem_free(item);
         }
-    pqueue_free(s->d1->processed_rcds.q);
 
     while( (item = pqueue_pop(s->d1->buffered_messages)) != NULL)
         {
@@ -157,7 +164,6 @@ void dtls1_free(SSL *s)
         OPENSSL_free(frag);
         pitem_free(item);
         }
-    pqueue_free(s->d1->buffered_messages);
 
     while ( (item = pqueue_pop(s->d1->sent_messages)) != NULL)
         {
@@ -166,7 +172,6 @@ void dtls1_free(SSL *s)
         OPENSSL_free(frag);
         pitem_free(item);
         }
-        pqueue_free(s->d1->sent_messages);
 
         while ( (item = pqueue_pop(s->d1->buffered_app_data.q)) != NULL)
                 {
@@ -175,6 +180,18 @@ void dtls1_free(SSL *s)
                 OPENSSL_free(frag);
                 pitem_free(item);
                 }
+        }
+
+void dtls1_free(SSL *s)
+        {
+        ssl3_free(s);
+
+        dtls1_clear_queues(s);
+
+    pqueue_free(s->d1->unprocessed_rcds.q);
+    pqueue_free(s->d1->processed_rcds.q);
+    pqueue_free(s->d1->buffered_messages);
+        pqueue_free(s->d1->sent_messages);
         pqueue_free(s->d1->buffered_app_data.q);
 
         OPENSSL_free(s->d1);
@@ -182,6 +199,36 @@ void dtls1_free(SSL *s)
 
 void dtls1_clear(SSL *s)
         {
+    pqueue unprocessed_rcds;
+    pqueue processed_rcds;
+    pqueue buffered_messages;
+        pqueue sent_messages;
+        pqueue buffered_app_data;
+        
+        if (s->d1)
+                {
+                unprocessed_rcds = s->d1->unprocessed_rcds.q;
+                processed_rcds = s->d1->processed_rcds.q;
+                buffered_messages = s->d1->buffered_messages;
+                sent_messages = s->d1->sent_messages;
+                buffered_app_data = s->d1->buffered_app_data.q;
+
+                dtls1_clear_queues(s);
+
+                memset(s->d1, 0, sizeof(*(s->d1)));
+
+                if (s->server)
+                        {
+                        s->d1->cookie_len = sizeof(s->d1->cookie);
+                        }
+
+                s->d1->unprocessed_rcds.q = unprocessed_rcds;
+                s->d1->processed_rcds.q = processed_rcds;
+                s->d1->buffered_messages = buffered_messages;
+                s->d1->sent_messages = sent_messages;
+                s->d1->buffered_app_data.q = buffered_app_data;
+                }
+
         ssl3_clear(s);
         if (s->options & SSL_OP_CISCO_ANYCONNECT)
                 s->version=DTLS1_BAD_VER;
@@ -330,6 +377,8 @@ void dtls1_stop_timer(SSL *s)
         memset(&(s->d1->next_timeout), 0, sizeof(struct timeval));
         s->d1->timeout_duration = 1;
         BIO_ctrl(SSL_get_rbio(s), BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT, 0, &(s->d1->next_timeout));
+        /* Clear retransmission buffer */
+        dtls1_clear_record_buffer(s);
         }
 
 int dtls1_handle_timeout(SSL *s)
@@ -349,7 +398,7 @@ int dtls1_handle_timeout(SSL *s)
                 {
                 /* fail the connection, enough alerts have been sent */
                 SSLerr(SSL_F_DTLS1_HANDLE_TIMEOUT,SSL_R_READ_TIMEOUT_EXPIRED);
-                return 0;
+                return -1;
                 }
 
         state->timeout.read_timeouts++;
diff --git a/src/lib/libssl/test/cms-test.pl b/src/lib/libssl/test/cms-test.pl
index 9c50dff3e9..c938bcf00d 100644
--- a/src/lib/libssl/test/cms-test.pl
+++ b/src/lib/libssl/test/cms-test.pl
@@ -54,9 +54,13 @@
 # OpenSSL PKCS#7 and CMS implementations.
 
 my $ossl_path;
-my $redir = " 2>cms.err 1>cms.out";
+my $redir = " 2> cms.err > cms.out";
+# Make VMS work
+if ( $^O eq "VMS" && -f "OSSLX:openssl.exe" ) {
+    $ossl_path = "pipe mcr OSSLX:openssl";
+}
 # Make MSYS work
-if ( $^O eq "MSWin32" && -f "../apps/openssl.exe" ) {
+elsif ( $^O eq "MSWin32" && -f "../apps/openssl.exe" ) {
     $ossl_path = "cmd /c ..\\apps\\openssl";
 }
 elsif ( -f "../apps/openssl$ENV{EXE_EXT}" ) {
@@ -84,79 +88,79 @@ my @smime_pkcs7_tests = (
 
     [
         "signed content DER format, RSA key",
-        "-sign -in smcont.txt -outform DER -nodetach"
+        "-sign -in smcont.txt -outform \"DER\" -nodetach"
           . " -certfile $smdir/smroot.pem"
           . " -signer $smdir/smrsa1.pem -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
         "signed detached content DER format, RSA key",
-        "-sign -in smcont.txt -outform DER"
+        "-sign -in smcont.txt -outform \"DER\""
           . " -signer $smdir/smrsa1.pem -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt -content smcont.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt -content smcont.txt"
     ],
 
     [
         "signed content test streaming BER format, RSA",
-        "-sign -in smcont.txt -outform DER -nodetach"
+        "-sign -in smcont.txt -outform \"DER\" -nodetach"
           . " -stream -signer $smdir/smrsa1.pem -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
         "signed content DER format, DSA key",
-        "-sign -in smcont.txt -outform DER -nodetach"
+        "-sign -in smcont.txt -outform \"DER\" -nodetach"
           . " -signer $smdir/smdsa1.pem -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
         "signed detached content DER format, DSA key",
-        "-sign -in smcont.txt -outform DER"
+        "-sign -in smcont.txt -outform \"DER\""
           . " -signer $smdir/smdsa1.pem -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt -content smcont.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt -content smcont.txt"
     ],
 
     [
         "signed detached content DER format, add RSA signer",
-        "-resign -inform DER -in test.cms -outform DER"
+        "-resign -inform \"DER\" -in test.cms -outform \"DER\""
           . " -signer $smdir/smrsa1.pem -out test2.cms",
-        "-verify -in test2.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt -content smcont.txt"
+        "-verify -in test2.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt -content smcont.txt"
     ],
 
     [
         "signed content test streaming BER format, DSA key",
-        "-sign -in smcont.txt -outform DER -nodetach"
+        "-sign -in smcont.txt -outform \"DER\" -nodetach"
           . " -stream -signer $smdir/smdsa1.pem -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
         "signed content test streaming BER format, 2 DSA and 2 RSA keys",
-        "-sign -in smcont.txt -outform DER -nodetach"
+        "-sign -in smcont.txt -outform \"DER\" -nodetach"
           . " -signer $smdir/smrsa1.pem -signer $smdir/smrsa2.pem"
           . " -signer $smdir/smdsa1.pem -signer $smdir/smdsa2.pem"
           . " -stream -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
 "signed content test streaming BER format, 2 DSA and 2 RSA keys, no attributes",
-        "-sign -in smcont.txt -outform DER -noattr -nodetach"
+        "-sign -in smcont.txt -outform \"DER\" -noattr -nodetach"
           . " -signer $smdir/smrsa1.pem -signer $smdir/smrsa2.pem"
           . " -signer $smdir/smdsa1.pem -signer $smdir/smdsa2.pem"
           . " -stream -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
@@ -165,7 +169,7 @@ my @smime_pkcs7_tests = (
           . " -signer $smdir/smrsa1.pem -signer $smdir/smrsa2.pem"
           . " -signer $smdir/smdsa1.pem -signer $smdir/smdsa2.pem"
           . " -stream -out test.cms",
-        "-verify -in test.cms " . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms " . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
@@ -174,7 +178,7 @@ my @smime_pkcs7_tests = (
           . " -signer $smdir/smrsa1.pem -signer $smdir/smrsa2.pem"
           . " -signer $smdir/smdsa1.pem -signer $smdir/smdsa2.pem"
           . " -stream -out test.cms",
-        "-verify -in test.cms " . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms " . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
@@ -215,12 +219,12 @@ my @smime_cms_tests = (
 
     [
         "signed content test streaming BER format, 2 DSA and 2 RSA keys, keyid",
-        "-sign -in smcont.txt -outform DER -nodetach -keyid"
+        "-sign -in smcont.txt -outform \"DER\" -nodetach -keyid"
           . " -signer $smdir/smrsa1.pem -signer $smdir/smrsa2.pem"
           . " -signer $smdir/smdsa1.pem -signer $smdir/smdsa2.pem"
           . " -stream -out test.cms",
-        "-verify -in test.cms -inform DER "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+        "-verify -in test.cms -inform \"DER\" "
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
@@ -230,7 +234,7 @@ my @smime_cms_tests = (
           . " -signer $smdir/smdsa1.pem -signer $smdir/smdsa2.pem"
           . " -stream -out test.cms",
         "-verify -in test.cms -inform PEM "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
@@ -239,7 +243,7 @@ my @smime_cms_tests = (
           . " -receipt_request_to test\@openssl.org -receipt_request_all"
           . " -out test.cms",
         "-verify -in test.cms "
-          . " -CAfile $smdir/smroot.pem -out smtst.txt"
+          . " \"-CAfile\" $smdir/smroot.pem -out smtst.txt"
     ],
 
     [
@@ -248,7 +252,7 @@ my @smime_cms_tests = (
           . " -signer $smdir/smrsa2.pem"
           . " -out test2.cms",
         "-verify_receipt test2.cms -in test.cms"
-          . " -CAfile $smdir/smroot.pem"
+          . " \"-CAfile\" $smdir/smroot.pem"
     ],
 
     [
@@ -289,38 +293,38 @@ my @smime_cms_tests = (
 
     [
         "encrypted content test streaming PEM format, 128 bit RC2 key",
-        "-EncryptedData_encrypt -in smcont.txt -outform PEM"
+        "\"-EncryptedData_encrypt\" -in smcont.txt -outform PEM"
           . " -rc2 -secretkey 000102030405060708090A0B0C0D0E0F"
           . " -stream -out test.cms",
-        "-EncryptedData_decrypt -in test.cms -inform PEM "
+        "\"-EncryptedData_decrypt\" -in test.cms -inform PEM "
           . " -secretkey 000102030405060708090A0B0C0D0E0F -out smtst.txt"
     ],
 
     [
         "encrypted content test streaming PEM format, 40 bit RC2 key",
-        "-EncryptedData_encrypt -in smcont.txt -outform PEM"
+        "\"-EncryptedData_encrypt\" -in smcont.txt -outform PEM"
           . " -rc2 -secretkey 0001020304"
           . " -stream -out test.cms",
-        "-EncryptedData_decrypt -in test.cms -inform PEM "
+        "\"-EncryptedData_decrypt\" -in test.cms -inform PEM "
           . " -secretkey 0001020304 -out smtst.txt"
     ],
 
     [
         "encrypted content test streaming PEM format, triple DES key",
-        "-EncryptedData_encrypt -in smcont.txt -outform PEM"
+        "\"-EncryptedData_encrypt\" -in smcont.txt -outform PEM"
           . " -des3 -secretkey 000102030405060708090A0B0C0D0E0F1011121314151617"
           . " -stream -out test.cms",
-        "-EncryptedData_decrypt -in test.cms -inform PEM "
+        "\"-EncryptedData_decrypt\" -in test.cms -inform PEM "
           . " -secretkey 000102030405060708090A0B0C0D0E0F1011121314151617"
           . " -out smtst.txt"
     ],
 
     [
         "encrypted content test streaming PEM format, 128 bit AES key",
-        "-EncryptedData_encrypt -in smcont.txt -outform PEM"
+        "\"-EncryptedData_encrypt\" -in smcont.txt -outform PEM"
           . " -aes128 -secretkey 000102030405060708090A0B0C0D0E0F"
           . " -stream -out test.cms",
-        "-EncryptedData_decrypt -in test.cms -inform PEM "
+        "\"-EncryptedData_decrypt\" -in test.cms -inform PEM "
           . " -secretkey 000102030405060708090A0B0C0D0E0F -out smtst.txt"
     ],