The usual idiom to cope with systems not defining socklen_t is to add a

#define socklen_t int
somewhere (or a typedef, whatever gives you an integer type of the size
your system expects as the 3rd argument of accept(2), really).

OpenSSL here is a bit more creative by using an union of an int and a size_t,
and extra code if sizeof(int) != sizeof(size_t) in order to recover the
proper size. With a comment mentioning that this has no chance to work on
a platform with a stack growing up and accept() returning an int, fortunately
this seems to work on HP-UX.

Switch to the light side of the force and declare and use socklen_t variables,
period. If your system does not define socklen_t, consider bringing it back
to your vendor for a refund.

ok matthew@ tedu@

4 files changed, 16 insertions, 98 deletions

diff --git a/src/lib/libcrypto/bio/b_sock.c b/src/lib/libcrypto/bio/b_sock.c
index e5f42398df..05eb362cc6 100644
--- a/src/lib/libcrypto/bio/b_sock.c
+++ b/src/lib/libcrypto/bio/b_sock.c
@@ -452,31 +452,7 @@ BIO_accept(int sock, char **addr)
         char *p, *tmp;
 
         struct {
-                /*
-                 * As for following union. Trouble is that there are platforms
-                 * that have socklen_t and there are platforms that don't, on
-                 * some platforms socklen_t is int and on some size_t. So what
-                 * one can do? One can cook #ifdef spaghetti, which is nothing
-                 * but masochistic. Or one can do union between int and size_t.
-                 * One naturally does it primarily for 64-bit platforms where
-                 * sizeof(int) != sizeof(size_t). But would it work? Note that
-                 * if size_t member is initialized to 0, then later int member
-                 * assignment naturally does the job on little-endian platforms
-                 * regardless accept's expectations! What about big-endians?
-                 * If accept expects int*, then it works, and if size_t*, then
-                 * length value would appear as unreasonably large. But this
-                 * won't prevent it from filling in the address structure. The
-                 * trouble of course would be if accept returns more data than
-                 * actual buffer can accomodate and overwrite stack... That's
-                 * where early OPENSSL_assert comes into picture. Besides, the
-                 * only 64-bit big-endian platform found so far that expects
-                 * size_t* is HP-UX, where stack grows towards higher address.
-                 * <appro>
-                 */
-                union {
-                        size_t s;
-                        int i;
-                } len;
+                socklen_t len;
                 union {
                         struct sockaddr sa;
                         struct sockaddr_in sa_in;
@@ -484,15 +460,9 @@ BIO_accept(int sock, char **addr)
                 } from;
         } sa;
 
-        sa.len.s = 0;
-        sa.len.i = sizeof(sa.from);
+        sa.len = sizeof(sa.from);
         memset(&sa.from, 0, sizeof(sa.from));
-        ret = accept(sock, &sa.from.sa, (void *)&sa.len);
-        if (sizeof(sa.len.i) != sizeof(sa.len.s) && sa.len.i == 0) {
-                OPENSSL_assert(sa.len.s <= sizeof(sa.from));
-                sa.len.i = (int)sa.len.s;
-                /* use sa.len.i from this point */
-        }
+        ret = accept(sock, &sa.from.sa, &sa.len);
         if (ret == -1) {
                 if (BIO_sock_should_retry(ret))
                         return -2;
@@ -511,7 +481,7 @@ BIO_accept(int sock, char **addr)
                 static union {
                         void *p;
                         int (*f)(const struct sockaddr *,
-                        size_t/*socklen_t*/, char *, size_t,
+                        socklen_t, char *, size_t,
                             char *, size_t, int);
                 } p_getnameinfo = {NULL};
                 /* 2nd argument to getnameinfo is specified to
@@ -527,7 +497,7 @@ BIO_accept(int sock, char **addr)
                 if (p_getnameinfo.p == (void *) - 1)
                         break;
 
-                if ((*p_getnameinfo.f)(&sa.from.sa, sa.len.i, h, sizeof(h),
+                if ((*p_getnameinfo.f)(&sa.from.sa, sa.len, h, sizeof(h),
                     s, sizeof(s), NI_NUMERICHOST|NI_NUMERICSERV))
                         break;
                 nl = strlen(h) + strlen(s) + 2;
diff --git a/src/lib/libcrypto/bio/bss_dgram.c b/src/lib/libcrypto/bio/bss_dgram.c
index a3b5bb6574..ab269aeba3 100644
--- a/src/lib/libcrypto/bio/bss_dgram.c
+++ b/src/lib/libcrypto/bio/bss_dgram.c
@@ -331,13 +331,7 @@ dgram_read(BIO *b, char *out, int outl)
         bio_dgram_data *data = (bio_dgram_data *)b->ptr;
 
         struct  {
-                /*
-                 * See commentary in b_sock.c. <appro>
-                 */
-                union   {
-                        size_t s;
-                        int i;
-                } len;
+                socklen_t len;
                 union   {
                         struct sockaddr sa;
                         struct sockaddr_in sa_in;
@@ -345,18 +339,13 @@ dgram_read(BIO *b, char *out, int outl)
                 } peer;
         } sa;
 
-        sa.len.s = 0;
-        sa.len.i = sizeof(sa.peer);
+        sa.len = sizeof(sa.peer);
 
         if (out != NULL) {
                 errno = 0;
                 memset(&sa.peer, 0x00, sizeof(sa.peer));
                 dgram_adjust_rcv_timeout(b);
-                ret = recvfrom(b->num, out, outl, 0, &sa.peer.sa,(void *)&sa.len);
-                if (sizeof(sa.len.i) != sizeof(sa.len.s) && sa.len.i == 0) {
-                        OPENSSL_assert(sa.len.s <= sizeof(sa.peer));
-                        sa.len.i = (int)sa.len.s;
-                }
+                ret = recvfrom(b->num, out, outl, 0, &sa.peer.sa, &sa.len);
 
                 if (! data->connected  && ret >= 0)
                         BIO_ctrl(b, BIO_CTRL_DGRAM_SET_PEER, 0, &sa.peer);
diff --git a/src/lib/libssl/src/crypto/bio/b_sock.c b/src/lib/libssl/src/crypto/bio/b_sock.c
index e5f42398df..05eb362cc6 100644
--- a/src/lib/libssl/src/crypto/bio/b_sock.c
+++ b/src/lib/libssl/src/crypto/bio/b_sock.c
@@ -452,31 +452,7 @@ BIO_accept(int sock, char **addr)
         char *p, *tmp;
 
         struct {
-                /*
-                 * As for following union. Trouble is that there are platforms
-                 * that have socklen_t and there are platforms that don't, on
-                 * some platforms socklen_t is int and on some size_t. So what
-                 * one can do? One can cook #ifdef spaghetti, which is nothing
-                 * but masochistic. Or one can do union between int and size_t.
-                 * One naturally does it primarily for 64-bit platforms where
-                 * sizeof(int) != sizeof(size_t). But would it work? Note that
-                 * if size_t member is initialized to 0, then later int member
-                 * assignment naturally does the job on little-endian platforms
-                 * regardless accept's expectations! What about big-endians?
-                 * If accept expects int*, then it works, and if size_t*, then
-                 * length value would appear as unreasonably large. But this
-                 * won't prevent it from filling in the address structure. The
-                 * trouble of course would be if accept returns more data than
-                 * actual buffer can accomodate and overwrite stack... That's
-                 * where early OPENSSL_assert comes into picture. Besides, the
-                 * only 64-bit big-endian platform found so far that expects
-                 * size_t* is HP-UX, where stack grows towards higher address.
-                 * <appro>
-                 */
-                union {
-                        size_t s;
-                        int i;
-                } len;
+                socklen_t len;
                 union {
                         struct sockaddr sa;
                         struct sockaddr_in sa_in;
@@ -484,15 +460,9 @@ BIO_accept(int sock, char **addr)
                 } from;
         } sa;
 
-        sa.len.s = 0;
-        sa.len.i = sizeof(sa.from);
+        sa.len = sizeof(sa.from);
         memset(&sa.from, 0, sizeof(sa.from));
-        ret = accept(sock, &sa.from.sa, (void *)&sa.len);
-        if (sizeof(sa.len.i) != sizeof(sa.len.s) && sa.len.i == 0) {
-                OPENSSL_assert(sa.len.s <= sizeof(sa.from));
-                sa.len.i = (int)sa.len.s;
-                /* use sa.len.i from this point */
-        }
+        ret = accept(sock, &sa.from.sa, &sa.len);
         if (ret == -1) {
                 if (BIO_sock_should_retry(ret))
                         return -2;
@@ -511,7 +481,7 @@ BIO_accept(int sock, char **addr)
                 static union {
                         void *p;
                         int (*f)(const struct sockaddr *,
-                        size_t/*socklen_t*/, char *, size_t,
+                        socklen_t, char *, size_t,
                             char *, size_t, int);
                 } p_getnameinfo = {NULL};
                 /* 2nd argument to getnameinfo is specified to
@@ -527,7 +497,7 @@ BIO_accept(int sock, char **addr)
                 if (p_getnameinfo.p == (void *) - 1)
                         break;
 
-                if ((*p_getnameinfo.f)(&sa.from.sa, sa.len.i, h, sizeof(h),
+                if ((*p_getnameinfo.f)(&sa.from.sa, sa.len, h, sizeof(h),
                     s, sizeof(s), NI_NUMERICHOST|NI_NUMERICSERV))
                         break;
                 nl = strlen(h) + strlen(s) + 2;
diff --git a/src/lib/libssl/src/crypto/bio/bss_dgram.c b/src/lib/libssl/src/crypto/bio/bss_dgram.c
index a3b5bb6574..ab269aeba3 100644
--- a/src/lib/libssl/src/crypto/bio/bss_dgram.c
+++ b/src/lib/libssl/src/crypto/bio/bss_dgram.c
@@ -331,13 +331,7 @@ dgram_read(BIO *b, char *out, int outl)
         bio_dgram_data *data = (bio_dgram_data *)b->ptr;
 
         struct  {
-                /*
-                 * See commentary in b_sock.c. <appro>
-                 */
-                union   {
-                        size_t s;
-                        int i;
-                } len;
+                socklen_t len;
                 union   {
                         struct sockaddr sa;
                         struct sockaddr_in sa_in;
@@ -345,18 +339,13 @@ dgram_read(BIO *b, char *out, int outl)
                 } peer;
         } sa;
 
-        sa.len.s = 0;
-        sa.len.i = sizeof(sa.peer);
+        sa.len = sizeof(sa.peer);
 
         if (out != NULL) {
                 errno = 0;
                 memset(&sa.peer, 0x00, sizeof(sa.peer));
                 dgram_adjust_rcv_timeout(b);
-                ret = recvfrom(b->num, out, outl, 0, &sa.peer.sa,(void *)&sa.len);
-                if (sizeof(sa.len.i) != sizeof(sa.len.s) && sa.len.i == 0) {
-                        OPENSSL_assert(sa.len.s <= sizeof(sa.peer));
-                        sa.len.i = (int)sa.len.s;
-                }
+                ret = recvfrom(b->num, out, outl, 0, &sa.peer.sa, &sa.len);
 
                 if (! data->connected  && ret >= 0)
                         BIO_ctrl(b, BIO_CTRL_DGRAM_SET_PEER, 0, &sa.peer);