Merge branch 'busybox' into merge

1 files changed, 140 insertions, 86 deletions

diff --git a/networking/wget.c b/networking/wget.c
index 460b4b833..b9d840328 100644
--- a/networking/wget.c
+++ b/networking/wget.c
@@ -16,12 +16,15 @@
 //config:         wget is a utility for non-interactive download of files from HTTP
 //config:         and FTP servers.
 //config:
+//config:config FEATURE_WGET_LONG_OPTIONS
+//config:       bool "Enable long options"
+//config:       default y
+//config:       depends on WGET && LONG_OPTS
+//config:
 //config:config FEATURE_WGET_STATUSBAR
-//config:       bool "Enable a nifty process meter (+2k)"
+//config:       bool "Enable progress bar (+2k)"
 //config:       default y
 //config:       depends on WGET
-//config:       help
-//config:         Enable the transfer progress bar for wget transfers.
 //config:
 //config:config FEATURE_WGET_AUTHENTICATION
 //config:       bool "Enable HTTP authentication"
@@ -30,13 +33,6 @@
 //config:       help
 //config:         Support authenticated HTTP transfers.
 //config:
-//config:config FEATURE_WGET_LONG_OPTIONS
-//config:       bool "Enable long options"
-//config:       default y
-//config:       depends on WGET && LONG_OPTS
-//config:       help
-//config:         Support long options for the wget applet.
-//config:
 //config:config FEATURE_WGET_TIMEOUT
 //config:       bool "Enable timeout option -T SEC"
 //config:       default y
@@ -51,18 +47,59 @@
 //config:         FEATURE_WGET_LONG_OPTIONS is also enabled, the --timeout option
 //config:         will work in addition to -T.
 //config:
+//config:config FEATURE_WGET_HTTPS
+//config:       bool "Support HTTPS using internal TLS code"
+//config:       default y
+//config:       depends on WGET
+//config:       select TLS
+//config:       help
+//config:         wget will use internal TLS code to connect to https:// URLs.
+//config:         Note:
+//config:         On NOMMU machines, ssl_helper applet should be available
+//config:         in the $PATH for this to work. Make sure to select that applet.
+//config:
+//config:         Note: currently, TLS code only makes TLS I/O work, it
+//config:         does *not* check that the peer is who it claims to be, etc.
+//config:         IOW: it uses peer-supplied public keys to establish encryption
+//config:         and signing keys, then encrypts and signs outgoing data and
+//config:         decrypts incoming data.
+//config:         It does not check signature hashes on the incoming data:
+//config:         this means that attackers manipulating TCP packets can
+//config:         send altered data and we unknowingly receive garbage.
+//config:         (This check might be relatively easy to add).
+//config:         It does not check public key's certificate:
+//config:         this means that the peer may be an attacker impersonating
+//config:         the server we think we are talking to.
+//config:
+//config:         If you think this is unacceptable, consider this. As more and more
+//config:         servers switch to HTTPS-only operation, without such "crippled"
+//config:         TLS code it is *impossible* to simply download a kernel source
+//config:         from kernel.org. Which can in real world translate into
+//config:         "my small automatic tooling to build cross-compilers from sources
+//config:         no longer works, I need to additionally keep a local copy
+//config:         of ~4 megabyte source tarball of a SSL library and ~2 megabyte
+//config:         source of wget, need to compile and built both before I can
+//config:         download anything. All this despite the fact that the build
+//config:         is done in a QEMU sandbox on a machine with absolutely nothing
+//config:         worth stealing, so I don't care if someone would go to a lot
+//config:         of trouble to intercept my HTTPS download to send me an altered
+//config:         kernel tarball".
+//config:
+//config:         If you still think this is unacceptable, send patches.
+//config:
+//config:         If you still think this is unacceptable, do not want to send
+//config:         patches, but do want to waste bandwidth expaining how wrong
+//config:         it is, you will be ignored.
+//config:
 //config:config FEATURE_WGET_OPENSSL
 //config:       bool "Try to connect to HTTPS using openssl"
 //config:       default y
 //config:       depends on WGET
 //config:       help
-//config:         Choose how wget establishes SSL connection for https:// URLs.
-//config:
-//config:         Busybox itself contains no SSL code. wget will spawn
-//config:         a helper program to talk over HTTPS.
+//config:         Try to use openssl to handle HTTPS.
 //config:
 //config:         OpenSSL has a simple SSL client for debug purposes.
-//config:         If you select "openssl" helper, wget will effectively run:
+//config:         If you select this option, wget will effectively run:
 //config:         "openssl s_client -quiet -connect hostname:443
 //config:         -servername hostname 2>/dev/null" and pipe its data
 //config:         through it. -servername is not used if hostname is numeric.
@@ -75,24 +112,9 @@
 //config:         openssl is also a big binary, often dynamically linked
 //config:         against ~15 libraries.
 //config:
-//config:config FEATURE_WGET_SSL_HELPER
-//config:       bool "Try to connect to HTTPS using ssl_helper"
-//config:       default y
-//config:       depends on WGET
-//config:       help
-//config:         Choose how wget establishes SSL connection for https:// URLs.
-//config:
-//config:         Busybox itself contains no SSL code. wget will spawn
-//config:         a helper program to talk over HTTPS.
-//config:
-//config:         ssl_helper is a tool which can be built statically
-//config:         from busybox sources against a small embedded SSL library.
-//config:         Please see networking/ssl_helper/README.
-//config:         It does not require double host resolution and emits
-//config:         error messages to stderr.
-//config:
-//config:         Precompiled static binary may be available at
-//config:         http://busybox.net/downloads/binaries/
+//config:         If openssl can't be executed, internal TLS code will be used
+//config:         (if you enabled it); if openssl can be executed but fails later,
+//config:         wget can't detect this, and download will fail.
 
 //applet:IF_WGET(APPLET(wget, BB_DIR_USR_BIN, BB_SUID_DROP))
 
@@ -105,20 +127,21 @@
 /* Since we ignore these opts, we don't show them in --help */
 /* //usage:    "        [--no-check-certificate] [--no-cache] [--passive-ftp] [-t TRIES]" */
 /* //usage:    "        [-nv] [-nc] [-nH] [-np]" */
-//usage:       "        [-U|--user-agent AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
+//usage:       "        [-S|--server-response] [-U|--user-agent AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
 //usage:        )
 //usage:        IF_NOT_FEATURE_WGET_LONG_OPTIONS(
-//usage:       "[-cq] [-O FILE] [-Y on/off] [-P DIR] [-U AGENT]"
+//usage:       "[-cq] [-O FILE] [-Y on/off] [-P DIR] [-S] [-U AGENT]"
 //usage:                        IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
 //usage:        )
 //usage:#define wget_full_usage "\n\n"
 //usage:       "Retrieve files via HTTP or FTP\n"
 //usage:        IF_FEATURE_WGET_LONG_OPTIONS(
-//usage:     "\n        --spider        Spider mode - only check file existence"
+//usage:     "\n        --spider        Only check URL existence: $? is 0 if exists"
 //usage:        )
 //usage:     "\n        -c              Continue retrieval of aborted transfer"
 //usage:     "\n        -q              Quiet"
 //usage:     "\n        -P DIR          Save to DIR (default .)"
+//usage:     "\n        -S              Show server response"
 //usage:        IF_FEATURE_WGET_TIMEOUT(
 //usage:     "\n        -T SEC          Network read timeout is SEC seconds"
 //usage:        )
@@ -141,6 +164,8 @@
 #endif
 
 
+#define SSL_SUPPORTED (ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_HTTPS)
+
 struct host_info {
         char *allocated;
         const char *path;
@@ -151,7 +176,7 @@ struct host_info {
 };
 static const char P_FTP[] ALIGN1 = "ftp";
 static const char P_HTTP[] ALIGN1 = "http";
-#if ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER
+#if SSL_SUPPORTED
 static const char P_HTTPS[] ALIGN1 = "https";
 #endif
 
@@ -232,16 +257,17 @@ struct globals {
 enum {
         WGET_OPT_CONTINUE   = (1 << 0),
         WGET_OPT_QUIET      = (1 << 1),
-        WGET_OPT_OUTNAME    = (1 << 2),
-        WGET_OPT_PREFIX     = (1 << 3),
-        WGET_OPT_PROXY      = (1 << 4),
-        WGET_OPT_USER_AGENT = (1 << 5),
-        WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 6),
-        WGET_OPT_RETRIES    = (1 << 7),
-        WGET_OPT_nsomething = (1 << 8),
-        WGET_OPT_HEADER     = (1 << 9) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
-        WGET_OPT_POST_DATA  = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
-        WGET_OPT_SPIDER     = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
+        WGET_OPT_SERVER_RESPONSE = (1 << 2),
+        WGET_OPT_OUTNAME    = (1 << 3),
+        WGET_OPT_PREFIX     = (1 << 4),
+        WGET_OPT_PROXY      = (1 << 5),
+        WGET_OPT_USER_AGENT = (1 << 6),
+        WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 7),
+        WGET_OPT_RETRIES    = (1 << 8),
+        WGET_OPT_nsomething = (1 << 9),
+        WGET_OPT_HEADER     = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
+        WGET_OPT_POST_DATA  = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
+        WGET_OPT_SPIDER     = (1 << 12) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
 };
 
 enum {
@@ -395,7 +421,7 @@ static FILE *open_socket(len_and_sockaddr *lsa)
 }
 
 /* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */
-static char fgets_and_trim(FILE *fp)
+static char fgets_and_trim(FILE *fp, const char *fmt)
 {
         char c;
         char *buf_ptr;
@@ -413,6 +439,9 @@ static char fgets_and_trim(FILE *fp)
 
         log_io("< %s", G.wget_buf);
 
+        if (fmt && (option_mask32 & WGET_OPT_SERVER_RESPONSE))
+                fprintf(stderr, fmt, G.wget_buf);
+
         return c;
 }
 
@@ -423,6 +452,9 @@ static int ftpcmd(const char *s1, const char *s2, FILE *fp)
                 if (!s2)
                         s2 = "";
                 fprintf(fp, "%s%s\r\n", s1, s2);
+                /* With --server-response, wget also shows its ftp commands */
+                if (option_mask32 & WGET_OPT_SERVER_RESPONSE)
+                        fprintf(stderr, "--> %s%s\n\n", s1, s2);
                 fflush(fp);
                 log_io("> %s%s", s1, s2);
 #if ENABLE_PLATFORM_MINGW32
@@ -431,7 +463,7 @@ static int ftpcmd(const char *s1, const char *s2, FILE *fp)
         }
 
         do {
-                fgets_and_trim(fp);
+                fgets_and_trim(fp, "%s\n");
         } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
 #if ENABLE_PLATFORM_MINGW32
         fseek(fp, 0L, SEEK_CUR);
@@ -458,7 +490,7 @@ static void parse_url(const char *src_url, struct host_info *h)
                 if (strcmp(url, P_FTP) == 0) {
                         h->port = bb_lookup_port(P_FTP, "tcp", 21);
                 } else
-#if ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER
+#if SSL_SUPPORTED
                 if (strcmp(url, P_HTTPS) == 0) {
                         h->port = bb_lookup_port(P_HTTPS, "tcp", 443);
                         h->protocol = P_HTTPS;
@@ -480,7 +512,7 @@ static void parse_url(const char *src_url, struct host_info *h)
 
         // FYI:
         // "Real" wget 'http://busybox.net?var=a/b' sends this request:
-        //   'GET /?var=a/b HTTP 1.0'
+        //   'GET /?var=a/b HTTP/1.0'
         //   and saves 'index.html?var=a%2Fb' (we save 'b')
         // wget 'http://busybox.net?login=john@doe':
         //   request: 'GET /?login=john@doe HTTP/1.0'
@@ -531,7 +563,7 @@ static char *gethdr(FILE *fp)
         int c;
 
         /* retrieve header line */
-        c = fgets_and_trim(fp);
+        c = fgets_and_trim(fp, "  %s\n");
 
         /* end of the headers? */
         if (G.wget_buf[0] == '\0')
@@ -665,7 +697,7 @@ static int spawn_https_helper_openssl(const char *host, unsigned port)
         char *servername;
         int sp[2];
         int pid;
-        IF_FEATURE_WGET_SSL_HELPER(volatile int child_failed = 0;)
+        IF_FEATURE_WGET_HTTPS(volatile int child_failed = 0;)
 
         if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
                 /* Kernel can have AF_UNIX support disabled */
@@ -710,7 +742,7 @@ static int spawn_https_helper_openssl(const char *host, unsigned port)
 
                 BB_EXECVP(argv[0], argv);
                 xmove_fd(3, 2);
-# if ENABLE_FEATURE_WGET_SSL_HELPER
+# if ENABLE_FEATURE_WGET_HTTPS
                 child_failed = 1;
                 xfunc_die();
 # else
@@ -723,7 +755,7 @@ static int spawn_https_helper_openssl(const char *host, unsigned port)
         free(servername);
         free(allocated);
         close(sp[1]);
-# if ENABLE_FEATURE_WGET_SSL_HELPER
+# if ENABLE_FEATURE_WGET_HTTPS
         if (child_failed) {
                 close(sp[0]);
                 return -1;
@@ -733,38 +765,51 @@ static int spawn_https_helper_openssl(const char *host, unsigned port)
 }
 #endif
 
-/* See networking/ssl_helper/README how to build one */
-#if ENABLE_FEATURE_WGET_SSL_HELPER
-static void spawn_https_helper_small(int network_fd)
+#if ENABLE_FEATURE_WGET_HTTPS
+static void spawn_ssl_client(const char *host, int network_fd)
 {
         int sp[2];
         int pid;
+        char *servername, *p;
+
+        servername = xstrdup(host);
+        p = strrchr(servername, ':');
+        if (p) *p = '\0';
 
         if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
                 /* Kernel can have AF_UNIX support disabled */
                 bb_perror_msg_and_die("socketpair");
 
+        fflush_all();
         pid = BB_MMU ? xfork() : xvfork();
         if (pid == 0) {
                 /* Child */
-                char *argv[3];
-
                 close(sp[0]);
                 xmove_fd(sp[1], 0);
                 xdup2(0, 1);
-                xmove_fd(network_fd, 3);
-                /*
-                 * A simple ssl/tls helper
-                 */
-                argv[0] = (char*)"ssl_helper";
-                argv[1] = (char*)"-d3";
-                argv[2] = NULL;
-                BB_EXECVP(argv[0], argv);
-                bb_perror_msg_and_die("can't execute '%s'", argv[0]);
+                if (BB_MMU) {
+                        tls_state_t *tls = new_tls_state();
+                        tls->ifd = tls->ofd = network_fd;
+                        tls_handshake(tls, servername);
+                        tls_run_copy_loop(tls);
+                        exit(0);
+                } else {
+                        char *argv[5];
+                        xmove_fd(network_fd, 3);
+                        argv[0] = (char*)"ssl_client";
+                        argv[1] = (char*)"-s3";
+                        //TODO: if (!is_ip_address(servername))...
+                        argv[2] = (char*)"-n";
+                        argv[3] = servername;
+                        argv[4] = NULL;
+                        BB_EXECVP(argv[0], argv);
+                        bb_perror_msg_and_die("can't execute '%s'", argv[0]);
+                }
                 /* notreached */
         }
 
         /* Parent */
+        free(servername);
         close(sp[1]);
         xmove_fd(sp[0], network_fd);
 }
@@ -886,9 +931,9 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
                 if (!G.chunked)
                         break;
 
-                fgets_and_trim(dfp); /* Eat empty line */
+                fgets_and_trim(dfp, NULL); /* Eat empty line */
  get_clen:
-                fgets_and_trim(dfp);
+                fgets_and_trim(dfp, NULL);
                 G.content_len = STRTOOFF(G.wget_buf, NULL, 16);
                 /* FIXME: error check? */
                 if (G.content_len == 0)
@@ -1013,16 +1058,16 @@ static void download_one_url(const char *url)
 
                 /* Open socket to http(s) server */
 #if ENABLE_FEATURE_WGET_OPENSSL
-                /* openssl (and maybe ssl_helper) support is configured */
+                /* openssl (and maybe internal TLS) support is configured */
                 if (target.protocol == P_HTTPS) {
                         /* openssl-based helper
                          * Inconvenient API since we can't give it an open fd
                          */
                         int fd = spawn_https_helper_openssl(server.host, server.port);
-# if ENABLE_FEATURE_WGET_SSL_HELPER
-                        if (fd < 0) { /* no openssl? try ssl_helper */
+# if ENABLE_FEATURE_WGET_HTTPS
+                        if (fd < 0) { /* no openssl? try internal */
                                 sfp = open_socket(lsa);
-                                spawn_https_helper_small(fileno(sfp));
+                                spawn_ssl_client(server.host, fileno(sfp));
                                 goto socket_opened;
                         }
 # else
@@ -1035,11 +1080,11 @@ static void download_one_url(const char *url)
                 }
                 sfp = open_socket(lsa);
  socket_opened:
-#elif ENABLE_FEATURE_WGET_SSL_HELPER
-                /* Only ssl_helper support is configured */
+#elif ENABLE_FEATURE_WGET_HTTPS
+                /* Only internal TLS support is configured */
                 sfp = open_socket(lsa);
                 if (target.protocol == P_HTTPS)
-                        spawn_https_helper_small(fileno(sfp));
+                        spawn_ssl_client(server.host, fileno(sfp));
 #else
                 /* ssl (https) support is not configured */
                 sfp = open_socket(lsa);
@@ -1099,18 +1144,26 @@ static void download_one_url(const char *url)
                 }
 
                 fflush(sfp);
-                /* If we use SSL helper, keeping our end of the socket open for writing
-                 * makes our end (i.e. the same fd!) readable (EAGAIN instead of EOF)
-                 * even after child closes its copy of the fd.
-                 * This helps:
-                 */
-                shutdown(fileno(sfp), SHUT_WR);
+
+/* Tried doing this unconditionally.
+ * Cloudflare and nginx/1.11.5 are shocked to see SHUT_WR on non-HTTPS.
+ */
+#if SSL_SUPPORTED
+                if (target.protocol == P_HTTPS) {
+                        /* If we use SSL helper, keeping our end of the socket open for writing
+                         * makes our end (i.e. the same fd!) readable (EAGAIN instead of EOF)
+                         * even after child closes its copy of the fd.
+                         * This helps:
+                         */
+                        shutdown(fileno(sfp), SHUT_WR);
+                }
+#endif
 
                 /*
                  * Retrieve HTTP response line and check for "200" status code.
                  */
  read_response:
-                fgets_and_trim(sfp);
+                fgets_and_trim(sfp, "  %s\n");
 
                 str = G.wget_buf;
                 str = skip_non_whitespace(str);
@@ -1295,6 +1348,7 @@ int wget_main(int argc UNUSED_PARAM, char **argv)
                 /* name, has_arg, val */
                 "continue\0"         No_argument       "c"
                 "quiet\0"            No_argument       "q"
+                "server-response\0"  No_argument       "S"
                 "output-document\0"  Required_argument "O"
                 "directory-prefix\0" Required_argument "P"
                 "proxy\0"            Required_argument "Y"
@@ -1337,7 +1391,7 @@ IF_DESKTOP(	"no-parent\0"        No_argument       "\xf0")
 #endif
         opt_complementary = "-1" /* at least one URL */
                 IF_FEATURE_WGET_LONG_OPTIONS(":\xff::"); /* --header is a list */
-        getopt32(argv, "cqO:P:Y:U:T:+"
+        getopt32(argv, "cqSO:P:Y:U:T:+"
                 /*ignored:*/ "t:"
                 /*ignored:*/ "n::"
                 /* wget has exactly four -n<letter> opts, all of which we can ignore: