diff options
Diffstat (limited to 'networking/wget.c')
-rw-r--r-- | networking/wget.c | 630 |
1 files changed, 342 insertions, 288 deletions
diff --git a/networking/wget.c b/networking/wget.c index 16594c9bf..bbc161be8 100644 --- a/networking/wget.c +++ b/networking/wget.c | |||
@@ -10,9 +10,12 @@ | |||
10 | */ | 10 | */ |
11 | #include "libbb.h" | 11 | #include "libbb.h" |
12 | 12 | ||
13 | //#define log_io(...) bb_error_msg(__VA_ARGS__) | ||
14 | #define log_io(...) ((void)0) | ||
15 | |||
16 | |||
13 | struct host_info { | 17 | struct host_info { |
14 | // May be used if we ever will want to free() all xstrdup()s... | 18 | char *allocated; |
15 | /* char *allocated; */ | ||
16 | const char *path; | 19 | const char *path; |
17 | const char *user; | 20 | const char *user; |
18 | char *host; | 21 | char *host; |
@@ -30,17 +33,31 @@ struct globals { | |||
30 | const char *curfile; /* Name of current file being transferred */ | 33 | const char *curfile; /* Name of current file being transferred */ |
31 | bb_progress_t pmt; | 34 | bb_progress_t pmt; |
32 | #endif | 35 | #endif |
36 | char *dir_prefix; | ||
37 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
38 | char *post_data; | ||
39 | char *extra_headers; | ||
40 | #endif | ||
41 | char *fname_out; /* where to direct output (-O) */ | ||
42 | const char *proxy_flag; /* Use proxies if env vars are set */ | ||
43 | const char *user_agent; /* "User-Agent" header field */ | ||
33 | #if ENABLE_FEATURE_WGET_TIMEOUT | 44 | #if ENABLE_FEATURE_WGET_TIMEOUT |
34 | unsigned timeout_seconds; | 45 | unsigned timeout_seconds; |
35 | #endif | 46 | #endif |
47 | int output_fd; | ||
48 | int o_flags; | ||
36 | smallint chunked; /* chunked transfer encoding */ | 49 | smallint chunked; /* chunked transfer encoding */ |
37 | smallint got_clen; /* got content-length: from server */ | 50 | smallint got_clen; /* got content-length: from server */ |
51 | /* Local downloads do benefit from big buffer. | ||
52 | * With 512 byte buffer, it was measured to be | ||
53 | * an order of magnitude slower than with big one. | ||
54 | */ | ||
55 | uint64_t just_to_align_next_member; | ||
56 | char wget_buf[CONFIG_FEATURE_COPYBUF_KB*1024]; | ||
38 | } FIX_ALIASING; | 57 | } FIX_ALIASING; |
39 | #define G (*(struct globals*)&bb_common_bufsiz1) | 58 | #define G (*ptr_to_globals) |
40 | struct BUG_G_too_big { | ||
41 | char BUG_G_too_big[sizeof(G) <= COMMON_BUFSIZE ? 1 : -1]; | ||
42 | }; | ||
43 | #define INIT_G() do { \ | 59 | #define INIT_G() do { \ |
60 | SET_PTR_TO_GLOBALS(xzalloc(sizeof(G))); \ | ||
44 | IF_FEATURE_WGET_TIMEOUT(G.timeout_seconds = 900;) \ | 61 | IF_FEATURE_WGET_TIMEOUT(G.timeout_seconds = 900;) \ |
45 | } while (0) | 62 | } while (0) |
46 | 63 | ||
@@ -73,12 +90,16 @@ static void progress_meter(int flag) | |||
73 | return; | 90 | return; |
74 | 91 | ||
75 | if (flag == PROGRESS_START) | 92 | if (flag == PROGRESS_START) |
76 | bb_progress_init(&G.pmt); | 93 | bb_progress_init(&G.pmt, G.curfile); |
77 | 94 | ||
78 | bb_progress_update(&G.pmt, G.curfile, G.beg_range, G.transferred, | 95 | bb_progress_update(&G.pmt, |
79 | G.chunked ? 0 : G.beg_range + G.transferred + G.content_len); | 96 | G.beg_range, |
97 | G.transferred, | ||
98 | (G.chunked || !G.got_clen) ? 0 : G.beg_range + G.transferred + G.content_len | ||
99 | ); | ||
80 | 100 | ||
81 | if (flag == PROGRESS_END) { | 101 | if (flag == PROGRESS_END) { |
102 | bb_progress_free(&G.pmt); | ||
82 | bb_putchar_stderr('\n'); | 103 | bb_putchar_stderr('\n'); |
83 | G.transferred = 0; | 104 | G.transferred = 0; |
84 | } | 105 | } |
@@ -124,48 +145,15 @@ static void strip_ipv6_scope_id(char *host) | |||
124 | overlapping_strcpy(scope, cp); | 145 | overlapping_strcpy(scope, cp); |
125 | } | 146 | } |
126 | 147 | ||
127 | /* Read NMEMB bytes into PTR from STREAM. Returns the number of bytes read, | ||
128 | * and a short count if an eof or non-interrupt error is encountered. */ | ||
129 | static size_t safe_fread(void *ptr, size_t nmemb, FILE *stream) | ||
130 | { | ||
131 | size_t ret; | ||
132 | char *p = (char*)ptr; | ||
133 | |||
134 | do { | ||
135 | clearerr(stream); | ||
136 | errno = 0; | ||
137 | ret = fread(p, 1, nmemb, stream); | ||
138 | p += ret; | ||
139 | nmemb -= ret; | ||
140 | } while (nmemb && ferror(stream) && errno == EINTR); | ||
141 | |||
142 | return p - (char*)ptr; | ||
143 | } | ||
144 | |||
145 | /* Read a line or SIZE-1 bytes into S, whichever is less, from STREAM. | ||
146 | * Returns S, or NULL if an eof or non-interrupt error is encountered. */ | ||
147 | static char *safe_fgets(char *s, int size, FILE *stream) | ||
148 | { | ||
149 | char *ret; | ||
150 | |||
151 | do { | ||
152 | clearerr(stream); | ||
153 | errno = 0; | ||
154 | ret = fgets(s, size, stream); | ||
155 | } while (ret == NULL && ferror(stream) && errno == EINTR); | ||
156 | |||
157 | return ret; | ||
158 | } | ||
159 | |||
160 | #if ENABLE_FEATURE_WGET_AUTHENTICATION | 148 | #if ENABLE_FEATURE_WGET_AUTHENTICATION |
161 | /* Base64-encode character string. buf is assumed to be char buf[512]. */ | 149 | /* Base64-encode character string. */ |
162 | static char *base64enc_512(char buf[512], const char *str) | 150 | static char *base64enc(const char *str) |
163 | { | 151 | { |
164 | unsigned len = strlen(str); | 152 | unsigned len = strlen(str); |
165 | if (len > 512/4*3 - 10) /* paranoia */ | 153 | if (len > sizeof(G.wget_buf)/4*3 - 10) /* paranoia */ |
166 | len = 512/4*3 - 10; | 154 | len = sizeof(G.wget_buf)/4*3 - 10; |
167 | bb_uuencode(buf, str, len, bb_uuenc_tbl_base64); | 155 | bb_uuencode(G.wget_buf, str, len, bb_uuenc_tbl_base64); |
168 | return buf; | 156 | return G.wget_buf; |
169 | } | 157 | } |
170 | #endif | 158 | #endif |
171 | 159 | ||
@@ -186,43 +174,58 @@ static FILE *open_socket(len_and_sockaddr *lsa) | |||
186 | /* hopefully it understands what ESPIPE means... */ | 174 | /* hopefully it understands what ESPIPE means... */ |
187 | fp = fdopen(xconnect_stream(lsa), "r+"); | 175 | fp = fdopen(xconnect_stream(lsa), "r+"); |
188 | if (fp == NULL) | 176 | if (fp == NULL) |
189 | bb_perror_msg_and_die("fdopen"); | 177 | bb_perror_msg_and_die(bb_msg_memory_exhausted); |
190 | 178 | ||
191 | return fp; | 179 | return fp; |
192 | } | 180 | } |
193 | 181 | ||
194 | static int ftpcmd(const char *s1, const char *s2, FILE *fp, char *buf) | 182 | /* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */ |
183 | static char fgets_and_trim(FILE *fp) | ||
184 | { | ||
185 | char c; | ||
186 | char *buf_ptr; | ||
187 | |||
188 | if (fgets(G.wget_buf, sizeof(G.wget_buf) - 1, fp) == NULL) | ||
189 | bb_perror_msg_and_die("error getting response"); | ||
190 | |||
191 | buf_ptr = strchrnul(G.wget_buf, '\n'); | ||
192 | c = *buf_ptr; | ||
193 | *buf_ptr = '\0'; | ||
194 | buf_ptr = strchrnul(G.wget_buf, '\r'); | ||
195 | *buf_ptr = '\0'; | ||
196 | |||
197 | log_io("< %s", G.wget_buf); | ||
198 | |||
199 | return c; | ||
200 | } | ||
201 | |||
202 | static int ftpcmd(const char *s1, const char *s2, FILE *fp) | ||
195 | { | 203 | { |
196 | int result; | 204 | int result; |
197 | if (s1) { | 205 | if (s1) { |
198 | if (!s2) s2 = ""; | 206 | if (!s2) |
207 | s2 = ""; | ||
199 | fprintf(fp, "%s%s\r\n", s1, s2); | 208 | fprintf(fp, "%s%s\r\n", s1, s2); |
200 | fflush(fp); | 209 | fflush(fp); |
210 | log_io("> %s%s", s1, s2); | ||
201 | } | 211 | } |
202 | 212 | ||
203 | do { | 213 | do { |
204 | char *buf_ptr; | 214 | fgets_and_trim(fp); |
205 | 215 | } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' '); | |
206 | if (fgets(buf, 510, fp) == NULL) { | ||
207 | bb_perror_msg_and_die("error getting response"); | ||
208 | } | ||
209 | buf_ptr = strstr(buf, "\r\n"); | ||
210 | if (buf_ptr) { | ||
211 | *buf_ptr = '\0'; | ||
212 | } | ||
213 | } while (!isdigit(buf[0]) || buf[3] != ' '); | ||
214 | 216 | ||
215 | buf[3] = '\0'; | 217 | G.wget_buf[3] = '\0'; |
216 | result = xatoi_positive(buf); | 218 | result = xatoi_positive(G.wget_buf); |
217 | buf[3] = ' '; | 219 | G.wget_buf[3] = ' '; |
218 | return result; | 220 | return result; |
219 | } | 221 | } |
220 | 222 | ||
221 | static void parse_url(char *src_url, struct host_info *h) | 223 | static void parse_url(const char *src_url, struct host_info *h) |
222 | { | 224 | { |
223 | char *url, *p, *sp; | 225 | char *url, *p, *sp; |
224 | 226 | ||
225 | /* h->allocated = */ url = xstrdup(src_url); | 227 | free(h->allocated); |
228 | h->allocated = url = xstrdup(src_url); | ||
226 | 229 | ||
227 | if (strncmp(url, "http://", 7) == 0) { | 230 | if (strncmp(url, "http://", 7) == 0) { |
228 | h->port = bb_lookup_port("http", "tcp", 80); | 231 | h->port = bb_lookup_port("http", "tcp", 80); |
@@ -278,7 +281,7 @@ static void parse_url(char *src_url, struct host_info *h) | |||
278 | sp = h->host; | 281 | sp = h->host; |
279 | } | 282 | } |
280 | 283 | ||
281 | static char *gethdr(char *buf, size_t bufsiz, FILE *fp /*, int *istrunc*/) | 284 | static char *gethdr(FILE *fp) |
282 | { | 285 | { |
283 | char *s, *hdrval; | 286 | char *s, *hdrval; |
284 | int c; | 287 | int c; |
@@ -286,43 +289,32 @@ static char *gethdr(char *buf, size_t bufsiz, FILE *fp /*, int *istrunc*/) | |||
286 | /* *istrunc = 0; */ | 289 | /* *istrunc = 0; */ |
287 | 290 | ||
288 | /* retrieve header line */ | 291 | /* retrieve header line */ |
289 | if (fgets(buf, bufsiz, fp) == NULL) | 292 | c = fgets_and_trim(fp); |
290 | return NULL; | ||
291 | 293 | ||
292 | /* see if we are at the end of the headers */ | 294 | /* end of the headers? */ |
293 | for (s = buf; *s == '\r'; ++s) | 295 | if (G.wget_buf[0] == '\0') |
294 | continue; | ||
295 | if (*s == '\n') | ||
296 | return NULL; | 296 | return NULL; |
297 | 297 | ||
298 | /* convert the header name to lower case */ | 298 | /* convert the header name to lower case */ |
299 | for (s = buf; isalnum(*s) || *s == '-' || *s == '.'; ++s) { | 299 | for (s = G.wget_buf; isalnum(*s) || *s == '-' || *s == '.'; ++s) { |
300 | /* tolower for "A-Z", no-op for "0-9a-z-." */ | 300 | /* tolower for "A-Z", no-op for "0-9a-z-." */ |
301 | *s = (*s | 0x20); | 301 | *s |= 0x20; |
302 | } | 302 | } |
303 | 303 | ||
304 | /* verify we are at the end of the header name */ | 304 | /* verify we are at the end of the header name */ |
305 | if (*s != ':') | 305 | if (*s != ':') |
306 | bb_error_msg_and_die("bad header line: %s", sanitize_string(buf)); | 306 | bb_error_msg_and_die("bad header line: %s", sanitize_string(G.wget_buf)); |
307 | 307 | ||
308 | /* locate the start of the header value */ | 308 | /* locate the start of the header value */ |
309 | *s++ = '\0'; | 309 | *s++ = '\0'; |
310 | hdrval = skip_whitespace(s); | 310 | hdrval = skip_whitespace(s); |
311 | 311 | ||
312 | /* locate the end of header */ | 312 | if (c != '\n') { |
313 | while (*s && *s != '\r' && *s != '\n') | 313 | /* Rats! The buffer isn't big enough to hold the entire header value */ |
314 | ++s; | 314 | while (c = getc(fp), c != EOF && c != '\n') |
315 | 315 | continue; | |
316 | /* end of header found */ | ||
317 | if (*s) { | ||
318 | *s = '\0'; | ||
319 | return hdrval; | ||
320 | } | 316 | } |
321 | 317 | ||
322 | /* Rats! The buffer isn't big enough to hold the entire header value */ | ||
323 | while (c = getc(fp), c != EOF && c != '\n') | ||
324 | continue; | ||
325 | /* *istrunc = 1; */ | ||
326 | return hdrval; | 318 | return hdrval; |
327 | } | 319 | } |
328 | 320 | ||
@@ -366,7 +358,6 @@ static char *URL_escape(const char *str) | |||
366 | 358 | ||
367 | static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa) | 359 | static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa) |
368 | { | 360 | { |
369 | char buf[512]; | ||
370 | FILE *sfp; | 361 | FILE *sfp; |
371 | char *str; | 362 | char *str; |
372 | int port; | 363 | int port; |
@@ -375,8 +366,8 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_ | |||
375 | target->user = xstrdup("anonymous:busybox@"); | 366 | target->user = xstrdup("anonymous:busybox@"); |
376 | 367 | ||
377 | sfp = open_socket(lsa); | 368 | sfp = open_socket(lsa); |
378 | if (ftpcmd(NULL, NULL, sfp, buf) != 220) | 369 | if (ftpcmd(NULL, NULL, sfp) != 220) |
379 | bb_error_msg_and_die("%s", sanitize_string(buf+4)); | 370 | bb_error_msg_and_die("%s", sanitize_string(G.wget_buf + 4)); |
380 | 371 | ||
381 | /* | 372 | /* |
382 | * Splitting username:password pair, | 373 | * Splitting username:password pair, |
@@ -385,24 +376,24 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_ | |||
385 | str = strchr(target->user, ':'); | 376 | str = strchr(target->user, ':'); |
386 | if (str) | 377 | if (str) |
387 | *str++ = '\0'; | 378 | *str++ = '\0'; |
388 | switch (ftpcmd("USER ", target->user, sfp, buf)) { | 379 | switch (ftpcmd("USER ", target->user, sfp)) { |
389 | case 230: | 380 | case 230: |
390 | break; | 381 | break; |
391 | case 331: | 382 | case 331: |
392 | if (ftpcmd("PASS ", str, sfp, buf) == 230) | 383 | if (ftpcmd("PASS ", str, sfp) == 230) |
393 | break; | 384 | break; |
394 | /* fall through (failed login) */ | 385 | /* fall through (failed login) */ |
395 | default: | 386 | default: |
396 | bb_error_msg_and_die("ftp login: %s", sanitize_string(buf+4)); | 387 | bb_error_msg_and_die("ftp login: %s", sanitize_string(G.wget_buf + 4)); |
397 | } | 388 | } |
398 | 389 | ||
399 | ftpcmd("TYPE I", NULL, sfp, buf); | 390 | ftpcmd("TYPE I", NULL, sfp); |
400 | 391 | ||
401 | /* | 392 | /* |
402 | * Querying file size | 393 | * Querying file size |
403 | */ | 394 | */ |
404 | if (ftpcmd("SIZE ", target->path, sfp, buf) == 213) { | 395 | if (ftpcmd("SIZE ", target->path, sfp) == 213) { |
405 | G.content_len = BB_STRTOOFF(buf+4, NULL, 10); | 396 | G.content_len = BB_STRTOOFF(G.wget_buf + 4, NULL, 10); |
406 | if (G.content_len < 0 || errno) { | 397 | if (G.content_len < 0 || errno) { |
407 | bb_error_msg_and_die("SIZE value is garbage"); | 398 | bb_error_msg_and_die("SIZE value is garbage"); |
408 | } | 399 | } |
@@ -412,20 +403,20 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_ | |||
412 | /* | 403 | /* |
413 | * Entering passive mode | 404 | * Entering passive mode |
414 | */ | 405 | */ |
415 | if (ftpcmd("PASV", NULL, sfp, buf) != 227) { | 406 | if (ftpcmd("PASV", NULL, sfp) != 227) { |
416 | pasv_error: | 407 | pasv_error: |
417 | bb_error_msg_and_die("bad response to %s: %s", "PASV", sanitize_string(buf)); | 408 | bb_error_msg_and_die("bad response to %s: %s", "PASV", sanitize_string(G.wget_buf)); |
418 | } | 409 | } |
419 | // Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage] | 410 | // Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage] |
420 | // Server's IP is N1.N2.N3.N4 (we ignore it) | 411 | // Server's IP is N1.N2.N3.N4 (we ignore it) |
421 | // Server's port for data connection is P1*256+P2 | 412 | // Server's port for data connection is P1*256+P2 |
422 | str = strrchr(buf, ')'); | 413 | str = strrchr(G.wget_buf, ')'); |
423 | if (str) str[0] = '\0'; | 414 | if (str) str[0] = '\0'; |
424 | str = strrchr(buf, ','); | 415 | str = strrchr(G.wget_buf, ','); |
425 | if (!str) goto pasv_error; | 416 | if (!str) goto pasv_error; |
426 | port = xatou_range(str+1, 0, 255); | 417 | port = xatou_range(str+1, 0, 255); |
427 | *str = '\0'; | 418 | *str = '\0'; |
428 | str = strrchr(buf, ','); | 419 | str = strrchr(G.wget_buf, ','); |
429 | if (!str) goto pasv_error; | 420 | if (!str) goto pasv_error; |
430 | port += xatou_range(str+1, 0, 255) * 256; | 421 | port += xatou_range(str+1, 0, 255) * 256; |
431 | set_nport(lsa, htons(port)); | 422 | set_nport(lsa, htons(port)); |
@@ -433,20 +424,19 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_ | |||
433 | *dfpp = open_socket(lsa); | 424 | *dfpp = open_socket(lsa); |
434 | 425 | ||
435 | if (G.beg_range) { | 426 | if (G.beg_range) { |
436 | sprintf(buf, "REST %"OFF_FMT"u", G.beg_range); | 427 | sprintf(G.wget_buf, "REST %"OFF_FMT"u", G.beg_range); |
437 | if (ftpcmd(buf, NULL, sfp, buf) == 350) | 428 | if (ftpcmd(G.wget_buf, NULL, sfp) == 350) |
438 | G.content_len -= G.beg_range; | 429 | G.content_len -= G.beg_range; |
439 | } | 430 | } |
440 | 431 | ||
441 | if (ftpcmd("RETR ", target->path, sfp, buf) > 150) | 432 | if (ftpcmd("RETR ", target->path, sfp) > 150) |
442 | bb_error_msg_and_die("bad response to %s: %s", "RETR", sanitize_string(buf)); | 433 | bb_error_msg_and_die("bad response to %s: %s", "RETR", sanitize_string(G.wget_buf)); |
443 | 434 | ||
444 | return sfp; | 435 | return sfp; |
445 | } | 436 | } |
446 | 437 | ||
447 | static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd) | 438 | static void NOINLINE retrieve_file_data(FILE *dfp) |
448 | { | 439 | { |
449 | char buf[512]; | ||
450 | #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT | 440 | #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT |
451 | # if ENABLE_FEATURE_WGET_TIMEOUT | 441 | # if ENABLE_FEATURE_WGET_TIMEOUT |
452 | unsigned second_cnt; | 442 | unsigned second_cnt; |
@@ -455,7 +445,6 @@ static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd) | |||
455 | 445 | ||
456 | polldata.fd = fileno(dfp); | 446 | polldata.fd = fileno(dfp); |
457 | polldata.events = POLLIN | POLLPRI; | 447 | polldata.events = POLLIN | POLLPRI; |
458 | ndelay_on(polldata.fd); | ||
459 | #endif | 448 | #endif |
460 | progress_meter(PROGRESS_START); | 449 | progress_meter(PROGRESS_START); |
461 | 450 | ||
@@ -464,18 +453,30 @@ static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd) | |||
464 | 453 | ||
465 | /* Loops only if chunked */ | 454 | /* Loops only if chunked */ |
466 | while (1) { | 455 | while (1) { |
456 | |||
457 | #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT | ||
458 | /* Must use nonblocking I/O, otherwise fread will loop | ||
459 | * and *block* until it reads full buffer, | ||
460 | * which messes up progress bar and/or timeout logic. | ||
461 | * Because of nonblocking I/O, we need to dance | ||
462 | * very carefully around EAGAIN. See explanation at | ||
463 | * clearerr() call. | ||
464 | */ | ||
465 | ndelay_on(polldata.fd); | ||
466 | #endif | ||
467 | while (1) { | 467 | while (1) { |
468 | int n; | 468 | int n; |
469 | unsigned rdsz; | 469 | unsigned rdsz; |
470 | 470 | ||
471 | rdsz = sizeof(buf); | 471 | rdsz = sizeof(G.wget_buf); |
472 | if (G.got_clen) { | 472 | if (G.got_clen) { |
473 | if (G.content_len < (off_t)sizeof(buf)) { | 473 | if (G.content_len < (off_t)sizeof(G.wget_buf)) { |
474 | if ((int)G.content_len <= 0) | 474 | if ((int)G.content_len <= 0) |
475 | break; | 475 | break; |
476 | rdsz = (unsigned)G.content_len; | 476 | rdsz = (unsigned)G.content_len; |
477 | } | 477 | } |
478 | } | 478 | } |
479 | |||
479 | #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT | 480 | #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT |
480 | # if ENABLE_FEATURE_WGET_TIMEOUT | 481 | # if ENABLE_FEATURE_WGET_TIMEOUT |
481 | second_cnt = G.timeout_seconds; | 482 | second_cnt = G.timeout_seconds; |
@@ -486,150 +487,107 @@ static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd) | |||
486 | # if ENABLE_FEATURE_WGET_TIMEOUT | 487 | # if ENABLE_FEATURE_WGET_TIMEOUT |
487 | if (second_cnt != 0 && --second_cnt == 0) { | 488 | if (second_cnt != 0 && --second_cnt == 0) { |
488 | progress_meter(PROGRESS_END); | 489 | progress_meter(PROGRESS_END); |
489 | bb_perror_msg_and_die("download timed out"); | 490 | bb_error_msg_and_die("download timed out"); |
490 | } | 491 | } |
491 | # endif | 492 | # endif |
492 | /* Needed for "stalled" indicator */ | 493 | /* Needed for "stalled" indicator */ |
493 | progress_meter(PROGRESS_BUMP); | 494 | progress_meter(PROGRESS_BUMP); |
494 | } | 495 | } |
496 | |||
497 | /* fread internally uses read loop, which in our case | ||
498 | * is usually exited when we get EAGAIN. | ||
499 | * In this case, libc sets error marker on the stream. | ||
500 | * Need to clear it before next fread to avoid possible | ||
501 | * rare false positive ferror below. Rare because usually | ||
502 | * fread gets more than zero bytes, and we don't fall | ||
503 | * into if (n <= 0) ... | ||
504 | */ | ||
505 | clearerr(dfp); | ||
506 | errno = 0; | ||
495 | #endif | 507 | #endif |
496 | n = safe_fread(buf, rdsz, dfp); | 508 | n = fread(G.wget_buf, 1, rdsz, dfp); |
509 | /* man fread: | ||
510 | * If error occurs, or EOF is reached, the return value | ||
511 | * is a short item count (or zero). | ||
512 | * fread does not distinguish between EOF and error. | ||
513 | */ | ||
497 | if (n <= 0) { | 514 | if (n <= 0) { |
498 | if (ferror(dfp)) { | 515 | #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT |
499 | /* perror will not work: ferror doesn't set errno */ | 516 | if (errno == EAGAIN) /* poll lied, there is no data? */ |
500 | bb_error_msg_and_die(bb_msg_read_error); | 517 | continue; /* yes */ |
501 | } | 518 | #endif |
502 | break; | 519 | if (ferror(dfp)) |
520 | bb_perror_msg_and_die(bb_msg_read_error); | ||
521 | break; /* EOF, not error */ | ||
503 | } | 522 | } |
504 | xwrite(output_fd, buf, n); | 523 | |
524 | xwrite(G.output_fd, G.wget_buf, n); | ||
525 | |||
505 | #if ENABLE_FEATURE_WGET_STATUSBAR | 526 | #if ENABLE_FEATURE_WGET_STATUSBAR |
506 | G.transferred += n; | 527 | G.transferred += n; |
507 | progress_meter(PROGRESS_BUMP); | 528 | progress_meter(PROGRESS_BUMP); |
508 | #endif | 529 | #endif |
509 | if (G.got_clen) | 530 | if (G.got_clen) { |
510 | G.content_len -= n; | 531 | G.content_len -= n; |
532 | if (G.content_len == 0) | ||
533 | break; | ||
534 | } | ||
511 | } | 535 | } |
512 | 536 | #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT | |
537 | clearerr(dfp); | ||
538 | ndelay_off(polldata.fd); /* else fgets can get very unhappy */ | ||
539 | #endif | ||
513 | if (!G.chunked) | 540 | if (!G.chunked) |
514 | break; | 541 | break; |
515 | 542 | ||
516 | safe_fgets(buf, sizeof(buf), dfp); /* This is a newline */ | 543 | fgets_and_trim(dfp); /* Eat empty line */ |
517 | get_clen: | 544 | get_clen: |
518 | safe_fgets(buf, sizeof(buf), dfp); | 545 | fgets_and_trim(dfp); |
519 | G.content_len = STRTOOFF(buf, NULL, 16); | 546 | G.content_len = STRTOOFF(G.wget_buf, NULL, 16); |
520 | /* FIXME: error check? */ | 547 | /* FIXME: error check? */ |
521 | if (G.content_len == 0) | 548 | if (G.content_len == 0) |
522 | break; /* all done! */ | 549 | break; /* all done! */ |
523 | G.got_clen = 1; | 550 | G.got_clen = 1; |
524 | } | 551 | } |
525 | 552 | ||
553 | /* Draw full bar and free its resources */ | ||
554 | G.chunked = 0; /* makes it show 100% even for chunked download */ | ||
555 | G.got_clen = 1; /* makes it show 100% even for download of (formerly) unknown size */ | ||
526 | progress_meter(PROGRESS_END); | 556 | progress_meter(PROGRESS_END); |
527 | } | 557 | } |
528 | 558 | ||
529 | int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; | 559 | static void download_one_url(const char *url) |
530 | int wget_main(int argc UNUSED_PARAM, char **argv) | ||
531 | { | 560 | { |
532 | char buf[512]; | 561 | bool use_proxy; /* Use proxies if env vars are set */ |
533 | struct host_info server, target; | ||
534 | len_and_sockaddr *lsa; | ||
535 | unsigned opt; | ||
536 | int redir_limit; | 562 | int redir_limit; |
537 | char *proxy = NULL; | 563 | len_and_sockaddr *lsa; |
538 | char *dir_prefix = NULL; | ||
539 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
540 | char *post_data; | ||
541 | char *extra_headers = NULL; | ||
542 | llist_t *headers_llist = NULL; | ||
543 | #endif | ||
544 | FILE *sfp; /* socket to web/ftp server */ | 564 | FILE *sfp; /* socket to web/ftp server */ |
545 | FILE *dfp; /* socket to ftp server (data) */ | 565 | FILE *dfp; /* socket to ftp server (data) */ |
546 | char *fname_out; /* where to direct output (-O) */ | 566 | char *proxy = NULL; |
547 | int output_fd = -1; | 567 | char *fname_out_alloc; |
548 | bool use_proxy; /* Use proxies if env vars are set */ | 568 | struct host_info server; |
549 | const char *proxy_flag = "on"; /* Use proxies if env vars are set */ | 569 | struct host_info target; |
550 | const char *user_agent = "Wget";/* "User-Agent" header field */ | ||
551 | |||
552 | static const char keywords[] ALIGN1 = | ||
553 | "content-length\0""transfer-encoding\0""chunked\0""location\0"; | ||
554 | enum { | ||
555 | KEY_content_length = 1, KEY_transfer_encoding, KEY_chunked, KEY_location | ||
556 | }; | ||
557 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
558 | static const char wget_longopts[] ALIGN1 = | ||
559 | /* name, has_arg, val */ | ||
560 | "continue\0" No_argument "c" | ||
561 | "spider\0" No_argument "s" | ||
562 | "quiet\0" No_argument "q" | ||
563 | "output-document\0" Required_argument "O" | ||
564 | "directory-prefix\0" Required_argument "P" | ||
565 | "proxy\0" Required_argument "Y" | ||
566 | "user-agent\0" Required_argument "U" | ||
567 | #if ENABLE_FEATURE_WGET_TIMEOUT | ||
568 | "timeout\0" Required_argument "T" | ||
569 | #endif | ||
570 | /* Ignored: */ | ||
571 | // "tries\0" Required_argument "t" | ||
572 | /* Ignored (we always use PASV): */ | ||
573 | "passive-ftp\0" No_argument "\xff" | ||
574 | "header\0" Required_argument "\xfe" | ||
575 | "post-data\0" Required_argument "\xfd" | ||
576 | /* Ignored (we don't do ssl) */ | ||
577 | "no-check-certificate\0" No_argument "\xfc" | ||
578 | ; | ||
579 | #endif | ||
580 | |||
581 | INIT_G(); | ||
582 | IF_WIN32_NET(init_winsock();) | ||
583 | |||
584 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
585 | applet_long_options = wget_longopts; | ||
586 | #endif | ||
587 | /* server.allocated = target.allocated = NULL; */ | ||
588 | opt_complementary = "-1" IF_FEATURE_WGET_TIMEOUT(":T+") IF_FEATURE_WGET_LONG_OPTIONS(":\xfe::"); | ||
589 | opt = getopt32(argv, "csqO:P:Y:U:T:" /*ignored:*/ "t:", | ||
590 | &fname_out, &dir_prefix, | ||
591 | &proxy_flag, &user_agent, | ||
592 | IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL), | ||
593 | NULL /* -t RETRIES */ | ||
594 | IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist) | ||
595 | IF_FEATURE_WGET_LONG_OPTIONS(, &post_data) | ||
596 | ); | ||
597 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
598 | if (headers_llist) { | ||
599 | int size = 1; | ||
600 | char *cp; | ||
601 | llist_t *ll = headers_llist; | ||
602 | while (ll) { | ||
603 | size += strlen(ll->data) + 2; | ||
604 | ll = ll->link; | ||
605 | } | ||
606 | extra_headers = cp = xmalloc(size); | ||
607 | while (headers_llist) { | ||
608 | cp += sprintf(cp, "%s\r\n", (char*)llist_pop(&headers_llist)); | ||
609 | } | ||
610 | } | ||
611 | #endif | ||
612 | |||
613 | /* TODO: compat issue: should handle "wget URL1 URL2..." */ | ||
614 | 570 | ||
571 | server.allocated = NULL; | ||
572 | target.allocated = NULL; | ||
573 | server.user = NULL; | ||
615 | target.user = NULL; | 574 | target.user = NULL; |
616 | parse_url(argv[optind], &target); | 575 | |
576 | parse_url(url, &target); | ||
617 | 577 | ||
618 | /* Use the proxy if necessary */ | 578 | /* Use the proxy if necessary */ |
619 | use_proxy = (strcmp(proxy_flag, "off") != 0); | 579 | use_proxy = (strcmp(G.proxy_flag, "off") != 0); |
620 | if (use_proxy) { | 580 | if (use_proxy) { |
621 | proxy = getenv(target.is_ftp ? "ftp_proxy" : "http_proxy"); | 581 | proxy = getenv(target.is_ftp ? "ftp_proxy" : "http_proxy"); |
622 | if (proxy && proxy[0]) { | 582 | use_proxy = (proxy && proxy[0]); |
623 | server.user = NULL; | 583 | if (use_proxy) |
624 | parse_url(proxy, &server); | 584 | parse_url(proxy, &server); |
625 | } else { | ||
626 | use_proxy = 0; | ||
627 | } | ||
628 | } | 585 | } |
629 | if (!use_proxy) { | 586 | if (!use_proxy) { |
630 | server.port = target.port; | 587 | server.port = target.port; |
631 | if (ENABLE_FEATURE_IPV6) { | 588 | if (ENABLE_FEATURE_IPV6) { |
632 | server.host = xstrdup(target.host); | 589 | //free(server.allocated); - can't be non-NULL |
590 | server.host = server.allocated = xstrdup(target.host); | ||
633 | } else { | 591 | } else { |
634 | server.host = target.host; | 592 | server.host = target.host; |
635 | } | 593 | } |
@@ -638,50 +596,44 @@ int wget_main(int argc UNUSED_PARAM, char **argv) | |||
638 | if (ENABLE_FEATURE_IPV6) | 596 | if (ENABLE_FEATURE_IPV6) |
639 | strip_ipv6_scope_id(target.host); | 597 | strip_ipv6_scope_id(target.host); |
640 | 598 | ||
641 | /* Guess an output filename, if there was no -O FILE */ | 599 | /* If there was no -O FILE, guess output filename */ |
642 | if (!(opt & WGET_OPT_OUTNAME)) { | 600 | fname_out_alloc = NULL; |
643 | fname_out = bb_get_last_path_component_nostrip(target.path); | 601 | if (!(option_mask32 & WGET_OPT_OUTNAME)) { |
602 | G.fname_out = bb_get_last_path_component_nostrip(target.path); | ||
644 | /* handle "wget http://kernel.org//" */ | 603 | /* handle "wget http://kernel.org//" */ |
645 | if (fname_out[0] == '/' || !fname_out[0]) | 604 | if (G.fname_out[0] == '/' || !G.fname_out[0]) |
646 | fname_out = (char*)"index.html"; | 605 | G.fname_out = (char*)"index.html"; |
647 | /* -P DIR is considered only if there was no -O FILE */ | 606 | /* -P DIR is considered only if there was no -O FILE */ |
648 | if (dir_prefix) | 607 | if (G.dir_prefix) |
649 | fname_out = concat_path_file(dir_prefix, fname_out); | 608 | G.fname_out = fname_out_alloc = concat_path_file(G.dir_prefix, G.fname_out); |
650 | } else { | ||
651 | if (LONE_DASH(fname_out)) { | ||
652 | /* -O - */ | ||
653 | output_fd = 1; | ||
654 | opt &= ~WGET_OPT_CONTINUE; | ||
655 | } | ||
656 | } | 609 | } |
657 | #if ENABLE_FEATURE_WGET_STATUSBAR | 610 | #if ENABLE_FEATURE_WGET_STATUSBAR |
658 | G.curfile = bb_get_last_path_component_nostrip(fname_out); | 611 | G.curfile = bb_get_last_path_component_nostrip(G.fname_out); |
659 | #endif | 612 | #endif |
660 | 613 | ||
661 | /* Impossible? | ||
662 | if ((opt & WGET_OPT_CONTINUE) && !fname_out) | ||
663 | bb_error_msg_and_die("can't specify continue (-c) without a filename (-O)"); | ||
664 | */ | ||
665 | |||
666 | /* Determine where to start transfer */ | 614 | /* Determine where to start transfer */ |
667 | if (opt & WGET_OPT_CONTINUE) { | 615 | G.beg_range = 0; |
668 | output_fd = open(fname_out, O_WRONLY); | 616 | if (option_mask32 & WGET_OPT_CONTINUE) { |
669 | if (output_fd >= 0) { | 617 | G.output_fd = open(G.fname_out, O_WRONLY); |
670 | G.beg_range = xlseek(output_fd, 0, SEEK_END); | 618 | if (G.output_fd >= 0) { |
619 | G.beg_range = xlseek(G.output_fd, 0, SEEK_END); | ||
671 | } | 620 | } |
672 | /* File doesn't exist. We do not create file here yet. | 621 | /* File doesn't exist. We do not create file here yet. |
673 | * We are not sure it exists on remove side */ | 622 | * We are not sure it exists on remote side */ |
674 | } | 623 | } |
675 | 624 | ||
676 | redir_limit = 5; | 625 | redir_limit = 5; |
677 | resolve_lsa: | 626 | resolve_lsa: |
678 | lsa = xhost2sockaddr(server.host, server.port); | 627 | lsa = xhost2sockaddr(server.host, server.port); |
679 | if (!(opt & WGET_OPT_QUIET)) { | 628 | if (!(option_mask32 & WGET_OPT_QUIET)) { |
680 | char *s = xmalloc_sockaddr2dotted(&lsa->u.sa); | 629 | char *s = xmalloc_sockaddr2dotted(&lsa->u.sa); |
681 | fprintf(stderr, "Connecting to %s (%s)\n", server.host, s); | 630 | fprintf(stderr, "Connecting to %s (%s)\n", server.host, s); |
682 | free(s); | 631 | free(s); |
683 | } | 632 | } |
684 | establish_session: | 633 | establish_session: |
634 | /*G.content_len = 0; - redundant, got_clen = 0 is enough */ | ||
635 | G.got_clen = 0; | ||
636 | G.chunked = 0; | ||
685 | if (use_proxy || !target.is_ftp) { | 637 | if (use_proxy || !target.is_ftp) { |
686 | /* | 638 | /* |
687 | * HTTP session | 639 | * HTTP session |
@@ -689,6 +641,7 @@ int wget_main(int argc UNUSED_PARAM, char **argv) | |||
689 | char *str; | 641 | char *str; |
690 | int status; | 642 | int status; |
691 | 643 | ||
644 | |||
692 | /* Open socket to http server */ | 645 | /* Open socket to http server */ |
693 | sfp = open_socket(lsa); | 646 | sfp = open_socket(lsa); |
694 | 647 | ||
@@ -698,44 +651,52 @@ int wget_main(int argc UNUSED_PARAM, char **argv) | |||
698 | target.is_ftp ? "f" : "ht", target.host, | 651 | target.is_ftp ? "f" : "ht", target.host, |
699 | target.path); | 652 | target.path); |
700 | } else { | 653 | } else { |
701 | if (opt & WGET_OPT_POST_DATA) | 654 | if (option_mask32 & WGET_OPT_POST_DATA) |
702 | fprintf(sfp, "POST /%s HTTP/1.1\r\n", target.path); | 655 | fprintf(sfp, "POST /%s HTTP/1.1\r\n", target.path); |
703 | else | 656 | else |
704 | fprintf(sfp, "GET /%s HTTP/1.1\r\n", target.path); | 657 | fprintf(sfp, "GET /%s HTTP/1.1\r\n", target.path); |
705 | } | 658 | } |
706 | 659 | ||
707 | fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n", | 660 | fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n", |
708 | target.host, user_agent); | 661 | target.host, G.user_agent); |
662 | |||
663 | /* Ask server to close the connection as soon as we are done | ||
664 | * (IOW: we do not intend to send more requests) | ||
665 | */ | ||
666 | fprintf(sfp, "Connection: close\r\n"); | ||
709 | 667 | ||
710 | #if ENABLE_FEATURE_WGET_AUTHENTICATION | 668 | #if ENABLE_FEATURE_WGET_AUTHENTICATION |
711 | if (target.user) { | 669 | if (target.user) { |
712 | fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6, | 670 | fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6, |
713 | base64enc_512(buf, target.user)); | 671 | base64enc(target.user)); |
714 | } | 672 | } |
715 | if (use_proxy && server.user) { | 673 | if (use_proxy && server.user) { |
716 | fprintf(sfp, "Proxy-Authorization: Basic %s\r\n", | 674 | fprintf(sfp, "Proxy-Authorization: Basic %s\r\n", |
717 | base64enc_512(buf, server.user)); | 675 | base64enc(server.user)); |
718 | } | 676 | } |
719 | #endif | 677 | #endif |
720 | 678 | ||
721 | if (G.beg_range) | 679 | if (G.beg_range) |
722 | fprintf(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range); | 680 | fprintf(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range); |
681 | |||
723 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | 682 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
724 | if (extra_headers) | 683 | if (G.extra_headers) |
725 | fputs(extra_headers, sfp); | 684 | fputs(G.extra_headers, sfp); |
726 | 685 | ||
727 | if (opt & WGET_OPT_POST_DATA) { | 686 | if (option_mask32 & WGET_OPT_POST_DATA) { |
728 | char *estr = URL_escape(post_data); | 687 | char *estr = URL_escape(G.post_data); |
729 | fprintf(sfp, "Content-Type: application/x-www-form-urlencoded\r\n"); | 688 | fprintf(sfp, |
730 | fprintf(sfp, "Content-Length: %u\r\n" "\r\n" "%s", | 689 | "Content-Type: application/x-www-form-urlencoded\r\n" |
731 | (int) strlen(estr), estr); | 690 | "Content-Length: %u\r\n" |
732 | /*fprintf(sfp, "Connection: Keep-Alive\r\n\r\n");*/ | 691 | "\r\n" |
733 | /*fprintf(sfp, "%s\r\n", estr);*/ | 692 | "%s", |
693 | (int) strlen(estr), estr | ||
694 | ); | ||
734 | free(estr); | 695 | free(estr); |
735 | } else | 696 | } else |
736 | #endif | 697 | #endif |
737 | { /* If "Connection:" is needed, document why */ | 698 | { |
738 | fprintf(sfp, /* "Connection: close\r\n" */ "\r\n"); | 699 | fprintf(sfp, "\r\n"); |
739 | } | 700 | } |
740 | 701 | ||
741 | fflush(sfp); | 702 | fflush(sfp); |
@@ -744,10 +705,9 @@ int wget_main(int argc UNUSED_PARAM, char **argv) | |||
744 | * Retrieve HTTP response line and check for "200" status code. | 705 | * Retrieve HTTP response line and check for "200" status code. |
745 | */ | 706 | */ |
746 | read_response: | 707 | read_response: |
747 | if (fgets(buf, sizeof(buf), sfp) == NULL) | 708 | fgets_and_trim(sfp); |
748 | bb_error_msg_and_die("no response from server"); | ||
749 | 709 | ||
750 | str = buf; | 710 | str = G.wget_buf; |
751 | str = skip_non_whitespace(str); | 711 | str = skip_non_whitespace(str); |
752 | str = skip_whitespace(str); | 712 | str = skip_whitespace(str); |
753 | // FIXME: no error check | 713 | // FIXME: no error check |
@@ -756,7 +716,7 @@ int wget_main(int argc UNUSED_PARAM, char **argv) | |||
756 | switch (status) { | 716 | switch (status) { |
757 | case 0: | 717 | case 0: |
758 | case 100: | 718 | case 100: |
759 | while (gethdr(buf, sizeof(buf), sfp /*, &n*/) != NULL) | 719 | while (gethdr(sfp) != NULL) |
760 | /* eat all remaining headers */; | 720 | /* eat all remaining headers */; |
761 | goto read_response; | 721 | goto read_response; |
762 | case 200: | 722 | case 200: |
@@ -796,22 +756,29 @@ However, in real world it was observed that some web servers | |||
796 | break; | 756 | break; |
797 | /* fall through */ | 757 | /* fall through */ |
798 | default: | 758 | default: |
799 | bb_error_msg_and_die("server returned error: %s", sanitize_string(buf)); | 759 | bb_error_msg_and_die("server returned error: %s", sanitize_string(G.wget_buf)); |
800 | } | 760 | } |
801 | 761 | ||
802 | /* | 762 | /* |
803 | * Retrieve HTTP headers. | 763 | * Retrieve HTTP headers. |
804 | */ | 764 | */ |
805 | while ((str = gethdr(buf, sizeof(buf), sfp /*, &n*/)) != NULL) { | 765 | while ((str = gethdr(sfp)) != NULL) { |
806 | /* gethdr converted "FOO:" string to lowercase */ | 766 | static const char keywords[] ALIGN1 = |
767 | "content-length\0""transfer-encoding\0""location\0"; | ||
768 | enum { | ||
769 | KEY_content_length = 1, KEY_transfer_encoding, KEY_location | ||
770 | }; | ||
807 | smalluint key; | 771 | smalluint key; |
772 | |||
773 | /* gethdr converted "FOO:" string to lowercase */ | ||
774 | |||
808 | /* strip trailing whitespace */ | 775 | /* strip trailing whitespace */ |
809 | char *s = strchrnul(str, '\0') - 1; | 776 | char *s = strchrnul(str, '\0') - 1; |
810 | while (s >= str && (*s == ' ' || *s == '\t')) { | 777 | while (s >= str && (*s == ' ' || *s == '\t')) { |
811 | *s = '\0'; | 778 | *s = '\0'; |
812 | s--; | 779 | s--; |
813 | } | 780 | } |
814 | key = index_in_strings(keywords, buf) + 1; | 781 | key = index_in_strings(keywords, G.wget_buf) + 1; |
815 | if (key == KEY_content_length) { | 782 | if (key == KEY_content_length) { |
816 | G.content_len = BB_STRTOOFF(str, NULL, 10); | 783 | G.content_len = BB_STRTOOFF(str, NULL, 10); |
817 | if (G.content_len < 0 || errno) { | 784 | if (G.content_len < 0 || errno) { |
@@ -821,23 +788,23 @@ However, in real world it was observed that some web servers | |||
821 | continue; | 788 | continue; |
822 | } | 789 | } |
823 | if (key == KEY_transfer_encoding) { | 790 | if (key == KEY_transfer_encoding) { |
824 | if (index_in_strings(keywords, str_tolower(str)) + 1 != KEY_chunked) | 791 | if (strcmp(str_tolower(str), "chunked") != 0) |
825 | bb_error_msg_and_die("transfer encoding '%s' is not supported", sanitize_string(str)); | 792 | bb_error_msg_and_die("transfer encoding '%s' is not supported", sanitize_string(str)); |
826 | G.chunked = G.got_clen = 1; | 793 | G.chunked = 1; |
827 | } | 794 | } |
828 | if (key == KEY_location && status >= 300) { | 795 | if (key == KEY_location && status >= 300) { |
829 | if (--redir_limit == 0) | 796 | if (--redir_limit == 0) |
830 | bb_error_msg_and_die("too many redirections"); | 797 | bb_error_msg_and_die("too many redirections"); |
831 | fclose(sfp); | 798 | fclose(sfp); |
832 | G.got_clen = 0; | 799 | if (str[0] == '/') { |
833 | G.chunked = 0; | 800 | free(target.allocated); |
834 | if (str[0] == '/') | 801 | target.path = target.allocated = xstrdup(str+1); |
835 | /* free(target.allocated); */ | ||
836 | target.path = /* target.allocated = */ xstrdup(str+1); | ||
837 | /* lsa stays the same: it's on the same server */ | 802 | /* lsa stays the same: it's on the same server */ |
838 | else { | 803 | } else { |
839 | parse_url(str, &target); | 804 | parse_url(str, &target); |
840 | if (!use_proxy) { | 805 | if (!use_proxy) { |
806 | free(server.allocated); | ||
807 | server.allocated = NULL; | ||
841 | server.host = target.host; | 808 | server.host = target.host; |
842 | /* strip_ipv6_scope_id(target.host); - no! */ | 809 | /* strip_ipv6_scope_id(target.host); - no! */ |
843 | /* we assume remote never gives us IPv6 addr with scope id */ | 810 | /* we assume remote never gives us IPv6 addr with scope id */ |
@@ -862,30 +829,117 @@ However, in real world it was observed that some web servers | |||
862 | sfp = prepare_ftp_session(&dfp, &target, lsa); | 829 | sfp = prepare_ftp_session(&dfp, &target, lsa); |
863 | } | 830 | } |
864 | 831 | ||
865 | if (opt & WGET_OPT_SPIDER) { | 832 | free(lsa); |
866 | if (ENABLE_FEATURE_CLEAN_UP) | ||
867 | fclose(sfp); | ||
868 | return EXIT_SUCCESS; | ||
869 | } | ||
870 | 833 | ||
871 | if (output_fd < 0) { | 834 | if (!(option_mask32 & WGET_OPT_SPIDER)) { |
872 | int o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL; | 835 | if (G.output_fd < 0) |
873 | /* compat with wget: -O FILE can overwrite */ | 836 | G.output_fd = xopen(G.fname_out, G.o_flags); |
874 | if (opt & WGET_OPT_OUTNAME) | 837 | retrieve_file_data(dfp); |
875 | o_flags = O_WRONLY | O_CREAT | O_TRUNC; | 838 | if (!(option_mask32 & WGET_OPT_OUTNAME)) { |
876 | output_fd = xopen(fname_out, o_flags); | 839 | xclose(G.output_fd); |
840 | G.output_fd = -1; | ||
841 | } | ||
877 | } | 842 | } |
878 | 843 | ||
879 | retrieve_file_data(dfp, output_fd); | ||
880 | xclose(output_fd); | ||
881 | |||
882 | if (dfp != sfp) { | 844 | if (dfp != sfp) { |
883 | /* It's ftp. Close it properly */ | 845 | /* It's ftp. Close data connection properly */ |
884 | fclose(dfp); | 846 | fclose(dfp); |
885 | if (ftpcmd(NULL, NULL, sfp, buf) != 226) | 847 | if (ftpcmd(NULL, NULL, sfp) != 226) |
886 | bb_error_msg_and_die("ftp error: %s", sanitize_string(buf+4)); | 848 | bb_error_msg_and_die("ftp error: %s", sanitize_string(G.wget_buf + 4)); |
887 | /* ftpcmd("QUIT", NULL, sfp, buf); - why bother? */ | 849 | /* ftpcmd("QUIT", NULL, sfp); - why bother? */ |
850 | } | ||
851 | fclose(sfp); | ||
852 | |||
853 | free(server.allocated); | ||
854 | free(target.allocated); | ||
855 | free(fname_out_alloc); | ||
856 | } | ||
857 | |||
858 | int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; | ||
859 | int wget_main(int argc UNUSED_PARAM, char **argv) | ||
860 | { | ||
861 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
862 | static const char wget_longopts[] ALIGN1 = | ||
863 | /* name, has_arg, val */ | ||
864 | "continue\0" No_argument "c" | ||
865 | //FIXME: -s isn't --spider, it's --save-headers! | ||
866 | "spider\0" No_argument "s" | ||
867 | "quiet\0" No_argument "q" | ||
868 | "output-document\0" Required_argument "O" | ||
869 | "directory-prefix\0" Required_argument "P" | ||
870 | "proxy\0" Required_argument "Y" | ||
871 | "user-agent\0" Required_argument "U" | ||
872 | #if ENABLE_FEATURE_WGET_TIMEOUT | ||
873 | "timeout\0" Required_argument "T" | ||
874 | #endif | ||
875 | /* Ignored: */ | ||
876 | // "tries\0" Required_argument "t" | ||
877 | /* Ignored (we always use PASV): */ | ||
878 | "passive-ftp\0" No_argument "\xff" | ||
879 | "header\0" Required_argument "\xfe" | ||
880 | "post-data\0" Required_argument "\xfd" | ||
881 | /* Ignored (we don't do ssl) */ | ||
882 | "no-check-certificate\0" No_argument "\xfc" | ||
883 | ; | ||
884 | #endif | ||
885 | |||
886 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
887 | llist_t *headers_llist = NULL; | ||
888 | #endif | ||
889 | |||
890 | INIT_G(); | ||
891 | IF_WIN32_NET(init_winsock();) | ||
892 | |||
893 | IF_FEATURE_WGET_TIMEOUT(G.timeout_seconds = 900;) | ||
894 | G.proxy_flag = "on"; /* use proxies if env vars are set */ | ||
895 | G.user_agent = "Wget"; /* "User-Agent" header field */ | ||
896 | |||
897 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
898 | applet_long_options = wget_longopts; | ||
899 | #endif | ||
900 | opt_complementary = "-1" IF_FEATURE_WGET_TIMEOUT(":T+") IF_FEATURE_WGET_LONG_OPTIONS(":\xfe::"); | ||
901 | getopt32(argv, "csqO:P:Y:U:T:" /*ignored:*/ "t:", | ||
902 | &G.fname_out, &G.dir_prefix, | ||
903 | &G.proxy_flag, &G.user_agent, | ||
904 | IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL), | ||
905 | NULL /* -t RETRIES */ | ||
906 | IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist) | ||
907 | IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data) | ||
908 | ); | ||
909 | argv += optind; | ||
910 | |||
911 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS | ||
912 | if (headers_llist) { | ||
913 | int size = 1; | ||
914 | char *cp; | ||
915 | llist_t *ll = headers_llist; | ||
916 | while (ll) { | ||
917 | size += strlen(ll->data) + 2; | ||
918 | ll = ll->link; | ||
919 | } | ||
920 | G.extra_headers = cp = xmalloc(size); | ||
921 | while (headers_llist) { | ||
922 | cp += sprintf(cp, "%s\r\n", (char*)llist_pop(&headers_llist)); | ||
923 | } | ||
888 | } | 924 | } |
925 | #endif | ||
926 | |||
927 | G.output_fd = -1; | ||
928 | G.o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL; | ||
929 | if (G.fname_out) { /* -O FILE ? */ | ||
930 | if (LONE_DASH(G.fname_out)) { /* -O - ? */ | ||
931 | G.output_fd = 1; | ||
932 | option_mask32 &= ~WGET_OPT_CONTINUE; | ||
933 | } | ||
934 | /* compat with wget: -O FILE can overwrite */ | ||
935 | G.o_flags = O_WRONLY | O_CREAT | O_TRUNC; | ||
936 | } | ||
937 | |||
938 | while (*argv) | ||
939 | download_one_url(*argv++); | ||
940 | |||
941 | if (G.output_fd >= 0) | ||
942 | xclose(G.output_fd); | ||
889 | 943 | ||
890 | return EXIT_SUCCESS; | 944 | return EXIT_SUCCESS; |
891 | } | 945 | } |