8 files changed, 175 insertions, 98 deletions

diff --git a/libbb/xfuncs_printf.c b/libbb/xfuncs_printf.c
index 7247c915b..6cc60f6c0 100644
--- a/libbb/xfuncs_printf.c
+++ b/libbb/xfuncs_printf.c
@@ -222,6 +222,7 @@ void FAST_FUNC xdup2(int from, int to)
 {
         if (dup2(from, to) != to)
                 bb_perror_msg_and_die("can't duplicate file descriptor");
+                //              " %d to %d", from, to);
 }
 
 // "Renumber" opened fd
diff --git a/shell/ash_test/ash-heredoc/heredocB.right b/shell/ash_test/ash-heredoc/heredocB.right
new file mode 100644
index 000000000..43ba0b4f9
--- /dev/null
+++ b/shell/ash_test/ash-heredoc/heredocB.right
@@ -0,0 +1,3 @@
+one - alpha
+two - beta
+three - gamma
diff --git a/shell/ash_test/ash-heredoc/heredocB.tests b/shell/ash_test/ash-heredoc/heredocB.tests
new file mode 100755
index 000000000..45ea4687f
--- /dev/null
+++ b/shell/ash_test/ash-heredoc/heredocB.tests
@@ -0,0 +1,12 @@
+while read line1; do
+        read line2 <&3
+        echo $line1 - $line2
+done <<EOF1 3<<EOF2
+one
+two
+three
+EOF1
+alpha
+beta
+gamma
+EOF2
diff --git a/shell/ash_test/ash-redir/redir_script.tests b/shell/ash_test/ash-redir/redir_script.tests
index 740daa461..a8d93ce4f 100755
--- a/shell/ash_test/ash-redir/redir_script.tests
+++ b/shell/ash_test/ash-redir/redir_script.tests
@@ -27,6 +27,10 @@ test x"$fds1" = x"$fds" \
 test x"$fds1" = x" 10>&- 3>&-" && \
 test x"$fds"  = x" 11>&- 3>&-" \
 && { echo "Ok: script fd is not closed"; exit 0; }
+# or we see that fd 3 moved to fd 10:
+test x"$fds1" = x" 3>&- 4>&-" && \
+test x"$fds"  = x" 10>&- 3>&-" \
+&& { echo "Ok: script fd is not closed"; exit 0; }
 
 echo "Bug: script fd is closed"
 echo "fds1:$fds1"
diff --git a/shell/hush.c b/shell/hush.c
index c26484b49..7a1513c24 100644
--- a/shell/hush.c
+++ b/shell/hush.c
@@ -357,6 +357,9 @@
 #else
 # define CLEAR_RANDOM_T(rnd) ((void)0)
 #endif
+#ifndef O_CLOEXEC
+# define O_CLOEXEC 0
+#endif
 #ifndef F_DUPFD_CLOEXEC
 # define F_DUPFD_CLOEXEC F_DUPFD
 #endif
@@ -556,11 +559,27 @@ static const char *const assignment_flag[] = {
 };
 #endif
 
+/* We almost can use standard FILE api, but we need an ability to move
+ * its fd when redirects coincide with it. No api exists for that
+ * (RFE for it at https://sourceware.org/bugzilla/show_bug.cgi?id=21902).
+ * HFILE is our internal alternative. Only supports reading.
+ * Since we now can, we incorporate linked list of all opened HFILEs
+ * into the struct (used to be a separate mini-list).
+ */
+typedef struct HFILE {
+        char *cur;
+        char *end;
+        struct HFILE *next_hfile;
+        int is_stdin;
+        int fd;
+        char buf[1024];
+} HFILE;
+
 typedef struct in_str {
         const char *p;
         int peek_buf[2];
         int last_char;
-        FILE *file;
+        HFILE *file;
 } in_str;
 
 /* The descrip member of this structure is only used to make
@@ -814,14 +833,6 @@ enum {
         NUM_OPT_O
 };
 
-
-struct FILE_list {
-        struct FILE_list *next;
-        FILE *fp;
-        int fd;
-};
-
-
 /* "Globals" within this file */
 /* Sorted roughly by size (smaller offsets == smaller code) */
 struct globals {
@@ -954,7 +965,7 @@ struct globals {
         unsigned lineno;
         char *lineno_var;
 #endif
-        struct FILE_list *FILE_list;
+        HFILE *HFILE_list;
         /* Which signals have non-DFL handler (even with no traps set)?
          * Set at the start to:
          * (SIGQUIT + maybe SPECIAL_INTERACTIVE_SIGS + maybe SPECIAL_JOBSTOP_SIGS)
@@ -1563,83 +1574,115 @@ static int xdup_CLOEXEC_and_close(int fd, int avoid_fd)
 }
 
 
-/* Manipulating the list of open FILEs */
-static FILE *remember_FILE(FILE *fp)
+/* Manipulating HFILEs */
+static HFILE *hfopen(const char *name)
 {
-        if (fp) {
-                struct FILE_list *n = xmalloc(sizeof(*n));
-                n->next = G.FILE_list;
-                G.FILE_list = n;
-                n->fp = fp;
-                n->fd = fileno(fp);
-                close_on_exec_on(n->fd);
+        HFILE *fp;
+        int fd;
+
+        fd = STDIN_FILENO;
+        if (name) {
+                fd = open(name, O_RDONLY | O_CLOEXEC);
+                if (fd < 0)
+                        return NULL;
+                if (O_CLOEXEC == 0) /* ancient libc */
+                        close_on_exec_on(fd);
         }
+
+        fp = xmalloc(sizeof(*fp));
+        fp->is_stdin = (name == NULL);
+        fp->fd = fd;
+        fp->cur = fp->end = fp->buf;
+        fp->next_hfile = G.HFILE_list;
+        G.HFILE_list = fp;
         return fp;
 }
-static void fclose_and_forget(FILE *fp)
+static void hfclose(HFILE *fp)
 {
-        struct FILE_list **pp = &G.FILE_list;
+        HFILE **pp = &G.HFILE_list;
         while (*pp) {
-                struct FILE_list *cur = *pp;
-                if (cur->fp == fp) {
-                        *pp = cur->next;
-                        free(cur);
+                HFILE *cur = *pp;
+                if (cur == fp) {
+                        *pp = cur->next_hfile;
                         break;
                 }
-                pp = &cur->next;
+                pp = &cur->next_hfile;
         }
-        fclose(fp);
+        if (fp->fd >= 0)
+                close(fp->fd);
+        free(fp);
 }
-static int save_FILEs_on_redirect(int fd, int avoid_fd)
+static int refill_HFILE_and_getc(HFILE *fp)
 {
-        struct FILE_list *fl = G.FILE_list;
+        int n;
+
+        if (fp->fd < 0) {
+                /* Already saw EOF */
+                return EOF;
+        }
+        /* Try to buffer more input */
+        fp->cur = fp->buf;
+        n = safe_read(fp->fd, fp->buf, sizeof(fp->buf));
+        if (n < 0) {
+                bb_perror_msg("read error");
+                n = 0;
+        }
+        fp->end = fp->buf + n;
+        if (n == 0) {
+                /* EOF/error */
+                close(fp->fd);
+                fp->fd = -1;
+                return EOF;
+        }
+        return (unsigned char)(*fp->cur++);
+}
+/* Inlined for common case of non-empty buffer.
+ */
+static ALWAYS_INLINE int hfgetc(HFILE *fp)
+{
+        if (fp->cur < fp->end)
+                return (unsigned char)(*fp->cur++);
+        /* Buffer empty */
+        return refill_HFILE_and_getc(fp);
+}
+static int move_HFILEs_on_redirect(int fd, int avoid_fd)
+{
+        HFILE *fl = G.HFILE_list;
         while (fl) {
                 if (fd == fl->fd) {
                         /* We use it only on script files, they are all CLOEXEC */
                         fl->fd = xdup_CLOEXEC_and_close(fd, avoid_fd);
                         debug_printf_redir("redirect_fd %d: matches a script fd, moving it to %d\n", fd, fl->fd);
-                        return 1;
-                }
-                fl = fl->next;
-        }
-        return 0;
-}
-static void restore_redirected_FILEs(void)
-{
-        struct FILE_list *fl = G.FILE_list;
-        while (fl) {
-                int should_be = fileno(fl->fp);
-                if (fl->fd != should_be) {
-                        debug_printf_redir("restoring script fd from %d to %d\n", fl->fd, should_be);
-                        xmove_fd(fl->fd, should_be);
-                        fl->fd = should_be;
+                        return 1; /* "found and moved" */
                 }
-                fl = fl->next;
+                fl = fl->next_hfile;
         }
+        return 0; /* "not in the list" */
 }
 #if ENABLE_FEATURE_SH_STANDALONE && BB_MMU
-static void close_all_FILE_list(void)
+static void close_all_HFILE_list(void)
 {
-        struct FILE_list *fl = G.FILE_list;
+        HFILE *fl = G.HFILE_list;
         while (fl) {
-                /* fclose would also free FILE object.
+                /* hfclose would also free HFILE object.
                  * It is disastrous if we share memory with a vforked parent.
                  * I'm not sure we never come here after vfork.
                  * Therefore just close fd, nothing more.
                  */
-                /*fclose(fl->fp); - unsafe */
-                close(fl->fd);
-                fl = fl->next;
+                /*hfclose(fl); - unsafe */
+                if (fl->fd >= 0)
+                        close(fl->fd);
+                fl = fl->next_hfile;
         }
 }
 #endif
-static int fd_in_FILEs(int fd)
+static int fd_in_HFILEs(int fd)
 {
-        struct FILE_list *fl = G.FILE_list;
+        HFILE *fl = G.HFILE_list;
         while (fl) {
                 if (fl->fd == fd)
                         return 1;
-                fl = fl->next;
+                fl = fl->next_hfile;
         }
         return 0;
 }
@@ -2580,7 +2623,7 @@ static int get_user_input(struct in_str *i)
                 }
                 fflush_all();
 //FIXME: here ^C or SIGINT will have effect only after <Enter>
-                r = fgetc(i->file);
+                r = hfgetc(i->file);
                 /* In !ENABLE_FEATURE_EDITING we don't use read_line_input,
                  * no ^C masking happens during fgetc, no special code for ^C:
                  * it generates SIGINT as usual.
@@ -2600,22 +2643,22 @@ static int fgetc_interactive(struct in_str *i)
 {
         int ch;
         /* If it's interactive stdin, get new line. */
-        if (G_interactive_fd && i->file == stdin) {
+        if (G_interactive_fd && i->file->is_stdin) {
                 /* Returns first char (or EOF), the rest is in i->p[] */
                 ch = get_user_input(i);
                 G.promptmode = 1; /* PS2 */
                 debug_printf_prompt("%s promptmode=%d\n", __func__, G.promptmode);
         } else {
                 /* Not stdin: script file, sourced file, etc */
-                do ch = fgetc(i->file); while (ch == '\0');
+                do ch = hfgetc(i->file); while (ch == '\0');
         }
         return ch;
 }
 #else
-static inline int fgetc_interactive(struct in_str *i)
+static ALWAYS_INLINE int fgetc_interactive(struct in_str *i)
 {
         int ch;
-        do ch = fgetc(i->file); while (ch == '\0');
+        do ch = hfgetc(i->file); while (ch == '\0');
         return ch;
 }
 #endif  /* INTERACTIVE */
@@ -2730,7 +2773,7 @@ static int i_peek2(struct in_str *i)
         ch = i->peek_buf[1];
         if (ch == 0) {
                 /* We did not read it yet, get it now */
-                do ch = fgetc(i->file); while (ch == '\0');
+                do ch = hfgetc(i->file); while (ch == '\0');
                 i->peek_buf[1] = ch;
         }
 
@@ -2774,10 +2817,10 @@ static int i_peek_and_eat_bkslash_nl(struct in_str *input)
         }
 }
 
-static void setup_file_in_str(struct in_str *i, FILE *f)
+static void setup_file_in_str(struct in_str *i, HFILE *fp)
 {
         memset(i, 0, sizeof(*i));
-        i->file = f;
+        i->file = fp;
         /* i->p = NULL; */
 }
 
@@ -7106,19 +7149,19 @@ static void parse_and_run_string(const char *s)
         //IF_HUSH_LINENO_VAR(G.lineno = sv;)
 }
 
-static void parse_and_run_file(FILE *f)
+static void parse_and_run_file(HFILE *fp)
 {
         struct in_str input;
         IF_HUSH_LINENO_VAR(unsigned sv = G.lineno;)
 
         IF_HUSH_LINENO_VAR(G.lineno = 1;)
-        setup_file_in_str(&input, f);
+        setup_file_in_str(&input, fp);
         parse_and_run_stream(&input, ';');
         IF_HUSH_LINENO_VAR(G.lineno = sv;)
 }
 
 #if ENABLE_HUSH_TICK
-static FILE *generate_stream_from_string(const char *s, pid_t *pid_p)
+static int generate_stream_from_string(const char *s, pid_t *pid_p)
 {
         pid_t pid;
         int channel[2];
@@ -7220,7 +7263,7 @@ static FILE *generate_stream_from_string(const char *s, pid_t *pid_p)
         free(to_free);
 # endif
         close(channel[1]);
-        return remember_FILE(xfdopen_for_read(channel[0]));
+        return channel[0];
 }
 
 /* Return code is exit status of the process that is run. */
@@ -7230,7 +7273,7 @@ static int process_command_subs(o_string *dest, const char *s)
         pid_t pid;
         int status, ch, eol_cnt;
 
-        fp = generate_stream_from_string(s, &pid);
+        fp = xfdopen_for_read(generate_stream_from_string(s, &pid));
 
         /* Now send results of command back into original context */
         eol_cnt = 0;
@@ -7249,7 +7292,7 @@ static int process_command_subs(o_string *dest, const char *s)
         }
 
         debug_printf("done reading from `cmd` pipe, closing it\n");
-        fclose_and_forget(fp);
+        fclose(fp);
         /* We need to extract exitcode. Test case
          * "true; echo `sleep 1; false` $?"
          * should print 1 */
@@ -7427,20 +7470,17 @@ static int save_fd_on_redirect(int fd, int avoid_fd, struct squirrel **sqp)
                 return 1; /* "we closed fd" */
         }
 #endif
+        /* If this one of script's fds? */
+        if (move_HFILEs_on_redirect(fd, avoid_fd))
+                return 1; /* yes. "we closed fd" (actually moved it) */
+
         /* Are we called from setup_redirects(squirrel==NULL)? Two cases:
-         * (1) Redirect in a forked child. No need to save FILEs' fds,
-         * we aren't going to use them anymore, ok to trash.
-         * (2) "exec 3>FILE". Bummer. We can save script FILEs' fds,
-         * but how are we doing to restore them?
-         * "fileno(fd) = new_fd" can't be done.
+         * (1) Redirect in a forked child.
+         * (2) "exec 3>FILE".
          */
         if (!sqp)
                 return 0;
 
-        /* If this one of script's fds? */
-        if (save_FILEs_on_redirect(fd, avoid_fd))
-                return 1; /* yes. "we closed fd" */
-
         /* Check whether it collides with any open fds (e.g. stdio), save fds as needed */
         *sqp = add_squirrel(*sqp, fd, avoid_fd);
         return 0; /* "we did not close fd" */
@@ -7465,8 +7505,6 @@ static void restore_redirects(struct squirrel *sq)
         }
 
         /* If moved, G.interactive_fd stays on new fd, not restoring it */
-
-        restore_redirected_FILEs();
 }
 
 #if ENABLE_FEATURE_SH_STANDALONE && BB_MMU
@@ -7474,7 +7512,7 @@ static void close_saved_fds_and_FILE_fds(void)
 {
         if (G_interactive_fd)
                 close(G_interactive_fd);
-        close_all_FILE_list();
+        close_all_HFILE_list();
 }
 #endif
 
@@ -7487,7 +7525,7 @@ static int internally_opened_fd(int fd, struct squirrel *sq)
                 return 1;
 #endif
         /* If this one of script's fds? */
-        if (fd_in_FILEs(fd))
+        if (fd_in_HFILEs(fd))
                 return 1;
 
         if (sq) for (i = 0; sq[i].orig_fd >= 0; i++) {
@@ -7512,7 +7550,7 @@ static int setup_redirects(struct command *prog, struct squirrel **sqp)
                         save_fd_on_redirect(redir->rd_fd, /*avoid:*/ 0, sqp);
                         /* for REDIRECT_HEREDOC2, rd_filename holds _contents_
                          * of the heredoc */
-                        debug_printf_parse("set heredoc '%s'\n",
+                        debug_printf_redir("set heredoc '%s'\n",
                                         redir->rd_filename);
                         setup_heredoc(redir);
                         continue;
@@ -7524,8 +7562,7 @@ static int setup_redirects(struct command *prog, struct squirrel **sqp)
                         int mode;
 
                         if (redir->rd_filename == NULL) {
-                                /*
-                                 * Examples:
+                                /* Examples:
                                  * "cmd >" (no filename)
                                  * "cmd > <file" (2nd redirect starts too early)
                                  */
@@ -8706,7 +8743,7 @@ static NOINLINE int run_pipe(struct pipe *pi)
                 }
 #endif
                 /* { list } */
-                debug_printf("non-subshell group\n");
+                debug_printf_exec("non-subshell group\n");
                 rcode = 1; /* exitcode if redir failed */
                 if (setup_redirects(command, &squirrel) == 0) {
                         debug_printf_exec(": run_list\n");
@@ -9871,14 +9908,13 @@ int hush_main(int argc, char **argv)
 
         /* If we are login shell... */
         if (flags & OPT_login) {
-                FILE *input;
+                HFILE *input;
                 debug_printf("sourcing /etc/profile\n");
-                input = fopen_for_read("/etc/profile");
+                input = hfopen("/etc/profile");
                 if (input != NULL) {
-                        remember_FILE(input);
                         install_special_sighandlers();
                         parse_and_run_file(input);
-                        fclose_and_forget(input);
+                        hfclose(input);
                 }
                 /* bash: after sourcing /etc/profile,
                  * tries to source (in the given order):
@@ -9891,7 +9927,7 @@ int hush_main(int argc, char **argv)
 
         /* -s is: hush -s ARGV1 ARGV2 (no SCRIPT) */
         if (!(flags & OPT_s) && G.global_argv[1]) {
-                FILE *input;
+                HFILE *input;
                 /*
                  * "bash <script>" (which is never interactive (unless -i?))
                  * sources $BASH_ENV here (without scanning $PATH).
@@ -9902,13 +9938,15 @@ int hush_main(int argc, char **argv)
                 G.global_argv++;
                 debug_printf("running script '%s'\n", G.global_argv[0]);
                 xfunc_error_retval = 127; /* for "hush /does/not/exist" case */
-                input = xfopen_for_read(G.global_argv[0]);
+                input = hfopen(G.global_argv[0]);
+                if (!input) {
+                        bb_simple_perror_msg_and_die(G.global_argv[0]);
+                }
                 xfunc_error_retval = 1;
-                remember_FILE(input);
                 install_special_sighandlers();
                 parse_and_run_file(input);
 #if ENABLE_FEATURE_CLEAN_UP
-                fclose_and_forget(input);
+                hfclose(input);
 #endif
                 goto final_return;
         }
@@ -10038,7 +10076,7 @@ int hush_main(int argc, char **argv)
                 );
         }
 
-        parse_and_run_file(stdin);
+        parse_and_run_file(hfopen(NULL)); /* stdin */
 
  final_return:
         hush_exit(G.last_exitcode);
@@ -10849,7 +10887,7 @@ Test that VAR is a valid variable name?
 static int FAST_FUNC builtin_source(char **argv)
 {
         char *arg_path, *filename;
-        FILE *input;
+        HFILE *input;
         save_arg_t sv;
         char *args_need_save;
 #if ENABLE_HUSH_FUNCTIONS
@@ -10873,10 +10911,10 @@ static int FAST_FUNC builtin_source(char **argv)
                         return EXIT_FAILURE;
                 }
         }
-        input = remember_FILE(fopen_or_warn(filename, "r"));
+        input = hfopen(filename);
         free(arg_path);
         if (!input) {
-                /* bb_perror_msg("%s", *argv); - done by fopen_or_warn */
+                bb_perror_msg("%s", filename);
                 /* POSIX: non-interactive shell should abort here,
                  * not merely fail. So far no one complained :)
                  */
@@ -10895,7 +10933,7 @@ static int FAST_FUNC builtin_source(char **argv)
         /* "false; . ./empty_line; echo Zero:$?" should print 0 */
         G.last_exitcode = 0;
         parse_and_run_file(input);
-        fclose_and_forget(input);
+        hfclose(input);
 
         if (args_need_save) /* can't use argv[1] instead: "shift" can mangle it */
                 restore_G_args(&sv, argv);
diff --git a/shell/hush_test/hush-heredoc/heredocB.right b/shell/hush_test/hush-heredoc/heredocB.right
new file mode 100644
index 000000000..43ba0b4f9
--- /dev/null
+++ b/shell/hush_test/hush-heredoc/heredocB.right
@@ -0,0 +1,3 @@
+one - alpha
+two - beta
+three - gamma
diff --git a/shell/hush_test/hush-heredoc/heredocB.tests b/shell/hush_test/hush-heredoc/heredocB.tests
new file mode 100755
index 000000000..45ea4687f
--- /dev/null
+++ b/shell/hush_test/hush-heredoc/heredocB.tests
@@ -0,0 +1,12 @@
+while read line1; do
+        read line2 <&3
+        echo $line1 - $line2
+done <<EOF1 3<<EOF2
+one
+two
+three
+EOF1
+alpha
+beta
+gamma
+EOF2
diff --git a/shell/hush_test/hush-redir/redir_script.tests b/shell/hush_test/hush-redir/redir_script.tests
index 740daa461..a8d93ce4f 100755
--- a/shell/hush_test/hush-redir/redir_script.tests
+++ b/shell/hush_test/hush-redir/redir_script.tests
@@ -27,6 +27,10 @@ test x"$fds1" = x"$fds" \
 test x"$fds1" = x" 10>&- 3>&-" && \
 test x"$fds"  = x" 11>&- 3>&-" \
 && { echo "Ok: script fd is not closed"; exit 0; }
+# or we see that fd 3 moved to fd 10:
+test x"$fds1" = x" 3>&- 4>&-" && \
+test x"$fds"  = x" 10>&- 3>&-" \
+&& { echo "Ok: script fd is not closed"; exit 0; }
 
 echo "Bug: script fd is closed"
 echo "fds1:$fds1"