win32: changes to console mode handling

Add the environment variable BB_TERMINAL_MODE as a more general way
of controlling console/terminal mode setting.  The default remains
unchanged:  use virtual terminal mode for output if possible but
fall back to the console API with emulated ANSI escape sequences.

Currently valid settings are:

   0  Force use of console mode
   1  Force use of virtual terminal mode for output
   5  Prefer virtual terminal mode for output, fall back to console

Other values won't do anything useful until code elsewhere has been
updated.

BB_SKIP_ANSI_EMULATION remains available for backwards compatibility.
If both variables are set BB_TERMINAL_MODE takes precedence.

11 files changed, 93 insertions, 43 deletions

diff --git a/Config.in b/Config.in
index 2a2672f29..11990ed41 100644
--- a/Config.in
+++ b/Config.in
@@ -455,22 +455,27 @@ config FEATURE_EURO
         requires the OEM code page to be 858.  If the OEM code page of
         the console is 850 when BusyBox starts it's changed to 858.
 
-config SKIP_ANSI_EMULATION_DEFAULT
-        int "Default setting for ANSI escape sequences"
-        default 2
-        range 0 2
+config TERMINAL_MODE
+        int "Default setting for terminal mode"
+        default 5
+        range 0 7
         depends on PLATFORM_MINGW32
         help
-        Control how ANSI escape sequences are handled.  Possible values
-        are:
+        Control I/O mode of the Windows console/terminal.  Possible
+        values are:
 
-        0  Always emulate escape sequences.
-        1  Always emit escape sequences.
-        2  Emit escape sequences if the terminal supports them, otherwise
-           emulate them.
+        0  Console mode.
+        1  Force terminal mode output.
+        2  Force terminal mode input.
+        3  Force terminal mode input and output.
 
-        Setting the environment variable BB_SKIP_ANSI_EMULATION overrides
-        this default.
+        4  Console mode.
+        5  Prefer terminal mode output, fall back to console mode.
+        6  Prefer terminal mode input, fall back to console mode.
+        7  Prefer terminal mode input and output, fall back to console.
+
+        Setting the environment variable BB_TERMINAL_MODE overrides this
+        default.
 
 config FEATURE_IMPROVED_COLOUR_MAPPING
         bool "More accurate colour mapping for ANSI emulation (0.6 kb)"
diff --git a/README.md b/README.md
index 2ea2fe9c0..2857eb51d 100644
--- a/README.md
+++ b/README.md
@@ -31,5 +31,5 @@ Then just `make`.
  - Handling of users, groups and permissions is totally bogus.  The system only admits to knowing about the current user and always returns the same hardcoded uid, gid and permission values.
  - Some crufty old Windows code (Windows XP, cmd.exe) doesn't like forward slashes in environment variables.  The -X shell option (which must be the first argument) prevents busybox-w32 from changing backslashes to forward slashes.  If Windows programs don't run from the shell it's worth trying it.
  - If you want to install 32-bit BusyBox in a system directory on a 64-bit version of Windows you should put it in `C:\Windows\SysWOW64`, not `C:\Windows\System32` as you might expect.  On 64-bit systems the latter is for 64-bit binaries.
- - The system tries to detect the best way to handle ANSI escape sequences for the terminal being used.  If this doesn't work you can try setting the environment variable `BB_SKIP_ANSI_EMULATION=1` to force the use of literal ANSI escapes or `BB_SKIP_ANSI_EMULATION=0` to emulate them using the Windows console API.
+ - The system tries to detect the best way to handle the terminal being used.  If this doesn't work you can try setting the environment variable `BB_TERMINAL_MODE=1` to force the use of literal ANSI escapes or `BB_TERMINAL_MODE=0` to emulate them using the Windows console API.
  - It's possible to obtain pseudo-random numbers using `if=/dev/urandom` as the input file to `dd`.  The same emulation of `/dev/urandom` is used internally by the `shred` utility and to support https in `wget`.  Since the pseudo-random number generator isn't being seeded with sufficient entropy the randomness shouldn't be relied on for any serious use.
diff --git a/configs/make32_defconfig b/configs/make32_defconfig
index 7f7082abc..2c5c15ba9 100644
--- a/configs/make32_defconfig
+++ b/configs/make32_defconfig
@@ -52,7 +52,7 @@ CONFIG_FEATURE_PRNG_SHELL=y
 # CONFIG_FEATURE_ICON_STERM is not set
 # CONFIG_FEATURE_ICON_ALL is not set
 # CONFIG_FEATURE_EURO is not set
-CONFIG_SKIP_ANSI_EMULATION_DEFAULT=2
+CONFIG_TERMINAL_MODE=5
 # CONFIG_FEATURE_IMPROVED_COLOUR_MAPPING is not set
 CONFIG_FEATURE_EXTRA_FILE_DATA=y
 
diff --git a/configs/make64_defconfig b/configs/make64_defconfig
index 524025f6b..5fa5c4ebc 100644
--- a/configs/make64_defconfig
+++ b/configs/make64_defconfig
@@ -52,7 +52,7 @@ CONFIG_FEATURE_PRNG_SHELL=y
 # CONFIG_FEATURE_ICON_STERM is not set
 # CONFIG_FEATURE_ICON_ALL is not set
 # CONFIG_FEATURE_EURO is not set
-CONFIG_SKIP_ANSI_EMULATION_DEFAULT=2
+CONFIG_TERMINAL_MODE=5
 # CONFIG_FEATURE_IMPROVED_COLOUR_MAPPING is not set
 CONFIG_FEATURE_EXTRA_FILE_DATA=y
 
diff --git a/configs/mingw32_defconfig b/configs/mingw32_defconfig
index 78035dd25..360ea3b50 100644
--- a/configs/mingw32_defconfig
+++ b/configs/mingw32_defconfig
@@ -52,7 +52,7 @@ CONFIG_FEATURE_ICON=y
 # CONFIG_FEATURE_ICON_STERM is not set
 CONFIG_FEATURE_ICON_ALL=y
 CONFIG_FEATURE_EURO=y
-CONFIG_SKIP_ANSI_EMULATION_DEFAULT=2
+CONFIG_TERMINAL_MODE=5
 CONFIG_FEATURE_IMPROVED_COLOUR_MAPPING=y
 CONFIG_FEATURE_EXTRA_FILE_DATA=y
 
diff --git a/configs/mingw64_defconfig b/configs/mingw64_defconfig
index 1ef6024c1..0dc1d0d51 100644
--- a/configs/mingw64_defconfig
+++ b/configs/mingw64_defconfig
@@ -52,7 +52,7 @@ CONFIG_FEATURE_ICON=y
 # CONFIG_FEATURE_ICON_STERM is not set
 CONFIG_FEATURE_ICON_ALL=y
 CONFIG_FEATURE_EURO=y
-CONFIG_SKIP_ANSI_EMULATION_DEFAULT=2
+CONFIG_TERMINAL_MODE=5
 CONFIG_FEATURE_IMPROVED_COLOUR_MAPPING=y
 CONFIG_FEATURE_EXTRA_FILE_DATA=y
 
diff --git a/include/libbb.h b/include/libbb.h
index 94f0e40ec..4197a5e83 100644
--- a/include/libbb.h
+++ b/include/libbb.h
@@ -2375,7 +2375,8 @@ extern const char bbvar[] ALIGN1;
 #define bbafter(p) (p + sizeof(#p))
 #define BB_OVERRIDE_APPLETS bbvar
 #define BB_SKIP_ANSI_EMULATION bbafter(BB_OVERRIDE_APPLETS)
-#define BB_SYSTEMROOT bbafter(BB_SKIP_ANSI_EMULATION)
+#define BB_TERMINAL_MODE bbafter(BB_SKIP_ANSI_EMULATION)
+#define BB_SYSTEMROOT bbafter(BB_TERMINAL_MODE)
 #endif
 
 extern const int const_int_0;
diff --git a/include/mingw.h b/include/mingw.h
index 6259bc2c6..b5b2fe169 100644
--- a/include/mingw.h
+++ b/include/mingw.h
@@ -545,6 +545,9 @@ char *is_prefixed_with_case(const char *string, const char *key) FAST_FUNC;
 char *is_suffixed_with_case(const char *string, const char *key) FAST_FUNC;
 void qsort_string_vector_case(char **sv, unsigned count) FAST_FUNC;
 
+#define VT_OUTPUT       1
+#define VT_INPUT        2
+
 /*
  * helpers
  */
@@ -581,7 +584,7 @@ char *xabsolute_path(char *path);
 char *get_drive_cwd(const char *path, char *buffer, int size);
 void fix_path_case(char *path);
 void make_sparse(int fd, off_t start, off_t end);
-int skip_ansi_emulation(int reset);
+int terminal_mode(int reset);
 int unix_path(const char *path);
 int has_path(const char *file);
 int is_relative_path(const char *path);
diff --git a/libbb/messages.c b/libbb/messages.c
index e35bf8c6b..3c9d8683a 100644
--- a/libbb/messages.c
+++ b/libbb/messages.c
@@ -36,10 +36,14 @@ const char bb_busybox_exec_path[] ALIGN1 = CONFIG_BUSYBOX_EXEC_PATH;
  * BB_GLOBBING and BB_UMASK are excluded because users shouln't be
  * messing with them; BB_ALT_BUFFER and BB_FIX_BACKSLASH are excluded
  * because they only affect particular applets, not the shell itself.
+ *
+ * If you change any of these you should also update the definitions in
+ * include/libbb.h.
  */
 const char bbvar[] ALIGN1 =
                 "BB_OVERRIDE_APPLETS\0" \
                 "BB_SKIP_ANSI_EMULATION\0" \
+                "BB_TERMINAL_MODE\0" \
                 "BB_SYSTEMROOT\0";
 #endif
 const char bb_default_login_shell[] ALIGN1 = LIBBB_DEFAULT_LOGIN_SHELL;
diff --git a/shell/ash.c b/shell/ash.c
index 742067216..d78c6e828 100644
--- a/shell/ash.c
+++ b/shell/ash.c
@@ -2340,9 +2340,9 @@ static void change_realtime(const char *) FAST_FUNC;
 
 #if ENABLE_PLATFORM_MINGW32
 static void FAST_FUNC
-change_skip_ansi(const char *newval UNUSED_PARAM)
+change_terminal_mode(const char *newval UNUSED_PARAM)
 {
-        skip_ansi_emulation(TRUE);
+        terminal_mode(TRUE);
 }
 
 # define LINENO_INDEX (5 + 2 * ENABLE_ASH_MAIL + ENABLE_ASH_GETOPTS)
@@ -2387,7 +2387,8 @@ static const struct {
         { VSTRFIXED|VTEXTFIXED|VUNSET, "HISTFILE"  , NULL            },
 #endif
 #if ENABLE_PLATFORM_MINGW32
-        { VSTRFIXED|VTEXTFIXED|VUNSET, BB_SKIP_ANSI_EMULATION, change_skip_ansi },
+        { VSTRFIXED|VTEXTFIXED|VUNSET, BB_SKIP_ANSI_EMULATION, change_terminal_mode },
+        { VSTRFIXED|VTEXTFIXED|VUNSET, BB_TERMINAL_MODE, change_terminal_mode },
 #endif
 };
 
@@ -14603,7 +14604,7 @@ cmdloop(int top)
                         inter++;
                         chkmail();
 #if ENABLE_PLATFORM_MINGW32
-                        skip_ansi_emulation(TRUE);
+                        terminal_mode(TRUE);
 #endif
                 }
                 n = parsecmd(inter);
diff --git a/win32/winansi.c b/win32/winansi.c
index ef566684e..11a9327e1 100644
--- a/win32/winansi.c
+++ b/win32/winansi.c
@@ -71,22 +71,38 @@ static int is_wine(void)
 #define DISABLE_NEWLINE_AUTO_RETURN 0x0008
 #endif
 
-int skip_ansi_emulation(int reset)
+int terminal_mode(int reset)
 {
-        static int skip = -1;
+        static int mode = -1;
 
-        if (skip < 0 || reset) {
+        if (mode < 0 || reset) {
+                int prefer;
                 HANDLE h;
                 DWORD oldmode, newmode;
-                const char *var = getenv(BB_SKIP_ANSI_EMULATION);
+                const char *term = getenv(BB_TERMINAL_MODE);
+                const char *skip = getenv(BB_SKIP_ANSI_EMULATION);
+
+                if (term) {
+                        mode = atoi(term);
+                } else if (skip) {
+                        mode = atoi(skip);
+                        if (mode == 2)
+                                mode = 5;
+                        else if (mode != 1)
+                                mode = 0;
+                } else {
+                        mode = (getenv("CONEMUPID") != NULL || is_wine()) ? 0 :
+                                                CONFIG_TERMINAL_MODE;
+                }
 
-                if (var) {
-                        skip = atoi(var);
-                        if (skip < 0 || skip > 2)
-                                skip = 0;
+                if (mode < 0 || mode > 7) {
+                        prefer = mode = 0;
+                } else if (mode > 3) {
+                        // Try to get requested mode, fall back to console on failure.
+                        prefer = mode = mode - 4;
                 } else {
-                        skip = (getenv("CONEMUPID") != NULL || is_wine()) ? 0 :
-                                                CONFIG_SKIP_ANSI_EMULATION_DEFAULT;
+                        // Force the requested mode, even if we can't get it.
+                        prefer = 0;
                 }
 
                 if (is_console(STDOUT_FILENO)) {
@@ -94,28 +110,48 @@ int skip_ansi_emulation(int reset)
                         if (GetConsoleMode(h, &oldmode)) {
                                 // Try to recover from mode 0 induced by SSH.
                                 newmode = oldmode == 0 ? 3 : oldmode;
-                                if (skip)
+                                if ((mode & VT_OUTPUT))
                                         newmode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
                                 else
                                         newmode &= ~ENABLE_VIRTUAL_TERMINAL_PROCESSING;
                                 newmode &= ~DISABLE_NEWLINE_AUTO_RETURN;
+
                                 if (newmode != oldmode) {
-                                        if (!SetConsoleMode(h, newmode) && skip == 2)
-                                                skip = 0;
+                                        if (!SetConsoleMode(h, newmode)) {
+                                                if ((prefer & VT_OUTPUT))
+                                                        mode &= ~VT_OUTPUT;
+                                                newmode &= ~ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+                                                SetConsoleMode(h, newmode);
+                                        }
                                 }
                         }
                 }
 
-                // Try to recover from mode 0 induced by SSH.
-                if (reset && is_console_in(STDIN_FILENO)) {
+                if (is_console_in(STDIN_FILENO)) {
                         h = GetStdHandle(STD_INPUT_HANDLE);
-                        if (GetConsoleMode(h, &oldmode) && oldmode == 0) {
-                                SetConsoleMode(h, 0x1f7);
+                        if (GetConsoleMode(h, &oldmode)) {
+                                // Try to recover from mode 0 induced by SSH.
+                                newmode = oldmode == 0 ? 0x1f7 : oldmode;
+                                if ((mode & VT_INPUT))
+                                        newmode |= ENABLE_VIRTUAL_TERMINAL_INPUT;
+                                else
+                                        newmode &= ~ENABLE_VIRTUAL_TERMINAL_INPUT;
+
+                                if (newmode != oldmode) {
+                                        if (!SetConsoleMode(h, newmode)) {
+                                                if ((prefer & VT_INPUT))
+                                                        mode &= ~VT_INPUT;
+                                                // Failure to set the new mode seems to leave
+                                                // the flag set.  Forcibly unset it.
+                                                newmode &= ~ENABLE_VIRTUAL_TERMINAL_INPUT;
+                                                SetConsoleMode(h, newmode);
+                                        }
+                                }
                         }
                 }
         }
 
-        return skip;
+        return mode;
 }
 
 void set_title(const char *str)
@@ -768,7 +804,7 @@ static int ansi_emulate(const char *s, FILE *stream)
 
         while (*pos) {
                 pos = strchr(str, '\033');
-                if (pos && !skip_ansi_emulation(FALSE)) {
+                if (pos && !(terminal_mode(FALSE) & VT_OUTPUT)) {
                         size_t len = pos - str;
 
                         if (len) {
@@ -1037,7 +1073,7 @@ static int ansi_emulate_write(int fd, const void *buf, size_t count)
         /* we've checked the data doesn't contain any NULs */
         while (*pos) {
                 pos = strchr(str, '\033');
-                if (pos && !skip_ansi_emulation(FALSE)) {
+                if (pos && !(terminal_mode(FALSE) & VT_OUTPUT)) {
                         len = pos - str;
 
                         if (len) {