win32: more changes to console/terminal modes

Allowing virtual terminal input mode to be set if available proved
to be unhelpful:  cmd.exe doesn't understand such newfangled stuff
(though PowerShell does).

The allowed values of BB_TERMINAL_MODE are changed to:

   0  Force console mode.
   1  Force virtual terminal mode for output.
   2  Force virtual terminal mode for input.
   3  Force virtual terminal mode for input and output.
   4  Support virtual terminal input if enabled.  Don't alter mode.
   5  Support virtual terminal input if enabled.  Set virtual terminal
      mode for output, if possible.

The default is 5.

2 files changed, 31 insertions, 32 deletions

diff --git a/Config.in b/Config.in
index 11990ed41..6b3d98b37 100644
--- a/Config.in
+++ b/Config.in
@@ -458,24 +458,23 @@ config FEATURE_EURO
 config TERMINAL_MODE
         int "Default setting for terminal mode"
         default 5
-        range 0 7
+        range 0 5
         depends on PLATFORM_MINGW32
         help
-        Control I/O mode of the Windows console/terminal.  Possible
-        values are:
+        Set the default input/output modes of the Windows console/terminal.
+        Possible values are:
 
-        0  Console mode.
-        1  Force terminal mode output.
-        2  Force terminal mode input.
-        3  Force terminal mode input and output.
+        0  Force console mode.
+        1  Force virtual terminal mode for output.
+        2  Force virtual terminal mode for input.
+        3  Force virtual terminal mode for input and output.
 
-        4  Console mode.
-        5  Prefer terminal mode output, fall back to console mode.
-        6  Prefer terminal mode input, fall back to console mode.
-        7  Prefer terminal mode input and output, fall back to console.
+        4  Support virtual terminal input if enabled.  Don't alter mode.
+        5  Support virtual terminal input if enabled.  Set virtual terminal
+           mode for output, if possible.
 
-        Setting the environment variable BB_TERMINAL_MODE overrides this
-        default.
+        Values 0-3 are for testing only.  The default is 5.  The environment
+        variable BB_TERMINAL_MODE overrides this.
 
 config FEATURE_IMPROVED_COLOUR_MAPPING
         bool "More accurate colour mapping for ANSI emulation (0.6 kb)"
diff --git a/win32/winansi.c b/win32/winansi.c
index 11a9327e1..de6db08e2 100644
--- a/win32/winansi.c
+++ b/win32/winansi.c
@@ -76,7 +76,6 @@ int terminal_mode(int reset)
         static int mode = -1;
 
         if (mode < 0 || reset) {
-                int prefer;
                 HANDLE h;
                 DWORD oldmode, newmode;
                 const char *term = getenv(BB_TERMINAL_MODE);
@@ -95,30 +94,26 @@ int terminal_mode(int reset)
                                                 CONFIG_TERMINAL_MODE;
                 }
 
-                if (mode < 0 || mode > 7) {
-                        prefer = mode = 0;
-                } else if (mode > 3) {
-                        // Try to get requested mode, fall back to console on failure.
-                        prefer = mode = mode - 4;
-                } else {
-                        // Force the requested mode, even if we can't get it.
-                        prefer = 0;
-                }
+                if (mode < 0 || mode > 6)
+                        mode = CONFIG_TERMINAL_MODE;
 
                 if (is_console(STDOUT_FILENO)) {
                         h = get_console();
                         if (GetConsoleMode(h, &oldmode)) {
                                 // Try to recover from mode 0 induced by SSH.
                                 newmode = oldmode == 0 ? 3 : oldmode;
-                                if ((mode & VT_OUTPUT))
+
+                                if ((mode & VT_OUTPUT)) {
                                         newmode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
-                                else
+                                } else if (mode < 4) {
                                         newmode &= ~ENABLE_VIRTUAL_TERMINAL_PROCESSING;
-                                newmode &= ~DISABLE_NEWLINE_AUTO_RETURN;
+                                } else if ((oldmode & ENABLE_VIRTUAL_TERMINAL_PROCESSING)) {
+                                        mode |= VT_OUTPUT;
+                                }
 
                                 if (newmode != oldmode) {
                                         if (!SetConsoleMode(h, newmode)) {
-                                                if ((prefer & VT_OUTPUT))
+                                                if (mode >= 4)
                                                         mode &= ~VT_OUTPUT;
                                                 newmode &= ~ENABLE_VIRTUAL_TERMINAL_PROCESSING;
                                                 SetConsoleMode(h, newmode);
@@ -132,14 +127,19 @@ int terminal_mode(int reset)
                         if (GetConsoleMode(h, &oldmode)) {
                                 // Try to recover from mode 0 induced by SSH.
                                 newmode = oldmode == 0 ? 0x1f7 : oldmode;
-                                if ((mode & VT_INPUT))
-                                        newmode |= ENABLE_VIRTUAL_TERMINAL_INPUT;
-                                else
-                                        newmode &= ~ENABLE_VIRTUAL_TERMINAL_INPUT;
+
+                                if (mode < 4) {
+                                        if ((mode & VT_INPUT))
+                                                newmode |= ENABLE_VIRTUAL_TERMINAL_INPUT;
+                                        else
+                                                newmode &= ~ENABLE_VIRTUAL_TERMINAL_INPUT;
+                                } else if ((oldmode & ENABLE_VIRTUAL_TERMINAL_INPUT)) {
+                                        mode |= VT_INPUT;
+                                }
 
                                 if (newmode != oldmode) {
                                         if (!SetConsoleMode(h, newmode)) {
-                                                if ((prefer & VT_INPUT))
+                                                if (mode >= 4)
                                                         mode &= ~VT_INPUT;
                                                 // Failure to set the new mode seems to leave
                                                 // the flag set.  Forcibly unset it.