winansi: more fine-grained control of ANSI emulation

Change the meaning of the BB_SKIP_ANSI_EMULATION variable:

- if it isn't set or if it has any value other than 1 or 2, use
  ANSI emulation;

- if it's set to 1, always emit ANSI escape codes;

- if it's set to 2, attempt to enable virtual terminal processing in
  the current console.  If that works emit ANSI escape codes, if not
  revert to ANSI emulation.

On all platforms I've tested (Windows 10, Windows 8.1, ReactOS 0.4.13,
plus ConEmu) BB_SKIP_ANSI_EMULATION=2 does the right thing.

1 files changed, 34 insertions, 2 deletions

diff --git a/win32/winansi.c b/win32/winansi.c
index c4c7ee87c..dd99b42af 100644
--- a/win32/winansi.c
+++ b/win32/winansi.c
@@ -53,12 +53,44 @@ static int is_console_in(int fd)
         return isatty(fd) && GetStdHandle(STD_INPUT_HANDLE) != INVALID_HANDLE_VALUE;
 }
 
+#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
+#define DISABLE_NEWLINE_AUTO_RETURN 0x0008
+
 int skip_ansi_emulation(int reset)
 {
         static int skip = -1;
 
-        if (skip < 0 || reset)
-                skip = getenv(bb_skip_ansi_emulation) != NULL;
+        if (skip < 0 || reset) {
+                const char *var = getenv(bb_skip_ansi_emulation);
+                skip = var != NULL;
+                if (skip) {
+                        switch (xatou(var)) {
+                        case 1:
+                                break;
+                        case 2:
+                                skip = 2;
+                                break;
+                        default:
+                                skip = 0;
+                                break;
+                        }
+                }
+
+                if (is_console(STDOUT_FILENO)) {
+                        HANDLE h = get_console();
+                        DWORD mode;
+
+                        if (GetConsoleMode(h, &mode)) {
+                                if (skip)
+                                        mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+                                else
+                                        mode &= ~ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+                                mode &= ~DISABLE_NEWLINE_AUTO_RETURN;
+                                if (!SetConsoleMode(h, mode) && skip == 2)
+                                        skip = 0;
+                        }
+                }
+        }
 
         return skip;
 }