1 files changed, 163 insertions, 17 deletions

diff --git a/win32/winansi.c b/win32/winansi.c
index f280177e6..c88c096d2 100644
--- a/win32/winansi.c
+++ b/win32/winansi.c
@@ -10,6 +10,9 @@
 static BOOL charToConBuffA(LPSTR s, DWORD len);
 static BOOL charToConA(LPSTR s);
 
+static int conv_fwriteCon(FILE *stream, char *buf, size_t siz);
+static int conv_writeCon(int fd, char *buf, size_t siz);
+
 /*
  Functions to be wrapped:
 */
@@ -814,9 +817,7 @@ static int ansi_emulate(const char *s, FILE *stream)
                         size_t len = pos - str;
 
                         if (len) {
-                                *pos = '\0';    /* NB, '\033' has been overwritten */
-                                charToConA(str);
-                                if (fputs(str, stream) == EOF)
+                                if (conv_fwriteCon(stream, str, len) == EOF)
                                         return EOF;
                                 rv += len;
                         }
@@ -837,9 +838,9 @@ static int ansi_emulate(const char *s, FILE *stream)
                                 return EOF;
 
                 } else {
-                        rv += strlen(str);
-                        charToConA(str);
-                        return fputs(str, stream) == EOF ? EOF : rv;
+                        size_t len = strlen(str);
+                        rv += len;
+                        return conv_fwriteCon(stream, str, len) == EOF ? EOF : rv;
                 }
         }
         return rv;
@@ -853,8 +854,7 @@ int winansi_putchar(int c)
         if (!is_console(STDOUT_FILENO))
                 return putchar(c);
 
-        charToConBuffA(s, 1);
-        return putchar(t) == EOF ? EOF : (unsigned char)c;
+        return conv_fwriteCon(stdout, s, 1) == EOF ? EOF : (unsigned char)c;
 }
 
 int winansi_puts(const char *s)
@@ -952,8 +952,7 @@ int winansi_fputc(int c, FILE *stream)
                 return ret;
         }
 
-        charToConBuffA(s, 1);
-        return fputc(t, stream) == EOF ? EOF : (unsigned char )c;
+        return conv_fwriteCon(stream, s, 1) == EOF ? EOF : (unsigned char )c;
 }
 
 #if !defined(__USE_MINGW_ANSI_STDIO) || !__USE_MINGW_ANSI_STDIO
@@ -1083,8 +1082,7 @@ static int ansi_emulate_write(int fd, const void *buf, size_t count)
                         len = pos - str;
 
                         if (len) {
-                                charToConBuffA(str, len);
-                                out_len = write(fd, str, len);
+                                out_len = conv_writeCon(fd, str, len);
                                 if (out_len == -1)
                                         return -1;
                                 rv += out_len;
@@ -1100,8 +1098,7 @@ static int ansi_emulate_write(int fd, const void *buf, size_t count)
                         pos = str;
                 } else {
                         len = strlen(str);
-                        charToConA(str);
-                        out_len = write(fd, str, len);
+                        out_len = conv_writeCon(fd, str, len);
                         return (out_len == -1) ? -1 : rv+out_len;
                 }
         }
@@ -1442,13 +1439,162 @@ BOOL readConsoleInput_utf8(HANDLE h, INPUT_RECORD *r, DWORD len, DWORD *got)
 }
 #endif
 
+#if ENABLE_FEATURE_UTF8_OUTPUT
+// Write u8buf as if the console output CP is UTF8 - regardless of the CP.
+// fd should be associated with a console output.
+// Return: 0 on successful write[s], else -1 (e.g. if fd is not a console).
+//
+// Up to 3 bytes of an incomplete codepoint may be buffered from prior call[s].
+// All the completed codepoints in one call are written using WriteConsoleW.
+// Bad sequence of any length (till ASCII7 or UTF8 lead) prints 1 subst wchar.
+//
+// note: one console is assumed, and the (3 bytes) buffer is shared regardless
+//       of the original output stream (stdout/err), or even if the handle is
+//       of a different console. This can result in invalid codepoints output
+//       if streams are multiplexed mid-codepoint (same as elsewhere?)
+static int writeCon_utf8(int fd, const char *u8buf, size_t u8siz)
+{
+        static int state = 0;  // -1: bad, 0-3: remaining cp bytes (0: done/new)
+        static uint32_t codepoint = 0;  // accumulated from up to 4 UTF8 bytes
+
+        HANDLE h = (HANDLE)_get_osfhandle(fd);
+        wchar_t wbuf[256];
+        int wlen = 0;
+
+        // ASCII7 uses least logic, then UTF8 continuations, UTF8 lead, errors
+        while (u8siz--) {
+                unsigned char c = *u8buf++;
+                int topbits = 0;
+
+                while (c & (0x80 >> topbits))
+                        ++topbits;
+
+        process_byte:
+                if (state == 0 && topbits == 0) {
+                        // valid ASCII7, state remains 0
+                        codepoint = c;
+
+                } else if (state > 0 && topbits == 1) {
+                        // valid continuation byte
+                        codepoint = (codepoint << 6) | (c & 0x3f);
+                        if (--state)
+                                continue;
+
+                } else if (state == 0 && topbits >= 2 && topbits <= 4) {
+                        // valid UTF8 lead of 2/3/4 bytes codepoint
+                        codepoint = c & (0x7f >> topbits);
+                        state = topbits - 1;  // remaining bytes after lead
+                        continue;
+
+                } else if (state >= 0) {
+                        // invalid byte at state 0/1/2/3, add placeholder once
+                        codepoint = CONFIG_SUBST_WCHAR;
+                        state = -1;
+
+                } else {
+                        // inside bad sequence (placeholder char already added)
+                        if (topbits == 1 || topbits > 4)
+                                continue;  // still bad
+                        // c is valid for state 0, process it with clean slate
+                        state = 0;
+                        goto process_byte;
+                }
+
+                // codepoint is complete
+                // we don't reject surrogate halves, reserved, etc
+                if (codepoint < 0x10000) {
+                        wbuf[wlen++] = codepoint;
+                } else {
+                        // generate a surrogates pair (wbuf has room for 2+)
+                        codepoint -= 0x10000;
+                        wbuf[wlen++] = 0xd800 | (codepoint >> 10);
+                        wbuf[wlen++] = 0xdc00 | (codepoint & 0x3ff);
+                }
+
+                // flush if we have less than two empty spaces
+                if (wlen > ARRAY_SIZE(wbuf) - 2) {
+                        if (!WriteConsoleW(h, wbuf, wlen, 0, 0))
+                                return -1;
+                        wlen = 0;
+                }
+        }
+
+        if (wlen && !WriteConsoleW(h, wbuf, wlen, 0, 0))
+                return -1;
+        return 0;
+}
+#endif
+
 void console_write(const char *str, int len)
 {
         char *buf = xmemdup(str, len);
         int fd = _open("CONOUT$", _O_WRONLY);
-        HANDLE fh = (HANDLE)_get_osfhandle(fd);
-        charToConBuffA(buf, len);
-        WriteConsole(fh, buf, len, NULL, NULL);
+        conv_writeCon(fd, buf, len);
         close(fd);
         free(buf);
 }
+
+// LC_ALL=C disables console output conversion, so that the source
+// data is interpreted only by the console according to its output CP.
+static int conout_conv_enabled(void)
+{
+        static int enabled, tested;  /* = 0 */
+
+        if (!tested) {
+                // keep in sync with [re]init_unicode at libbb/unicode.c
+                char *s = getenv("LC_ALL");
+                if (!s) s = getenv("LC_CTYPE");
+                if (!s) s = getenv("LANG");
+
+                enabled = !(s && s[0] == 'C' && s[1] == 0);
+                tested = 1;
+        }
+
+        return enabled;
+}
+
+// TODO: improvements:
+//
+// 1. currently conv_[f]writeCon modify buf inplace, which means the caller
+// typically has to make a writable copy first just for this.
+// Sometimes it allocates a big copy once, and calls us with substrings.
+// Instead, we could make a writable copy here - it's not used later anyway.
+// To avoid the performance hit of many small allocations, we could use
+// a local buffer for short strings, and allocate only if it doesn't fit
+// (or maybe just reuse the local buffer with substring iterations).
+//
+// 2. Instead of converting from ACP to the console out CP - which guarantees
+// potential data-loss if they differ, we could convert it to wchar_t and
+// write it using WriteConsoleW. This should prevent all output data-loss.
+// care should be taken with DBCS codepages (e.g. 936) or other multi-byte
+// because then converting on arbitrary substring boundaries can fail.
+
+// convert buf inplace from ACP to console out CP and write it to stream
+// returns EOF on error, 0 on success
+static int conv_fwriteCon(FILE *stream, char *buf, size_t siz)
+{
+        if (conout_conv_enabled()) {
+#if ENABLE_FEATURE_UTF8_OUTPUT
+                if (GetConsoleOutputCP() != CP_UTF8)
+                        return writeCon_utf8(fileno(stream), buf, siz) ? EOF : 0;
+#else
+                charToConBuffA(buf, siz);
+#endif
+        }
+        return fwrite(buf, 1, siz, stream) < siz ? EOF : 0;
+}
+
+// similar to above, but using lower level write
+// returns -1 on error, actually-written bytes on suceess
+static int conv_writeCon(int fd, char *buf, size_t siz)
+{
+        if (conout_conv_enabled()) {
+#if ENABLE_FEATURE_UTF8_OUTPUT
+                if (GetConsoleOutputCP() != CP_UTF8)
+                        return writeCon_utf8(fd, buf, siz) ? -1 : siz;
+#else
+                charToConBuffA(buf, siz);
+#endif
+        }
+        return write(fd, buf, siz);
+}