Add support for UTF-8 console and use it for passing smartcab paths

Fixes 7024

6 files changed, 321 insertions, 164 deletions

diff --git a/src/libs/dutil/WixToolset.DUtil/conutil.cpp b/src/libs/dutil/WixToolset.DUtil/conutil.cpp
index 7ca17439..2aec36e5 100644
--- a/src/libs/dutil/WixToolset.DUtil/conutil.cpp
+++ b/src/libs/dutil/WixToolset.DUtil/conutil.cpp
@@ -18,18 +18,33 @@
 #define ConExitOnGdipFailure(g, x, s, ...) ExitOnGdipFailureSource(DUTIL_SOURCE_CONUTIL, g, x, s, __VA_ARGS__)
 
 
+LPCSTR NEWLINE = "\r\n";
+
 static HANDLE vhStdIn = INVALID_HANDLE_VALUE;
 static HANDLE vhStdOut = INVALID_HANDLE_VALUE;
-static BOOL vfConsoleIn = FALSE;
+static BOOL vfStdInInteractive = FALSE;
+static BOOL vfStdOutInteractive = FALSE;
 static BOOL vfConsoleOut = FALSE;
 static CONSOLE_SCREEN_BUFFER_INFO vcsbiInfo;
 
 
+static HRESULT DAPI ReadInteractiveStdIn(
+    __deref_out_z LPWSTR* ppwzBuffer
+);
+static HRESULT ReadRedirectedStdIn(
+    __deref_out_ecount_opt(*pcchSize) LPWSTR* ppwzBuffer,
+    __out DWORD* pcchSize,
+    BOOL fReadLine,
+    DWORD dwMaxRead
+);
+
+
 extern "C" HRESULT DAPI ConsoleInitialize()
 {
     Assert(INVALID_HANDLE_VALUE == vhStdOut);
     HRESULT hr = S_OK;
-    UINT er;
+    DWORD dwStdInMode = 0;
+    DWORD dwStdOutMode = 0;
 
     vhStdIn = ::GetStdHandle(STD_INPUT_HANDLE);
     if (INVALID_HANDLE_VALUE == vhStdIn)
@@ -37,48 +52,24 @@ extern "C" HRESULT DAPI ConsoleInitialize()
         ConExitOnLastError(hr, "failed to open stdin");
     }
 
+    vfStdInInteractive = ::GetFileType(vhStdIn) == FILE_TYPE_CHAR && ::GetConsoleMode(vhStdIn, &dwStdInMode);
+
     vhStdOut = ::GetStdHandle(STD_OUTPUT_HANDLE);
     if (INVALID_HANDLE_VALUE == vhStdOut)
     {
         ConExitOnLastError(hr, "failed to open stdout");
     }
 
-    // check if we have a std in on the console
-    if (::GetConsoleScreenBufferInfo(vhStdIn, &vcsbiInfo))
-    {
-        vfConsoleIn = TRUE;
-    }
-    else
-    {
-        er = ::GetLastError();
-        if (ERROR_INVALID_HANDLE == er)
-        {
-            vfConsoleIn= FALSE;
-            hr = S_OK;
-        }
-        else
-        {
-            ConExitOnWin32Error(er, hr, "failed to get input console screen buffer info");
-        }
-    }
+    vfStdOutInteractive = ::GetFileType(vhStdOut) == FILE_TYPE_CHAR && ::GetConsoleMode(vhStdOut, &dwStdOutMode);
 
     if (::GetConsoleScreenBufferInfo(vhStdOut, &vcsbiInfo))
     {
         vfConsoleOut = TRUE;
     }
-    else   // no console
+
+    if (!::SetConsoleCP(CP_UTF8) || !::SetConsoleOutputCP(CP_UTF8))
     {
-        memset(&vcsbiInfo, 0, sizeof(vcsbiInfo));
-        er = ::GetLastError();
-        if (ERROR_INVALID_HANDLE == er)
-        {
-            vfConsoleOut = FALSE;
-            hr = S_OK;
-        }
-        else
-        {
-            ConExitOnWin32Error(er, hr, "failed to get output console screen buffer info");
-        }
+        ConExitWithLastError(hr, "failed to set console codepage to UTF-8");
     }
 
 LExit:
@@ -162,13 +153,6 @@ extern "C" void DAPI ConsoleNormal()
     }
 }
 
-
-/********************************************************************
- ConsoleWrite - full color printfA without libc
-
- NOTE: use FormatMessage formatting ("%1" or "%1!d!") not plain printf formatting ("%ls" or "%d")
-       assumes already in normal color and resets the screen to normal color
-********************************************************************/
 extern "C" HRESULT DAPI ConsoleWrite(
     CONSOLE_COLOR cc,
     __in_z __format_string LPCSTR szFormat,
@@ -202,7 +186,7 @@ extern "C" HRESULT DAPI ConsoleWrite(
     {
         if (!::WriteFile(vhStdOut, reinterpret_cast<BYTE*>(pszOutput) + cbTotal, cchOutput * sizeof(*pszOutput) - cbTotal, &cbWrote, NULL))
         {
-            ConExitOnLastError(hr, "failed to write output to console: %s", pszOutput);
+            ConExitOnLastError(hr, "failed to write output to console with format: %s", szFormat);
         }
 
         cbTotal += cbWrote;
@@ -220,12 +204,53 @@ LExit:
 }
 
 
-/********************************************************************
- ConsoleWriteLine - full color printfA plus newline without libc
+extern "C" HRESULT DAPI ConsoleWriteW(
+    __in CONSOLE_COLOR cc,
+    __in_z LPCWSTR wzData
+)
+{
+    AssertSz(INVALID_HANDLE_VALUE != vhStdOut, "ConsoleInitialize() has not been called");
+    HRESULT hr = S_OK;
+    LPSTR pszOutput = NULL;
+    DWORD cchOutput = 0;
+    DWORD cbWrote = 0;
+    DWORD cbTotal = 0;
+
+    // set the color
+    switch (cc)
+    {
+    case CONSOLE_COLOR_NORMAL: break;   // do nothing
+    case CONSOLE_COLOR_RED: ConsoleRed(); break;
+    case CONSOLE_COLOR_YELLOW: ConsoleYellow(); break;
+    case CONSOLE_COLOR_GREEN: ConsoleGreen(); break;
+    }
+
+    hr = StrAnsiAllocString(&pszOutput, wzData, 0, CP_UTF8);
+    ExitOnFailure(hr, "failed to convert console output to utf-8: %ls", wzData);
+
+    cchOutput = lstrlenA(pszOutput);
+    while (cbTotal < (sizeof(*pszOutput) * cchOutput))
+    {
+        if (!::WriteFile(vhStdOut, reinterpret_cast<BYTE*>(pszOutput) + cbTotal, cchOutput * sizeof(*pszOutput) - cbTotal, &cbWrote, NULL))
+        {
+            ConExitOnLastError(hr, "failed to write output to console with format: %ls", wzData);
+        }
+
+        cbTotal += cbWrote;
+    }
+
+    // reset the color to normal
+    if (CONSOLE_COLOR_NORMAL != cc)
+    {
+        ConsoleNormal();
+    }
+
+LExit:
+    ReleaseStr(pszOutput);
+    return hr;
+}
+
 
- NOTE: use FormatMessage formatting ("%1" or "%1!d!") not plain printf formatting ("%ls" or "%d")
-       assumes already in normal color and resets the screen to normal color
-********************************************************************/
 extern "C" HRESULT DAPI ConsoleWriteLine(
     CONSOLE_COLOR cc,
     __in_z __format_string LPCSTR szFormat,
@@ -238,7 +263,6 @@ extern "C" HRESULT DAPI ConsoleWriteLine(
     DWORD cchOutput = 0;
     DWORD cbWrote = 0;
     DWORD cbTotal = 0;
-    LPCSTR szNewLine = "\r\n";
 
     // set the color
     switch (cc)
@@ -270,7 +294,7 @@ extern "C" HRESULT DAPI ConsoleWriteLine(
     //
     // write the newline
     //
-    if (!::WriteFile(vhStdOut, reinterpret_cast<const BYTE*>(szNewLine), 2, &cbWrote, NULL))
+    if (!::WriteFile(vhStdOut, reinterpret_cast<const BYTE*>(NEWLINE), 2, &cbWrote, NULL))
     {
         ConExitOnLastError(hr, "failed to write newline to console");
     }
@@ -287,11 +311,6 @@ LExit:
 }
 
 
-/********************************************************************
- ConsoleWriteError - display an error to the screen
-
- NOTE: use FormatMessage formatting ("%1" or "%1!d!") not plain printf formatting ("%s" or "%d")
-********************************************************************/
 HRESULT ConsoleWriteError(
     HRESULT hrError,
     CONSOLE_COLOR cc,
@@ -323,11 +342,6 @@ LExit:
 }
 
 
-/********************************************************************
- ConsoleReadW - get console input without libc
-
- NOTE: only supports reading ANSI characters
-********************************************************************/
 extern "C" HRESULT DAPI ConsoleReadW(
     __deref_out_z LPWSTR* ppwzBuffer
     )
@@ -336,57 +350,24 @@ extern "C" HRESULT DAPI ConsoleReadW(
     Assert(ppwzBuffer);
 
     HRESULT hr = S_OK;
-    LPSTR psz = NULL;
-    DWORD cch = 0;
-    DWORD cchRead = 0;
-    DWORD cchTotalRead = 0;
+    DWORD cchSize = 0;
 
-    cch  = 64;
-    hr = StrAnsiAlloc(&psz, cch);
-    ConExitOnFailure(hr, "failed to allocate memory to read from console");
-
-    // loop until we read the \r\n from the console
-    for (;;)
+    if (vfStdInInteractive)
     {
-        // read one character at a time, since that seems to be the only way to make this work
-        if (!::ReadFile(vhStdIn, psz + cchTotalRead, 1, &cchRead, NULL))
-            ConExitOnLastError(hr, "failed to read string from console");
-
-        cchTotalRead += cchRead;
-        if (1 < cchTotalRead && '\r' == psz[cchTotalRead - 2] && '\n' == psz[cchTotalRead - 1])
-        {
-            psz[cchTotalRead - 2] = '\0';  // chop off the \r\n
-            break;
-        }
-        else if (0 == cchRead)  // nothing more was read
-        {
-            psz[cchTotalRead] = '\0';  // null termintate and bail
-            break;
-        }
-
-        if (cchTotalRead == cch)
-        {
-            cch *= 2;   // double everytime we run out of space
-            hr = StrAnsiAlloc(&psz, cch);
-            ConExitOnFailure(hr, "failed to allocate memory to read from console");
-        }
+        hr = ReadInteractiveStdIn(ppwzBuffer);
+        ExitOnFailure(hr, "failed to read from interactive console");
+    }
+    else
+    {
+        hr = ReadRedirectedStdIn(ppwzBuffer, &cchSize, TRUE, 0);
+        ExitOnFailure(hr, "failed to read from redirected console");
     }
-
-    hr = StrAllocStringAnsi(ppwzBuffer, psz, 0, CP_ACP);
 
 LExit:
-    ReleaseStr(psz);
     return hr;
 }
 
 
-/********************************************************************
- ConsoleReadNonBlockingW - Read from the console without blocking
- Won't work for redirected files (exe < txtfile), but will work for stdin redirected to 
- an anonymous or named pipe
-
- if (fReadLine), stop reading immediately when \r\n is found
-*********************************************************************/
 extern "C" HRESULT DAPI ConsoleReadNonBlockingW(
     __deref_out_ecount_opt(*pcchSize) LPWSTR* ppwzBuffer,
     __out DWORD* pcchSize,
@@ -406,9 +387,6 @@ extern "C" HRESULT DAPI ConsoleReadNonBlockingW(
     DWORD cchTotal = 0;
     DWORD cch = 8;
 
-    DWORD cchRead = 0;
-    DWORD cchTotalRead = 0;
-    
     DWORD dwIndex = 0;
     DWORD er;
 
@@ -476,45 +454,13 @@ extern "C" HRESULT DAPI ConsoleReadNonBlockingW(
     {
         // otherwise, the peek worked, and we have the end of a pipe
         if (0 == dwRead)
-            ExitFunction1(hr = S_FALSE);
-
-        cch = 8;
-        hr = StrAnsiAlloc(&psz, cch);
-        ConExitOnFailure(hr, "failed to allocate memory to read from console");
-
-        for (/*dwRead from PeekNamedPipe*/; dwRead > 0; dwRead--)
         {
-            // read one character at a time, since that seems to be the only way to make this work
-            if (!::ReadFile(vhStdIn, psz + cchTotalRead, 1, &cchRead, NULL))
-            {
-                ConExitOnLastError(hr, "failed to read string from console");
-            }
-
-            cchTotalRead += cchRead;
-            if (fReadLine && '\r' == psz[cchTotalRead - 1] && '\n' == psz[cchTotalRead])
-            {
-                psz[cchTotalRead - 1] = '\0';  // chop off the \r\n
-                cchTotalRead -= 1;
-                break;
-            }
-            else if (0 == cchRead)  // nothing more was read
-            {
-                psz[cchTotalRead] = '\0';  // null termintate and bail
-                break;
-            }
-
-            if (cchTotalRead == cch)
-            {
-                cch *= 2;   // double everytime we run out of space
-                hr = StrAnsiAlloc(&psz, cch);
-                ConExitOnFailure(hr, "failed to allocate memory to read from console");
-            }
+            ExitFunction1(hr = S_FALSE);
         }
 
-        *pcchSize = cchTotalRead;
-        hr = StrAllocStringAnsi(ppwzBuffer, psz, cchTotalRead, CP_ACP);
+        hr = ReadRedirectedStdIn(ppwzBuffer, pcchSize, fReadLine, dwRead);
     }
-    
+
 LExit:
     ReleaseStr(psz);
 
@@ -522,10 +468,6 @@ LExit:
 }
 
 
-/********************************************************************
- ConsoleReadStringA - get console input without libc
-
-*********************************************************************/
 extern "C" HRESULT DAPI ConsoleReadStringA(
     __deref_inout_ecount_part(cchCharBuffer,*pcchNumCharReturn) LPSTR* ppszCharBuffer,
     CONST DWORD cchCharBuffer,
@@ -543,16 +485,18 @@ extern "C" HRESULT DAPI ConsoleReadStringA(
             do
             {
                 hr = StrAnsiAlloc(ppszCharBuffer, cchCharBuffer * iRead);
-                ConExitOnFailure(hr, "failed to allocate memory for ConsoleReadStringW");
+                ConExitOnFailure(hr, "failed to allocate memory for ConsoleReadStringA");
+
                 // ReadConsoleW will not return until <Return>, the last two chars are 13 and 10.
                 if (!::ReadConsoleA(vhStdIn, *ppszCharBuffer + iReadCharTotal, cchCharBuffer, pcchNumCharReturn, NULL) || *pcchNumCharReturn == 0)
                 {
                     ConExitOnLastError(hr, "failed to read string from console");
                 }
+
                 iReadCharTotal += *pcchNumCharReturn;
                 iRead += 1;
-            }
-            while((*ppszCharBuffer)[iReadCharTotal - 1] != 10 || (*ppszCharBuffer)[iReadCharTotal - 2] != 13);
+            } while((*ppszCharBuffer)[iReadCharTotal - 1] != 10 || (*ppszCharBuffer)[iReadCharTotal - 2] != 13);
+
             *pcchNumCharReturn = iReadCharTotal;
         }
         else
@@ -562,6 +506,7 @@ extern "C" HRESULT DAPI ConsoleReadStringA(
             {
                 ConExitOnLastError(hr, "failed to read string from console");
             }
+
             if ((*ppszCharBuffer)[*pcchNumCharReturn - 1] != 10 ||
                 (*ppszCharBuffer)[*pcchNumCharReturn - 2] != 13)
             {
@@ -579,18 +524,16 @@ LExit:
     return hr;
 }
 
-/********************************************************************
- ConsoleReadStringW - get console input without libc
 
-*********************************************************************/
 extern "C" HRESULT DAPI ConsoleReadStringW(
     __deref_inout_ecount_part(cchCharBuffer,*pcchNumCharReturn) LPWSTR* ppwzCharBuffer,
-    const DWORD cchCharBuffer,
+    CONST DWORD cchCharBuffer,
     __out DWORD* pcchNumCharReturn
     )
 {
     AssertSz(INVALID_HANDLE_VALUE != vhStdIn, "ConsoleInitialize() has not been called");
     HRESULT hr = S_OK;
+
     if (ppwzCharBuffer && (pcchNumCharReturn || cchCharBuffer < 2))
     {
         DWORD iRead = 1;
@@ -601,15 +544,17 @@ extern "C" HRESULT DAPI ConsoleReadStringW(
             {
                 hr = StrAlloc(ppwzCharBuffer, cchCharBuffer * iRead);
                 ConExitOnFailure(hr, "failed to allocate memory for ConsoleReadStringW");
+
                 // ReadConsoleW will not return until <Return>, the last two chars are 13 and 10.
                 if (!::ReadConsoleW(vhStdIn, *ppwzCharBuffer + iReadCharTotal, cchCharBuffer, pcchNumCharReturn, NULL) || *pcchNumCharReturn == 0)
                 {
                     ConExitOnLastError(hr, "failed to read string from console");
                 }
+
                 iReadCharTotal += *pcchNumCharReturn;
                 iRead += 1;
-            }
-            while((*ppwzCharBuffer)[iReadCharTotal - 1] != 10 || (*ppwzCharBuffer)[iReadCharTotal - 2] != 13);
+            } while((*ppwzCharBuffer)[iReadCharTotal - 1] != 10 || (*ppwzCharBuffer)[iReadCharTotal - 2] != 13);
+
             *pcchNumCharReturn = iReadCharTotal;
         }
         else
@@ -619,6 +564,7 @@ extern "C" HRESULT DAPI ConsoleReadStringW(
             {
                 ConExitOnLastError(hr, "failed to read string from console");
             }
+
             if ((*ppwzCharBuffer)[*pcchNumCharReturn - 1] != 10 ||
                 (*ppwzCharBuffer)[*pcchNumCharReturn - 2] != 13)
             {
@@ -636,14 +582,12 @@ LExit:
     return hr;
 }
 
-/********************************************************************
- ConsoleSetReadHidden - set console input no echo
 
-*********************************************************************/
 extern "C" HRESULT DAPI ConsoleSetReadHidden(void)
 {
     AssertSz(INVALID_HANDLE_VALUE != vhStdIn, "ConsoleInitialize() has not been called");
     HRESULT hr = S_OK;
+
     ::FlushConsoleInputBuffer(vhStdIn);
     if (!::SetConsoleMode(vhStdIn, ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT))
     {
@@ -654,14 +598,12 @@ LExit:
     return hr;
 }
 
-/********************************************************************
- ConsoleSetReadNormal - reset to echo
 
-*********************************************************************/
 extern "C" HRESULT DAPI ConsoleSetReadNormal(void)
 {
     AssertSz(INVALID_HANDLE_VALUE != vhStdIn, "ConsoleInitialize() has not been called");
     HRESULT hr = S_OK;
+
     if (!::SetConsoleMode(vhStdIn, ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT | ENABLE_MOUSE_INPUT))
     {
         ConExitOnLastError(hr, "failed to set console input mode to be normal");
@@ -671,3 +613,123 @@ LExit:
     return hr;
 }
 
+
+static HRESULT DAPI ReadInteractiveStdIn(
+    __deref_out_z LPWSTR* ppwzBuffer
+)
+{
+    HRESULT hr = S_OK;
+    LPWSTR pwz = NULL;
+    DWORD cch = 8;
+    DWORD cchRead = 0;
+    DWORD cchTotalRead = 0;
+
+    hr = StrAlloc(&pwz, cch);
+    ConExitOnFailure(hr, "failed to allocate memory to read from console");
+
+    // loop until we read the \r\n from the console
+    for (;;)
+    {
+        // read one character at a time, since that seems to be the only way to make this work
+        if (!::ReadConsoleW(vhStdIn, pwz + cchTotalRead, 1, &cchRead, NULL))
+        {
+            ConExitOnLastError(hr, "failed to read string from console");
+        }
+
+        cchTotalRead += cchRead;
+        if (0 == cchRead)  // nothing more was read
+        {
+            pwz[cchTotalRead] = L'\0';  // null terminate and bail
+            break;
+        }
+        else if (0 < cchTotalRead && L'\n' == pwz[cchTotalRead - 1])
+        {
+            if (1 < cchTotalRead && L'\r' == pwz[cchTotalRead - 2])
+            {
+                pwz[cchTotalRead - 2] = L'\0';  // chop off the \r\n
+            }
+            else
+            {
+                pwz[cchTotalRead - 1] = L'\0';  // chop off the \n
+            }
+
+            break;
+        }
+
+        if (cchTotalRead == cch)
+        {
+            cch *= 2;   // double everytime we run out of space
+            hr = StrAlloc(&pwz, cch);
+            ConExitOnFailure(hr, "failed to allocate memory to read from console");
+        }
+    }
+
+    hr = StrAllocString(ppwzBuffer, pwz, 0);
+    ConExitOnFailure(hr, "failed to copy stdin buffer to return buffer");
+
+LExit:
+    ReleaseStr(pwz);
+    return hr;
+}
+
+
+static HRESULT ReadRedirectedStdIn(
+    __deref_out_ecount_opt(*pcchSize) LPWSTR* ppwzBuffer,
+    __out DWORD* pcchSize,
+    BOOL fReadLine,
+    DWORD dwMaxRead
+)
+{
+    HRESULT hr = S_OK;
+    LPSTR psz = NULL;
+    DWORD cch = 8;
+    DWORD cchRead = 0;
+    DWORD cchTotalRead = 0;
+
+    hr = StrAnsiAlloc(&psz, cch);
+    ConExitOnFailure(hr, "failed to allocate memory to read from console");
+
+    while (dwMaxRead == 0 || cchTotalRead < dwMaxRead)
+    {
+        // read one character at a time, since that seems to be the only way to make this work
+        if (!::ReadFile(vhStdIn, psz + cchTotalRead, 1, &cchRead, NULL))
+        {
+            ConExitOnLastError(hr, "failed to read string from console");
+        }
+
+        cchTotalRead += cchRead;
+        if (0 == cchRead)  // nothing more was read
+        {
+            psz[cchTotalRead] = '\0';  // null terminate and bail
+            break;
+        }
+        else if (fReadLine && 0 < cchTotalRead && '\n' == psz[cchTotalRead - 1])
+        {
+            if (1 < cchTotalRead && '\r' == psz[cchTotalRead - 2])
+            {
+                psz[cchTotalRead - 2] = '\0';  // chop off the \r\n
+            }
+            else
+            {
+                psz[cchTotalRead - 1] = '\0';  // chop off the \n
+            }
+
+            break;
+        }
+
+        if (cchTotalRead == cch)
+        {
+            cch *= 2;   // double everytime we run out of space
+            hr = StrAnsiAlloc(&psz, cch);
+            ConExitOnFailure(hr, "failed to allocate memory to read from console");
+        }
+    }
+
+    *pcchSize = cchTotalRead;
+
+    hr = StrAllocStringAnsi(ppwzBuffer, psz, 0, CP_UTF8);
+    ConExitOnFailure(hr, "failed to convert console data");
+
+LExit:
+    return hr;
+}
diff --git a/src/libs/dutil/WixToolset.DUtil/inc/conutil.h b/src/libs/dutil/WixToolset.DUtil/inc/conutil.h
index 38aaea84..92e7f600 100644
--- a/src/libs/dutil/WixToolset.DUtil/inc/conutil.h
+++ b/src/libs/dutil/WixToolset.DUtil/inc/conutil.h
@@ -33,16 +33,46 @@ void DAPI ConsoleRed();
 void DAPI ConsoleYellow();
 void DAPI ConsoleNormal();
 
+/********************************************************************
+ ConsoleWrite - full color printfA without libc
+
+ NOTE: only supports ANSI characters
+       assumes already in normal color and resets the screen to normal color
+********************************************************************/
 HRESULT DAPI ConsoleWrite(
     CONSOLE_COLOR cc,
     __in_z __format_string LPCSTR szFormat,
     ...
     );
+
+/********************************************************************
+ ConsoleWriteW - sends UTF-8 characters to console out in color.
+
+ NOTE: assumes already in normal color and resets the screen to normal color
+********************************************************************/
+HRESULT DAPI ConsoleWriteW(
+    __in CONSOLE_COLOR cc,
+    __in_z LPCWSTR wzData
+    );
+
+/********************************************************************
+ ConsoleWriteLine - full color printfA plus newline without libc
+
+ NOTE: only supports ANSI characters
+       assumes already in normal color and resets the screen to normal color
+********************************************************************/
 HRESULT DAPI ConsoleWriteLine(
     CONSOLE_COLOR cc,
     __in_z __format_string LPCSTR szFormat,
     ...
     );
+
+/********************************************************************
+ ConsoleWriteError - display an error to the console out
+
+ NOTE: only supports ANSI characters
+       does not write to stderr
+********************************************************************/
 HRESULT DAPI ConsoleWriteError(
     HRESULT hrError,
     CONSOLE_COLOR cc,
@@ -50,28 +80,58 @@ HRESULT DAPI ConsoleWriteError(
     ...
     );
 
+/********************************************************************
+ ConsoleReadW - reads a line from console in as UTF-8 to populate Unicode buffer
+
+********************************************************************/
 HRESULT DAPI ConsoleReadW(
     __deref_out_z LPWSTR* ppwzBuffer
     );
 
+/********************************************************************
+ ConsoleReadNonBlockingW - Read from the console without blocking
+ Won't work for redirected files (exe < txtfile), but will work for stdin redirected to
+ an anonymous or named pipe
+
+ if (fReadLine), stop reading immediately when \r\n is found
+*********************************************************************/
+HRESULT DAPI ConsoleReadNonBlockingW(
+    __deref_out_ecount_opt(*pcchSize) LPWSTR* ppwzBuffer,
+    __out DWORD* pcchSize,
+    BOOL fReadLine
+    );
+
+/********************************************************************
+ ConsoleReadStringA - get console input without libc
+
+ NOTE: only supports ANSI characters
+*********************************************************************/
 HRESULT DAPI ConsoleReadStringA(
     __deref_inout_ecount_part(cchCharBuffer,*pcchNumCharReturn) LPSTR* szCharBuffer,
     CONST DWORD cchCharBuffer,
     __out DWORD* pcchNumCharReturn
     );
+
+/********************************************************************
+ ConsoleReadStringW - get console input without libc
+
+*********************************************************************/
 HRESULT DAPI ConsoleReadStringW(
     __deref_inout_ecount_part(cchCharBuffer,*pcchNumCharReturn) LPWSTR* szCharBuffer,
     CONST DWORD cchCharBuffer,
     __out DWORD* pcchNumCharReturn
     );
 
-HRESULT DAPI ConsoleReadNonBlockingW(
-    __deref_out_ecount_opt(*pcchSize) LPWSTR* ppwzBuffer,
-    __out DWORD* pcchSize,
-    BOOL fReadLine
-    );
+/********************************************************************
+ ConsoleSetReadHidden - set console input no echo
 
+*********************************************************************/
 HRESULT DAPI ConsoleSetReadHidden(void);
+
+/********************************************************************
+ ConsoleSetReadNormal - reset to echo
+
+*********************************************************************/
 HRESULT DAPI ConsoleSetReadNormal(void);
 
 #ifdef __cplusplus
diff --git a/src/wix/WixToolset.Core.Native/WixNativeExe.cs b/src/wix/WixToolset.Core.Native/WixNativeExe.cs
index b3b248c0..8b75f508 100644
--- a/src/wix/WixToolset.Core.Native/WixNativeExe.cs
+++ b/src/wix/WixToolset.Core.Native/WixNativeExe.cs
@@ -6,6 +6,7 @@ namespace WixToolset.Core.Native
     using System.Collections.Generic;
     using System.Diagnostics;
     using System.IO;
+    using System.Text;
 
     internal class WixNativeExe
     {
@@ -40,6 +41,7 @@ namespace WixToolset.Core.Native
                 RedirectStandardInput = true,
                 RedirectStandardOutput = true,
                 RedirectStandardError = true,
+                StandardOutputEncoding = Encoding.UTF8,
                 CreateNoWindow = true,
                 ErrorDialog = false,
                 UseShellExecute = false
@@ -61,7 +63,8 @@ namespace WixToolset.Core.Native
                 {
                     foreach (var line in this.stdinLines)
                     {
-                        process.StandardInput.WriteLine(line);
+                        var bytes = Encoding.UTF8.GetBytes(line + Environment.NewLine);
+                        process.StandardInput.BaseStream.Write(bytes, 0, bytes.Length);
                     }
 
                     // Trailing blank line indicates stdin complete.
diff --git a/src/wix/test/WixToolsetTest.Core.Native/CabinetFixture.cs b/src/wix/test/WixToolsetTest.Core.Native/CabinetFixture.cs
index 56bc8fb4..c15d2c74 100644
--- a/src/wix/test/WixToolsetTest.Core.Native/CabinetFixture.cs
+++ b/src/wix/test/WixToolsetTest.Core.Native/CabinetFixture.cs
@@ -34,6 +34,27 @@ namespace WixToolsetTest.CoreNative
         }
 
         [Fact]
+        public void CanCreateCabinetUsingLocalizedPath()
+        {
+            using (var fs = new DisposableFileSystem())
+            {
+                var intermediateFolder = fs.GetFolder(true);
+                var cabPath = Path.Combine(intermediateFolder, "testout.cab");
+
+                var files = new[] { new CabinetCompressFile(TestData.Get(@"TestData", "Testüber", "t道場t.txt"), "test.txt") };
+
+                var cabinet = new Cabinet(cabPath);
+                var created = cabinet.Compress(files, CompressionLevel.Low);
+
+                Assert.True(File.Exists(cabPath));
+                Assert.Equal(new[]
+                {
+                    "testout.cab, test.txt"
+                }, created.Select(c => String.Join(", ", c.CabinetName, c.FirstFileToken)).ToArray());
+            }
+        }
+
+        [Fact]
         public void CanCreateSpannedFileCabinet()
         {
             using (var fs = new DisposableFileSystem())
diff --git a/src/wix/test/WixToolsetTest.Core.Native/TestData/Testüber/t道場t.txt b/src/wix/test/WixToolsetTest.Core.Native/TestData/Testüber/t道場t.txt
new file mode 100644
index 00000000..cd0db0e1
--- /dev/null
+++ b/src/wix/test/WixToolsetTest.Core.Native/TestData/Testüber/t道場t.txt
@@ -0,0 +1 @@
+This is test.txt.
\ No newline at end of file
diff --git a/src/wix/wixnative/smartcab.cpp b/src/wix/wixnative/smartcab.cpp
index 9c441a34..1b925d42 100644
--- a/src/wix/wixnative/smartcab.cpp
+++ b/src/wix/wixnative/smartcab.cpp
@@ -123,7 +123,7 @@ static HRESULT CompressFiles(
         hr = StrSplitAllocArray(&rgsczSplit, &cSplit, sczLine, L"\t");
         ConsoleExitOnFailure(hr, CONSOLE_COLOR_RED, "failed to split smartcab line from stdin: %ls", sczLine);
 
-        if (cSplit != 2 && cSplit != 6)
+        if (!rgsczSplit || (cSplit != 2 && cSplit != 6))
         {
             hr = E_INVALIDARG;
             ConsoleExitOnFailure(hr, CONSOLE_COLOR_RED, "failed to split smartcab line into hash x 4, token, source file: %ls", sczLine);
@@ -176,5 +176,15 @@ static void __stdcall CabNamesCallback(
     __in_z LPCWSTR wzFileToken
     )
 {
-    ConsoleWriteLine(CONSOLE_COLOR_NORMAL, "%ls\t%ls\t%ls", wzFirstCabName, wzNewCabName, wzFileToken);
+    HRESULT hr;
+    LPWSTR scz = NULL;
+
+    hr = StrAllocFormatted(&scz, L"%s\t%s\t%s\r\n", wzFirstCabName, wzNewCabName, wzFileToken);
+    ConsoleExitOnFailure(hr, CONSOLE_COLOR_RED, "failed to allocate cabinet names message");
+
+    hr = ConsoleWriteW(CONSOLE_COLOR_NORMAL, scz);
+    ConsoleExitOnFailure(hr, CONSOLE_COLOR_RED, "failed to send cabinet names message");
+
+LExit:
+    ReleaseStr(scz);
 }