Handle unexpected BOM when reading from stdin in wixnative.exe

In certain cases, the encoding of stdin/stdout can be changed in
Windows to cause a BOM to be added on the first read of stdin.
To handle the unexpected BOM, the first line read from stdin is
now a "preamble" where the last character is expected to be a ":".
We use the last character because a multibyte BOM value may appear
before it.

Also updated SAL annotations to resolve IDE warnings.

7 files changed, 57 insertions, 29 deletions

diff --git a/src/wix/WixToolset.Core.Native/Cabinet.cs b/src/wix/WixToolset.Core.Native/Cabinet.cs
index 9b77bd37..e383d8fa 100644
--- a/src/wix/WixToolset.Core.Native/Cabinet.cs
+++ b/src/wix/WixToolset.Core.Native/Cabinet.cs
@@ -57,19 +57,6 @@ namespace WixToolset.Core.Native
             }
 
             wixnative.Run();
-
-#if TOOD_ERROR_HANDLING
-            catch (COMException ce)
-            {
-                // If we get a "the file exists" error, we must have a full temp directory - so report the issue
-                if (0x80070050 == unchecked((uint)ce.ErrorCode))
-                {
-                    throw new WixException(WixErrors.FullTempDirectory("WSC", Path.GetTempPath()));
-                }
-
-                throw;
-            }
-#endif
         }
 
         /// <summary>
diff --git a/src/wix/WixToolset.Core.Native/WixNativeExe.cs b/src/wix/WixToolset.Core.Native/WixNativeExe.cs
index 6e8d64ca..13d7a4ea 100644
--- a/src/wix/WixToolset.Core.Native/WixNativeExe.cs
+++ b/src/wix/WixToolset.Core.Native/WixNativeExe.cs
@@ -4,7 +4,6 @@ namespace WixToolset.Core.Native
 {
     using System;
     using System.Collections.Generic;
-    using System.ComponentModel;
     using System.Diagnostics;
     using System.IO;
 
@@ -54,6 +53,9 @@ namespace WixToolset.Core.Native
                 process.BeginOutputReadLine();
                 process.BeginErrorReadLine();
 
+                // Send the stdin preamble.
+                process.StandardInput.WriteLine(":");
+
                 if (this.stdinLines.Count > 0)
                 {
                     foreach (var line in this.stdinLines)
diff --git a/src/wix/wixnative/extractcab.cpp b/src/wix/wixnative/extractcab.cpp
index 53f53266..2de37992 100644
--- a/src/wix/wixnative/extractcab.cpp
+++ b/src/wix/wixnative/extractcab.cpp
@@ -2,12 +2,12 @@
 
 #include "precomp.h"
 
-static HRESULT ProgressCallback(BOOL fBeginFile, LPCWSTR wzFileId, LPVOID pvContext);
+static HRESULT ProgressCallback(__in BOOL fBeginFile, __in_z LPCWSTR wzFileId, __in_opt LPVOID pvContext);
 
 
 HRESULT ExtractCabCommand(
     __in int argc,
-    __in LPWSTR argv[]
+    __in_ecount(argc) LPWSTR argv[]
 )
 {
     HRESULT hr = E_INVALIDARG;
@@ -37,8 +37,8 @@ LExit:
 
 static HRESULT ProgressCallback(
     __in BOOL fBeginFile,
-    __in LPCWSTR wzFileId,
-    __in LPVOID /*pvContext*/
+    __in_z LPCWSTR wzFileId,
+    __in_opt LPVOID /*pvContext*/
 )
 {
     if (fBeginFile)
diff --git a/src/wix/wixnative/precomp.h b/src/wix/wixnative/precomp.h
index 5bd617e5..490bbdcf 100644
--- a/src/wix/wixnative/precomp.h
+++ b/src/wix/wixnative/precomp.h
@@ -4,6 +4,7 @@
 #include <windows.h>
 #include <aclapi.h>
 #include <mergemod.h>
+#include <strsafe.h>
 
 #include "dutil.h"
 #include "conutil.h"
@@ -13,7 +14,8 @@
 #include "cabcutil.h"
 #include "cabutil.h"
 
-HRESULT SmartCabCommand(int argc, LPWSTR argv[]);
-HRESULT ResetAclsCommand(int argc, LPWSTR argv[]);
-HRESULT EnumCabCommand(int argc, LPWSTR argv[]);
-HRESULT ExtractCabCommand(int argc, LPWSTR argv[]);
+HRESULT WixNativeReadStdinPreamble();
+HRESULT SmartCabCommand(__in int argc, __in_ecount(argc) LPWSTR argv[]);
+HRESULT ResetAclsCommand(__in int argc, __in_ecount(argc) LPWSTR argv[]);
+HRESULT EnumCabCommand(__in int argc, __in_ecount(argc) LPWSTR argv[]);
+HRESULT ExtractCabCommand(__in int argc, __in_ecount(argc) LPWSTR argv[]);
diff --git a/src/wix/wixnative/resetacls.cpp b/src/wix/wixnative/resetacls.cpp
index 8c5bdc56..91bc633d 100644
--- a/src/wix/wixnative/resetacls.cpp
+++ b/src/wix/wixnative/resetacls.cpp
@@ -2,7 +2,9 @@
 
 #include "precomp.h"
 
-HRESULT ResetAclsCommand(int argc, LPWSTR argv[])
+HRESULT ResetAclsCommand(
+    __in int argc,
+    __in_ecount(argc) LPWSTR argv[])
 {
     Unused(argc);
     Unused(argv);
@@ -24,6 +26,9 @@ HRESULT ResetAclsCommand(int argc, LPWSTR argv[])
         ConsoleExitOnLastError(hr, CONSOLE_COLOR_RED, "failed to initialize ACL");
     }
 
+    hr = WixNativeReadStdinPreamble();
+    ExitOnFailure(hr, "failed to read stdin preamble before resetting ACLs");
+
     // Reset the existing security permissions on each provided file.
     for (;;)
     {
diff --git a/src/wix/wixnative/smartcab.cpp b/src/wix/wixnative/smartcab.cpp
index 0dff6c94..1483dfe8 100644
--- a/src/wix/wixnative/smartcab.cpp
+++ b/src/wix/wixnative/smartcab.cpp
@@ -1,14 +1,15 @@
 // Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
 
 #include "precomp.h"
+#include "fileutil.h"
 
-static HRESULT CompressFiles(HANDLE hCab);
-static void __stdcall CabNamesCallback(LPWSTR wzFirstCabName, LPWSTR wzNewCabName, LPWSTR wzFileToken);
+static HRESULT CompressFiles(__in HANDLE hCab);
+static void __stdcall CabNamesCallback(__in_z LPWSTR wzFirstCabName, __in_z LPWSTR wzNewCabName, __in_z LPWSTR wzFileToken);
 
 
 HRESULT SmartCabCommand(
     __in int argc,
-    __in LPWSTR argv[]
+    __in_ecount(argc) LPWSTR argv[]
 )
 {
     HRESULT hr = E_INVALIDARG;
@@ -66,6 +67,9 @@ HRESULT SmartCabCommand(
 
     if (uiFileCount > 0)
     {
+        hr = WixNativeReadStdinPreamble();
+        ExitOnFailure(hr, "failed to read stdin preamble before smartcabbing");
+
         hr = CompressFiles(hCab);
         ExitOnFailure(hr, "failed to compress files into cabinet: %ls", wzCabPath);
     }
@@ -151,9 +155,9 @@ LExit:
 // Second argument is name of the new cabinet that would be formed by splitting e.g. "cab1b.cab"
 // Third argument is the file token of the first file present in the splitting cabinet
 static void __stdcall CabNamesCallback(
-    __in LPWSTR wzFirstCabName,
-    __in LPWSTR wzNewCabName,
-    __in LPWSTR wzFileToken
+    __in_z LPWSTR wzFirstCabName,
+    __in_z LPWSTR wzNewCabName,
+    __in_z LPWSTR wzFileToken
 )
 {
     ConsoleWriteLine(CONSOLE_COLOR_NORMAL, "%ls\t%ls\t%ls", wzFirstCabName, wzNewCabName, wzFileToken);
diff --git a/src/wix/wixnative/wixnative.cpp b/src/wix/wixnative/wixnative.cpp
index 7bd8dbca..d1236da1 100644
--- a/src/wix/wixnative/wixnative.cpp
+++ b/src/wix/wixnative/wixnative.cpp
@@ -36,3 +36,31 @@ int __cdecl wmain(int argc, LPWSTR argv[])
     ConsoleUninitialize();
     return HRESULT_CODE(hr);
 }
+
+HRESULT WixNativeReadStdinPreamble()
+{
+    HRESULT hr = S_OK;
+    LPWSTR sczLine = NULL;
+    size_t cchPreamble = 0;
+
+    // Read the first line to determine if a byte-order-mark was prepended to stdin.
+    // A byte-order-mark is not normally expected but has been seen in some CI/CD systems.
+    // The preable is a single line with ":".
+    hr = ConsoleReadW(&sczLine);
+    ConsoleExitOnFailure(hr, CONSOLE_COLOR_RED, "failed to read preamble from stdin");
+
+    hr = ::StringCchLengthW(sczLine, STRSAFE_MAX_CCH, &cchPreamble);
+    ConsoleExitOnFailure(hr, CONSOLE_COLOR_RED, "failed to get length of stdin preamble");
+
+    // Ensure the preamble ends with ":" and ignore anything before that (since it may be a BOM).
+    if (!cchPreamble || sczLine[cchPreamble - 1] != L':')
+    {
+        hr = E_INVALIDDATA;
+        ConsoleExitOnFailure(hr, CONSOLE_COLOR_RED, "expected ':' as preamble on first line of stdin");
+    }
+
+LExit:
+    ReleaseStr(sczLine);
+
+    return hr;
+}