Add ability to skip tests at runtime, and skip long running cache tests

6665

10 files changed, 397 insertions, 0 deletions

diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkipTestException.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkipTestException.cs
new file mode 100644
index 00000000..bd7d23f9
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkipTestException.cs
@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System;
+
+    public class SkipTestException : Exception
+    {
+        public SkipTestException(string reason)
+            : base(reason)
+        {
+
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactAttribute.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactAttribute.cs
new file mode 100644
index 00000000..4974d489
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactAttribute.cs
@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using Xunit;
+    using Xunit.Sdk;
+
+    // https://github.com/xunit/samples.xunit/blob/5dc1d35a63c3394a8678ac466b882576a70f56f6/DynamicSkipExample
+    [XunitTestCaseDiscoverer("WixBuildTools.TestSupport.XunitExtensions.SkippableFactDiscoverer", "WixBuildTools.TestSupport")]
+    public class SkippableFactAttribute : FactAttribute
+    {
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactDiscoverer.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactDiscoverer.cs
new file mode 100644
index 00000000..b692c912
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactDiscoverer.cs
@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System.Collections.Generic;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableFactDiscoverer : IXunitTestCaseDiscoverer
+    {
+        private IMessageSink DiagnosticMessageSink { get; }
+
+        public SkippableFactDiscoverer(IMessageSink diagnosticMessageSink)
+        {
+            this.DiagnosticMessageSink = diagnosticMessageSink;
+        }
+
+        public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
+        {
+            yield return new SkippableFactTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod);
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactMessageBus.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactMessageBus.cs
new file mode 100644
index 00000000..6d01889e
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactMessageBus.cs
@@ -0,0 +1,40 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System.Linq;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableFactMessageBus : IMessageBus
+    {
+        private IMessageBus InnerBus { get; }
+
+        public SkippableFactMessageBus(IMessageBus innerBus)
+        {
+            this.InnerBus = innerBus;
+        }
+
+        public int DynamicallySkippedTestCount { get; private set; }
+
+        public void Dispose()
+        {
+        }
+
+        public bool QueueMessage(IMessageSinkMessage message)
+        {
+            if (message is ITestFailed testFailed)
+            {
+                var exceptionType = testFailed.ExceptionTypes.FirstOrDefault();
+                if (exceptionType == typeof(SkipTestException).FullName)
+                {
+                    ++this.DynamicallySkippedTestCount;
+                    return this.InnerBus.QueueMessage(new TestSkipped(testFailed.Test, testFailed.Messages.FirstOrDefault()));
+                }
+            }
+
+            // Nothing we care about, send it on its way
+            return this.InnerBus.QueueMessage(message);
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactTestCase.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactTestCase.cs
new file mode 100644
index 00000000..f13fec83
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactTestCase.cs
@@ -0,0 +1,40 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System;
+    using System.ComponentModel;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableFactTestCase : XunitTestCase
+    {
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
+        public SkippableFactTestCase() { }
+
+        public SkippableFactTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, object[] testMethodArguments = null)
+            : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments)
+        {
+        }
+
+        public override async Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
+                                                        IMessageBus messageBus,
+                                                        object[] constructorArguments,
+                                                        ExceptionAggregator aggregator,
+                                                        CancellationTokenSource cancellationTokenSource)
+        {
+            var skipMessageBus = new SkippableFactMessageBus(messageBus);
+            var result = await base.RunAsync(diagnosticMessageSink, skipMessageBus, constructorArguments, aggregator, cancellationTokenSource);
+            if (skipMessageBus.DynamicallySkippedTestCount > 0)
+            {
+                result.Failed -= skipMessageBus.DynamicallySkippedTestCount;
+                result.Skipped += skipMessageBus.DynamicallySkippedTestCount;
+            }
+
+            return result;
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryAttribute.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryAttribute.cs
new file mode 100644
index 00000000..e026bb59
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryAttribute.cs
@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using Xunit;
+    using Xunit.Sdk;
+
+    [XunitTestCaseDiscoverer("WixBuildTools.TestSupport.XunitExtensions.SkippableFactDiscoverer", "WixBuildTools.TestSupport")]
+    public class SkippableTheoryAttribute : TheoryAttribute
+    {
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryDiscoverer.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryDiscoverer.cs
new file mode 100644
index 00000000..cf4e2b43
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryDiscoverer.cs
@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System.Collections.Generic;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableTheoryDiscoverer : IXunitTestCaseDiscoverer
+    {
+        private IMessageSink DiagnosticMessageSink { get; }
+        private TheoryDiscoverer TheoryDiscoverer { get; }
+
+        public SkippableTheoryDiscoverer(IMessageSink diagnosticMessageSink)
+        {
+            this.DiagnosticMessageSink = diagnosticMessageSink;
+
+            this.TheoryDiscoverer = new TheoryDiscoverer(diagnosticMessageSink);
+        }
+
+        public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
+        {
+            var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault();
+            var defaultMethodDisplayOptions = discoveryOptions.MethodDisplayOptionsOrDefault();
+
+            // Unlike fact discovery, the underlying algorithm for theories is complex, so we let the theory discoverer
+            // do its work, and do a little on-the-fly conversion into our own test cases.
+            foreach (var testCase in this.TheoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute))
+            {
+                if (testCase is XunitTheoryTestCase)
+                {
+                    yield return new SkippableTheoryTestCase(this.DiagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testCase.TestMethod);
+                }
+                else
+                {
+                    yield return new SkippableFactTestCase(this.DiagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testCase.TestMethod, testCase.TestMethodArguments);
+                }
+            }
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryTestCase.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryTestCase.cs
new file mode 100644
index 00000000..3299fe7e
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryTestCase.cs
@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System;
+    using System.ComponentModel;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableTheoryTestCase : XunitTheoryTestCase
+    {
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
+        public SkippableTheoryTestCase() { }
+
+        public SkippableTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod)
+            : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod)
+        {
+        }
+
+        public override async Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
+                                                        IMessageBus messageBus,
+                                                        object[] constructorArguments,
+                                                        ExceptionAggregator aggregator,
+                                                        CancellationTokenSource cancellationTokenSource)
+        {
+            // Duplicated code from SkippableFactTestCase. I'm sure we could find a way to de-dup with some thought.
+            var skipMessageBus = new SkippableFactMessageBus(messageBus);
+            var result = await base.RunAsync(diagnosticMessageSink, skipMessageBus, constructorArguments, aggregator, cancellationTokenSource);
+            if (skipMessageBus.DynamicallySkippedTestCount > 0)
+            {
+                result.Failed -= skipMessageBus.DynamicallySkippedTestCount;
+                result.Skipped += skipMessageBus.DynamicallySkippedTestCount;
+            }
+
+            return result;
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SucceededException.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SucceededException.cs
new file mode 100644
index 00000000..704fba28
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SucceededException.cs
@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport
+{
+    using System;
+    using Xunit.Sdk;
+
+    public class SucceededException : XunitException
+    {
+        public SucceededException(int hr, string userMessage)
+            : base(String.Format("WixAssert.Succeeded() Failure\r\n" +
+                                 "HRESULT:  0x{0:X8}\r\n" +
+                                 "Message:  {1}",
+                                 hr, userMessage))
+        {
+            this.HResult = hr;
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/WixAssert.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/WixAssert.cs
new file mode 100644
index 00000000..10156547
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/WixAssert.cs
@@ -0,0 +1,153 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Xml.Linq;
+    using WixBuildTools.TestSupport.XunitExtensions;
+    using Xunit;
+
+    public class WixAssert : Assert
+    {
+        public static void CompareLineByLine(string[] expectedLines, string[] actualLines)
+        {
+            var lineNumber = 0;
+
+            for (; lineNumber < expectedLines.Length && lineNumber < actualLines.Length; ++lineNumber)
+            {
+                WixAssert.StringEqual($"{lineNumber}: {expectedLines[lineNumber]}", $"{lineNumber}: {actualLines[lineNumber]}");
+            }
+
+            var additionalExpectedLines = expectedLines.Length > lineNumber ? String.Join(Environment.NewLine, expectedLines.Skip(lineNumber).Select((s, i) => $"{lineNumber + i}: {s}")) : $"Missing {actualLines.Length - lineNumber} lines";
+            var additionalActualLines = actualLines.Length > lineNumber ? String.Join(Environment.NewLine, actualLines.Skip(lineNumber).Select((s, i) => $"{lineNumber + i}: {s}")) : $"Missing {expectedLines.Length - lineNumber} lines";
+
+            WixAssert.StringEqual(additionalExpectedLines, additionalActualLines);
+        }
+
+        public static void CompareXml(XContainer xExpected, XContainer xActual)
+        {
+            var expecteds = xExpected.Descendants().Select(x => $"{x.Name.LocalName}:{String.Join(",", x.Attributes().OrderBy(a => a.Name.LocalName).Select(a => $"{a.Name.LocalName}={a.Value}"))}");
+            var actuals = xActual.Descendants().Select(x => $"{x.Name.LocalName}:{String.Join(",", x.Attributes().OrderBy(a => a.Name.LocalName).Select(a => $"{a.Name.LocalName}={a.Value}"))}");
+
+            CompareLineByLine(expecteds.OrderBy(s => s).ToArray(), actuals.OrderBy(s => s).ToArray());
+        }
+
+        public static void CompareXml(string expectedPath, string actualPath)
+        {
+            var expectedDoc = XDocument.Load(expectedPath, LoadOptions.PreserveWhitespace | LoadOptions.SetBaseUri | LoadOptions.SetLineInfo);
+            var actualDoc = XDocument.Load(actualPath, LoadOptions.PreserveWhitespace | LoadOptions.SetBaseUri | LoadOptions.SetLineInfo);
+
+            CompareXml(expectedDoc, actualDoc);
+        }
+
+        /// <summary>
+        /// Dynamically skips the test.
+        /// Requires that the test was marked with a fact attribute derived from <see cref="WixBuildTools.TestSupport.XunitExtensions.SkippableFactAttribute" />
+        /// or <see cref="WixBuildTools.TestSupport.XunitExtensions.SkippableTheoryAttribute" />
+        /// </summary>
+        public static void Skip(string message)
+        {
+            throw new SkipTestException(message);
+        }
+
+        public static void Succeeded(int hr, string format, params object[] formatArgs)
+        {
+            if (0 > hr)
+            {
+                throw new SucceededException(hr, String.Format(format, formatArgs));
+            }
+        }
+
+        public static void StringCollectionEmpty(IList<string> collection)
+        {
+            if (collection.Count > 0)
+            {
+                Assert.True(false, $"The collection was expected to be empty, but instead was [{Environment.NewLine}\"{String.Join($"\", {Environment.NewLine}\"", collection)}\"{Environment.NewLine}]");
+            }
+        }
+
+        public static void StringEqual(string expected, string actual, bool ignoreCase = false)
+        {
+            var comparer = ignoreCase ? StringObjectEqualityComparer.InvariantCultureIgnoreCase : StringObjectEqualityComparer.InvariantCulture;
+            Assert.Equal<object>(expected, actual, comparer);
+        }
+
+        public static void NotStringEqual(string expected, string actual, bool ignoreCase = false)
+        {
+            var comparer = ignoreCase ? StringObjectEqualityComparer.InvariantCultureIgnoreCase : StringObjectEqualityComparer.InvariantCulture;
+            Assert.NotEqual<object>(expected, actual, comparer);
+        }
+
+        private class StringObjectEqualityComparer : IEqualityComparer<object>
+        {
+            public static readonly StringObjectEqualityComparer InvariantCultureIgnoreCase = new StringObjectEqualityComparer(true);
+            public static readonly StringObjectEqualityComparer InvariantCulture = new StringObjectEqualityComparer(false);
+
+            private readonly StringComparer stringComparer;
+
+            public StringObjectEqualityComparer(bool ignoreCase)
+            {
+                this.stringComparer = ignoreCase ? StringComparer.InvariantCultureIgnoreCase : StringComparer.InvariantCulture;
+            }
+
+            public new bool Equals(object x, object y)
+            {
+                return this.stringComparer.Equals((string)x,(string)y);
+            }
+
+            public int GetHashCode(object obj)
+            {
+                return this.stringComparer.GetHashCode((string)obj);
+            }
+        }
+
+        // There appears to have been a bug in VC++, which might or might not have been partially
+        // or completely corrected. It was unable to disambiguate a call to:
+        //     Xunit::Assert::Throws(System::Type^, System::Action^)
+        // from a call to:
+        //     Xunit::Assert::Throws(System::Type^, System::Func<System::Object^>^)
+        // that implicitly ignores its return value.
+        //
+        // The ambiguity may have been reported by some versions of the compiler and not by others.
+        // Some versions of the compiler may not have emitted any code in this situation, making it
+        // appear that the test has passed when, in fact, it hasn't been run.
+        //
+        // This situation is not an issue for C#.
+        //
+        // The following method is used to isolate DUtilTests in order to overcome the above problem.
+
+        /// <summary>
+        /// This shim allows C++/CLR code to call the Xunit method with the same signature
+        /// without getting an ambiguous overload error.  If the specified test code
+        /// fails to generate an exception of the exact specified type, an assertion
+        /// exception is thrown. Otherwise, execution flow proceeds as normal.
+        /// </summary>
+        /// <typeparam name="T">The type name of the expected exception.</typeparam>
+        /// <param name="testCode">An Action delegate to run the test code.</param>
+        public static new void Throws<T>(System.Action testCode)
+            where T : System.Exception
+        {
+            Xunit.Assert.Throws<T>(testCode);
+        }
+
+        // This shim has been tested, but is not currently used anywhere. It was provided
+        // at the same time as the preceding shim because it involved the same overload
+        // resolution conflict.
+
+        /// <summary>
+        /// This shim allows C++/CLR code to call the Xunit method with the same signature
+        /// without getting an ambiguous overload error.  If the specified test code
+        /// fails to generate an exception of the exact specified type, an assertion
+        /// exception is thrown. Otherwise, execution flow proceeds as normal.
+        /// </summary>
+        /// <param name="exceptionType">The type object associated with exceptions of the expected type.</param>
+        /// <param name="testCode">An Action delegate to run the test code.</param>
+        /// <returns>An exception of a type other than the type specified, is such an exception is thrown.</returns>
+        public static new System.Exception Throws(System.Type exceptionType, System.Action testCode)
+        {
+            return Xunit.Assert.Throws(exceptionType, testCode);
+        }
+    }
+}