Add ability to skip tests at runtime, and skip long running cache tests

6665

11 files changed, 237 insertions, 1 deletions

diff --git a/src/internal/WixBuildTools.TestSupport/WixBuildTools.TestSupport.csproj b/src/internal/WixBuildTools.TestSupport/WixBuildTools.TestSupport.csproj
index d614476f..8a5237d1 100644
--- a/src/internal/WixBuildTools.TestSupport/WixBuildTools.TestSupport.csproj
+++ b/src/internal/WixBuildTools.TestSupport/WixBuildTools.TestSupport.csproj
@@ -21,6 +21,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="xunit.assert" />
+    <PackageReference Include="xunit" />
   </ItemGroup>
 </Project>
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkipTestException.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkipTestException.cs
new file mode 100644
index 00000000..bd7d23f9
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkipTestException.cs
@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System;
+
+    public class SkipTestException : Exception
+    {
+        public SkipTestException(string reason)
+            : base(reason)
+        {
+
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactAttribute.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactAttribute.cs
new file mode 100644
index 00000000..4974d489
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactAttribute.cs
@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using Xunit;
+    using Xunit.Sdk;
+
+    // https://github.com/xunit/samples.xunit/blob/5dc1d35a63c3394a8678ac466b882576a70f56f6/DynamicSkipExample
+    [XunitTestCaseDiscoverer("WixBuildTools.TestSupport.XunitExtensions.SkippableFactDiscoverer", "WixBuildTools.TestSupport")]
+    public class SkippableFactAttribute : FactAttribute
+    {
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactDiscoverer.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactDiscoverer.cs
new file mode 100644
index 00000000..b692c912
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactDiscoverer.cs
@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System.Collections.Generic;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableFactDiscoverer : IXunitTestCaseDiscoverer
+    {
+        private IMessageSink DiagnosticMessageSink { get; }
+
+        public SkippableFactDiscoverer(IMessageSink diagnosticMessageSink)
+        {
+            this.DiagnosticMessageSink = diagnosticMessageSink;
+        }
+
+        public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
+        {
+            yield return new SkippableFactTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod);
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactMessageBus.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactMessageBus.cs
new file mode 100644
index 00000000..6d01889e
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactMessageBus.cs
@@ -0,0 +1,40 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System.Linq;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableFactMessageBus : IMessageBus
+    {
+        private IMessageBus InnerBus { get; }
+
+        public SkippableFactMessageBus(IMessageBus innerBus)
+        {
+            this.InnerBus = innerBus;
+        }
+
+        public int DynamicallySkippedTestCount { get; private set; }
+
+        public void Dispose()
+        {
+        }
+
+        public bool QueueMessage(IMessageSinkMessage message)
+        {
+            if (message is ITestFailed testFailed)
+            {
+                var exceptionType = testFailed.ExceptionTypes.FirstOrDefault();
+                if (exceptionType == typeof(SkipTestException).FullName)
+                {
+                    ++this.DynamicallySkippedTestCount;
+                    return this.InnerBus.QueueMessage(new TestSkipped(testFailed.Test, testFailed.Messages.FirstOrDefault()));
+                }
+            }
+
+            // Nothing we care about, send it on its way
+            return this.InnerBus.QueueMessage(message);
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactTestCase.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactTestCase.cs
new file mode 100644
index 00000000..f13fec83
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableFactTestCase.cs
@@ -0,0 +1,40 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System;
+    using System.ComponentModel;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableFactTestCase : XunitTestCase
+    {
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
+        public SkippableFactTestCase() { }
+
+        public SkippableFactTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, object[] testMethodArguments = null)
+            : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments)
+        {
+        }
+
+        public override async Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
+                                                        IMessageBus messageBus,
+                                                        object[] constructorArguments,
+                                                        ExceptionAggregator aggregator,
+                                                        CancellationTokenSource cancellationTokenSource)
+        {
+            var skipMessageBus = new SkippableFactMessageBus(messageBus);
+            var result = await base.RunAsync(diagnosticMessageSink, skipMessageBus, constructorArguments, aggregator, cancellationTokenSource);
+            if (skipMessageBus.DynamicallySkippedTestCount > 0)
+            {
+                result.Failed -= skipMessageBus.DynamicallySkippedTestCount;
+                result.Skipped += skipMessageBus.DynamicallySkippedTestCount;
+            }
+
+            return result;
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryAttribute.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryAttribute.cs
new file mode 100644
index 00000000..e026bb59
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryAttribute.cs
@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using Xunit;
+    using Xunit.Sdk;
+
+    [XunitTestCaseDiscoverer("WixBuildTools.TestSupport.XunitExtensions.SkippableFactDiscoverer", "WixBuildTools.TestSupport")]
+    public class SkippableTheoryAttribute : TheoryAttribute
+    {
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryDiscoverer.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryDiscoverer.cs
new file mode 100644
index 00000000..cf4e2b43
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryDiscoverer.cs
@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System.Collections.Generic;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableTheoryDiscoverer : IXunitTestCaseDiscoverer
+    {
+        private IMessageSink DiagnosticMessageSink { get; }
+        private TheoryDiscoverer TheoryDiscoverer { get; }
+
+        public SkippableTheoryDiscoverer(IMessageSink diagnosticMessageSink)
+        {
+            this.DiagnosticMessageSink = diagnosticMessageSink;
+
+            this.TheoryDiscoverer = new TheoryDiscoverer(diagnosticMessageSink);
+        }
+
+        public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
+        {
+            var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault();
+            var defaultMethodDisplayOptions = discoveryOptions.MethodDisplayOptionsOrDefault();
+
+            // Unlike fact discovery, the underlying algorithm for theories is complex, so we let the theory discoverer
+            // do its work, and do a little on-the-fly conversion into our own test cases.
+            foreach (var testCase in this.TheoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute))
+            {
+                if (testCase is XunitTheoryTestCase)
+                {
+                    yield return new SkippableTheoryTestCase(this.DiagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testCase.TestMethod);
+                }
+                else
+                {
+                    yield return new SkippableFactTestCase(this.DiagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testCase.TestMethod, testCase.TestMethodArguments);
+                }
+            }
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryTestCase.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryTestCase.cs
new file mode 100644
index 00000000..3299fe7e
--- /dev/null
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SkippableTheoryTestCase.cs
@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixBuildTools.TestSupport.XunitExtensions
+{
+    using System;
+    using System.ComponentModel;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Xunit.Abstractions;
+    using Xunit.Sdk;
+
+    public class SkippableTheoryTestCase : XunitTheoryTestCase
+    {
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
+        public SkippableTheoryTestCase() { }
+
+        public SkippableTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod)
+            : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod)
+        {
+        }
+
+        public override async Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
+                                                        IMessageBus messageBus,
+                                                        object[] constructorArguments,
+                                                        ExceptionAggregator aggregator,
+                                                        CancellationTokenSource cancellationTokenSource)
+        {
+            // Duplicated code from SkippableFactTestCase. I'm sure we could find a way to de-dup with some thought.
+            var skipMessageBus = new SkippableFactMessageBus(messageBus);
+            var result = await base.RunAsync(diagnosticMessageSink, skipMessageBus, constructorArguments, aggregator, cancellationTokenSource);
+            if (skipMessageBus.DynamicallySkippedTestCount > 0)
+            {
+                result.Failed -= skipMessageBus.DynamicallySkippedTestCount;
+                result.Skipped += skipMessageBus.DynamicallySkippedTestCount;
+            }
+
+            return result;
+        }
+    }
+}
diff --git a/src/internal/WixBuildTools.TestSupport/SucceededException.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SucceededException.cs
index 704fba28..704fba28 100644
--- a/src/internal/WixBuildTools.TestSupport/SucceededException.cs
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/SucceededException.cs
diff --git a/src/internal/WixBuildTools.TestSupport/WixAssert.cs b/src/internal/WixBuildTools.TestSupport/XunitExtensions/WixAssert.cs
index 1db842c3..10156547 100644
--- a/src/internal/WixBuildTools.TestSupport/WixAssert.cs
+++ b/src/internal/WixBuildTools.TestSupport/XunitExtensions/WixAssert.cs
@@ -6,6 +6,7 @@ namespace WixBuildTools.TestSupport
     using System.Collections.Generic;
     using System.Linq;
     using System.Xml.Linq;
+    using WixBuildTools.TestSupport.XunitExtensions;
     using Xunit;
 
     public class WixAssert : Assert
@@ -41,6 +42,16 @@ namespace WixBuildTools.TestSupport
             CompareXml(expectedDoc, actualDoc);
         }
 
+        /// <summary>
+        /// Dynamically skips the test.
+        /// Requires that the test was marked with a fact attribute derived from <see cref="WixBuildTools.TestSupport.XunitExtensions.SkippableFactAttribute" />
+        /// or <see cref="WixBuildTools.TestSupport.XunitExtensions.SkippableTheoryAttribute" />
+        /// </summary>
+        public static void Skip(string message)
+        {
+            throw new SkipTestException(message);
+        }
+
         public static void Succeeded(int hr, string format, params object[] formatArgs)
         {
             if (0 > hr)