Fix loading custom table definitions and support tupledef revisions

Loading multiple intermediates with the same custom table definition
would crash when trying to redefine the definition. Now definitions
contain a revision and the highest tuple definition is used when
loading multiple intermediates.

4 files changed, 253 insertions, 19 deletions

diff --git a/src/WixToolset.Data/Intermediate.cs b/src/WixToolset.Data/Intermediate.cs
index 4914ef88..4b7daa4e 100644
--- a/src/WixToolset.Data/Intermediate.cs
+++ b/src/WixToolset.Data/Intermediate.cs
@@ -17,7 +17,7 @@ namespace WixToolset.Data
         public const string XmlNamespaceUri = "http://wixtoolset.org/schemas/v4/wixobj";
         private static readonly Version CurrentVersion = new Version("4.0.0.0");
 
-        private Dictionary<string, Localization> localizationsByCulture;
+        private readonly Dictionary<string, Localization> localizationsByCulture;
 
         /// <summary>
         /// Instantiate a new Intermediate.
@@ -70,7 +70,7 @@ namespace WixToolset.Data
             {
                 var uri = new Uri(Path.GetFullPath(path));
                 var creator = new SimpleTupleDefinitionCreator();
-                return Intermediate.Load(stream, uri, creator, suppressVersionCheck);
+                return Intermediate.LoadIntermediate(stream, uri, creator, suppressVersionCheck);
             }
         }
 
@@ -97,13 +97,13 @@ namespace WixToolset.Data
         /// <returns>Returns the loaded intermediate.</returns>
         public static Intermediate Load(Assembly assembly, string resourceName, ITupleDefinitionCreator creator, bool suppressVersionCheck = false)
         {
-            using (Stream resourceStream = assembly.GetManifestResourceStream(resourceName))
+            using (var resourceStream = assembly.GetManifestResourceStream(resourceName))
             {
                 var uriBuilder = new UriBuilder(assembly.CodeBase);
                 uriBuilder.Scheme = "embeddedresource";
                 uriBuilder.Fragment = resourceName;
 
-                return Intermediate.Load(resourceStream, uriBuilder.Uri, creator, suppressVersionCheck);
+                return Intermediate.LoadIntermediate(resourceStream, uriBuilder.Uri, creator, suppressVersionCheck);
             }
         }
 
@@ -120,11 +120,62 @@ namespace WixToolset.Data
             {
                 var uri = new Uri(Path.GetFullPath(path));
 
-                return Intermediate.Load(stream, uri, creator, suppressVersionCheck);
+                return Intermediate.LoadIntermediate(stream, uri, creator, suppressVersionCheck);
             }
         }
 
         /// <summary>
+        /// Loads several intermediates from paths on disk using the same definitions.
+        /// </summary>
+        /// <param name="intermediateFiles">Paths to intermediate files saved on disk.</param>
+        /// <param name="suppressVersionCheck">Suppress checking for wix.dll version mismatches.</param>
+        /// <returns>Returns the loaded intermediates</returns>
+        public static IEnumerable<Intermediate> Load(IEnumerable<string> intermediateFiles, bool suppressVersionCheck = false)
+        {
+            var creator = new SimpleTupleDefinitionCreator();
+
+            return Intermediate.Load(intermediateFiles, creator);
+        }
+
+        /// <summary>
+        /// Loads several intermediates from paths on disk using the same definitions.
+        /// </summary>
+        /// <param name="intermediateFiles">Paths to intermediate files saved on disk.</param>
+        /// <param name="creator">ITupleDefinitionCreator to use when reconstituting the intermediates.</param>
+        /// <param name="suppressVersionCheck">Suppress checking for wix.dll version mismatches.</param>
+        /// <returns>Returns the loaded intermediates</returns>
+        public static IEnumerable<Intermediate> Load(IEnumerable<string> intermediateFiles, ITupleDefinitionCreator creator, bool suppressVersionCheck = false)
+        {
+            var jsons = new Queue<JsonWithPath>();
+            var intermediates = new List<Intermediate>();
+
+            foreach (var path in intermediateFiles)
+            {
+                using (var stream = File.OpenRead(path))
+                {
+                    var uri = new Uri(Path.GetFullPath(path));
+
+                    var json = Intermediate.LoadJson(stream, uri, suppressVersionCheck);
+
+                    Intermediate.LoadDefinitions(json, creator);
+
+                    jsons.Enqueue(new JsonWithPath { Json = json, Path = uri });
+                }
+            }
+
+            while (jsons.Count > 0)
+            {
+                var jsonWithPath = jsons.Dequeue();
+
+                var intermediate = Intermediate.FinalizeLoad(jsonWithPath.Json, jsonWithPath.Path, creator);
+
+                intermediates.Add(intermediate);
+            }
+
+            return intermediates;
+        }
+
+        /// <summary>
         /// Saves an intermediate to a path on disk.
         /// </summary>
         /// <param name="path">Path to save intermediate file to disk.</param>
@@ -151,7 +202,7 @@ namespace WixToolset.Data
 
                 jsonObject.Add("sections", sectionsJson);
 
-                var customDefinitions = GetCustomDefinitionsInSections();
+                var customDefinitions = this.GetCustomDefinitionsInSections();
 
                 if (customDefinitions.Count > 0)
                 {
@@ -191,10 +242,25 @@ namespace WixToolset.Data
         /// <param name="creator">ITupleDefinitionCreator to use when reconstituting the intermediate.</param>
         /// <param name="suppressVersionCheck">Suppress checking for wix.dll version mismatches.</param>
         /// <returns>Returns the loaded intermediate.</returns>
-        private static Intermediate Load(Stream stream, Uri baseUri, ITupleDefinitionCreator creator, bool suppressVersionCheck = false)
+        private static Intermediate LoadIntermediate(Stream stream, Uri baseUri, ITupleDefinitionCreator creator, bool suppressVersionCheck = false)
         {
-            JsonObject jsonObject;
+            var json = Intermediate.LoadJson(stream, baseUri, suppressVersionCheck);
 
+            Intermediate.LoadDefinitions(json, creator);
+
+            return Intermediate.FinalizeLoad(json, baseUri, creator);
+        }
+
+        /// <summary>
+        /// Loads json form of intermedaite from stream.
+        /// </summary>
+        /// <param name="stream">Stream to intermediate file.</param>
+        /// <param name="baseUri">Path name of intermediate file.</param>
+        /// <param name="suppressVersionCheck">Suppress checking for wix.dll version mismatches.</param>
+        /// <returns>Returns the loaded json.</returns>
+        private static JsonObject LoadJson(Stream stream, Uri baseUri, bool suppressVersionCheck)
+        {
+            JsonObject jsonObject;
             using (var fs = FileStructure.Read(stream))
             {
                 if (FileFormat.WixIR != fs.FileFormat)
@@ -216,7 +282,17 @@ namespace WixToolset.Data
                 }
             }
 
-            var definitionsJson = jsonObject.GetValueOrDefault<JsonArray>("definitions");
+            return jsonObject;
+        }
+
+        /// <summary>
+        /// Loads custom definitions in intermediate json into the creator.
+        /// </summary>
+        /// <param name="json">Json version of intermediate.</param>
+        /// <param name="creator">ITupleDefinitionCreator to use when reconstituting the intermediate.</param>
+        private static void LoadDefinitions(JsonObject json, ITupleDefinitionCreator creator)
+        {
+            var definitionsJson = json.GetValueOrDefault<JsonArray>("definitions");
 
             if (definitionsJson != null)
             {
@@ -226,12 +302,22 @@ namespace WixToolset.Data
                     creator.AddCustomTupleDefinition(definition);
                 }
             }
+        }
 
-            var id = jsonObject.GetValueOrDefault<string>("id");
+        /// <summary>
+        /// Loads the sections and localization for the intermediate.
+        /// </summary>
+        /// <param name="json">Json version of intermediate.</param>
+        /// <param name="baseUri">Path to the intermediate.</param>
+        /// <param name="creator">ITupleDefinitionCreator to use when reconstituting the intermediate.</param>
+        /// <returns>The finalized intermediate.</returns>
+        private static Intermediate FinalizeLoad(JsonObject json, Uri baseUri, ITupleDefinitionCreator creator)
+        {
+            var id = json.GetValueOrDefault<string>("id");
 
             var sections = new List<IntermediateSection>();
 
-            var sectionsJson = jsonObject.GetValueOrDefault<JsonArray>("sections");
+            var sectionsJson = json.GetValueOrDefault<JsonArray>("sections");
             foreach (JsonObject sectionJson in sectionsJson)
             {
                 var section = IntermediateSection.Deserialize(creator, baseUri, sectionJson);
@@ -240,7 +326,7 @@ namespace WixToolset.Data
 
             var localizations = new Dictionary<string, Localization>(StringComparer.OrdinalIgnoreCase);
 
-            var localizationsJson = jsonObject.GetValueOrDefault<JsonArray>("localizations") ?? new JsonArray();
+            var localizationsJson = json.GetValueOrDefault<JsonArray>("localizations") ?? new JsonArray();
             foreach (JsonObject localizationJson in localizationsJson)
             {
                 var localization = Localization.Deserialize(localizationJson);
@@ -409,5 +495,12 @@ namespace WixToolset.Data
 
             return customDefinitions;
         }
+
+        private struct JsonWithPath
+        {
+            public JsonObject Json { get; set; }
+
+            public Uri Path { get; set; }
+        }
     }
 }
diff --git a/src/WixToolset.Data/IntermediateTupleDefinition.cs b/src/WixToolset.Data/IntermediateTupleDefinition.cs
index eb4ab12e..ea15c2dd 100644
--- a/src/WixToolset.Data/IntermediateTupleDefinition.cs
+++ b/src/WixToolset.Data/IntermediateTupleDefinition.cs
@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
 
 namespace WixToolset.Data
 {
@@ -8,19 +8,25 @@ namespace WixToolset.Data
     public class IntermediateTupleDefinition
     {
         public IntermediateTupleDefinition(string name, IntermediateFieldDefinition[] fieldDefinitions, Type strongTupleType)
-            : this(TupleDefinitionType.MustBeFromAnExtension, name, fieldDefinitions, strongTupleType)
+            : this(TupleDefinitionType.MustBeFromAnExtension, name, 0, fieldDefinitions, strongTupleType)
+        {
+        }
+
+        public IntermediateTupleDefinition(string name, int revision, IntermediateFieldDefinition[] fieldDefinitions, Type strongTupleType)
+            : this(TupleDefinitionType.MustBeFromAnExtension, name, revision, fieldDefinitions, strongTupleType)
         {
         }
 
         internal IntermediateTupleDefinition(TupleDefinitionType type, IntermediateFieldDefinition[] fieldDefinitions, Type strongTupleType)
-            : this(type, type.ToString(), fieldDefinitions, strongTupleType)
+            : this(type, type.ToString(), 0, fieldDefinitions, strongTupleType)
         {
         }
 
-        private IntermediateTupleDefinition(TupleDefinitionType type, string name, IntermediateFieldDefinition[] fieldDefinitions, Type strongTupleType)
+        private IntermediateTupleDefinition(TupleDefinitionType type, string name, int revision, IntermediateFieldDefinition[] fieldDefinitions, Type strongTupleType)
         {
             this.Type = type;
             this.Name = name;
+            this.Revision = revision;
             this.FieldDefinitions = fieldDefinitions;
             this.StrongTupleType = strongTupleType ?? typeof(IntermediateTuple);
 #if DEBUG
@@ -28,6 +34,8 @@ namespace WixToolset.Data
 #endif
         }
 
+        public int Revision { get; }
+
         public TupleDefinitionType Type { get; }
 
         public string Name { get; }
@@ -48,6 +56,7 @@ namespace WixToolset.Data
         internal static IntermediateTupleDefinition Deserialize(JsonObject jsonObject)
         {
             var name = jsonObject.GetValueOrDefault<string>("name");
+            var revision = jsonObject.GetValueOrDefault("rev", 0);
             var definitionsJson = jsonObject.GetValueOrDefault<JsonArray>("fields");
 
             var fieldDefinitions = new IntermediateFieldDefinition[definitionsJson.Count];
@@ -60,7 +69,7 @@ namespace WixToolset.Data
                 fieldDefinitions[i] = new IntermediateFieldDefinition(fieldName, fieldType);
             }
 
-            return new IntermediateTupleDefinition(name, fieldDefinitions, null);
+            return new IntermediateTupleDefinition(name, revision, fieldDefinitions, null);
         }
 
         internal JsonObject Serialize()
@@ -70,6 +79,11 @@ namespace WixToolset.Data
                 { "name", this.Name }
             };
 
+            if (this.Revision > 0)
+            {
+                jsonObject.Add("rev", this.Revision);
+            }
+
             var fieldsJson = new JsonArray(this.FieldDefinitions.Length);
 
             foreach (var fieldDefinition in this.FieldDefinitions)
diff --git a/src/WixToolset.Data/SimpleTupleDefinitionCreator.cs b/src/WixToolset.Data/SimpleTupleDefinitionCreator.cs
index 6a86639a..257f028c 100644
--- a/src/WixToolset.Data/SimpleTupleDefinitionCreator.cs
+++ b/src/WixToolset.Data/SimpleTupleDefinitionCreator.cs
@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
 
 namespace WixToolset.Data
 {
@@ -10,7 +10,10 @@ namespace WixToolset.Data
 
         public void AddCustomTupleDefinition(IntermediateTupleDefinition definition)
         {
-            this.CustomDefinitionByName.Add(definition.Name, definition);
+            if (!this.CustomDefinitionByName.TryGetValue(definition.Name, out var existing) || definition.Revision > existing.Revision)
+            {
+                this.CustomDefinitionByName[definition.Name] = definition;
+            }
         }
 
         public bool TryGetTupleDefinitionByName(string name, out IntermediateTupleDefinition tupleDefinition)
diff --git a/src/test/WixToolsetTest.Data/SerializeFixture.cs b/src/test/WixToolsetTest.Data/SerializeFixture.cs
index 1e03481d..ac2f5454 100644
--- a/src/test/WixToolsetTest.Data/SerializeFixture.cs
+++ b/src/test/WixToolsetTest.Data/SerializeFixture.cs
@@ -42,6 +42,130 @@ namespace WixToolsetTest.Data
         }
 
         [Fact]
+        public void CanSaveAndLoadIntermediateWithCustomDefinitions()
+        {
+            var sln = new SourceLineNumber("test.wxs", 1);
+
+            var section = new IntermediateSection("test", SectionType.Product, 65001);
+
+            var fieldDefs = new[]
+            {
+                new IntermediateFieldDefinition("A", IntermediateFieldType.String),
+                new IntermediateFieldDefinition("B", IntermediateFieldType.Number),
+                new IntermediateFieldDefinition("C", IntermediateFieldType.Bool),
+            };
+
+            var tupleDef = new IntermediateTupleDefinition("CustomDef", fieldDefs, null);
+
+            var tuple = tupleDef.CreateTuple(sln, new Identifier(AccessModifier.Public, "customT"));
+            tuple.Set(0, "foo");
+            tuple.Set(1, 2);
+            tuple.Set(2, true);
+
+            section.Tuples.Add(tuple);
+
+            var intermediate = new Intermediate("TestIntermediate", new[] { section }, null, null);
+
+            var path = Path.GetTempFileName();
+            try
+            {
+                intermediate.Save(path);
+
+                var loaded = Intermediate.Load(path);
+                var loadedSection = loaded.Sections.Single();
+                var loadedTuple = loadedSection.Tuples.Single();
+
+                Assert.Equal("foo", loadedTuple.AsString(0));
+                Assert.Equal(2, loadedTuple[1].AsNumber());
+                Assert.True(loadedTuple[2].AsBool());
+            }
+            finally
+            {
+                File.Delete(path);
+            }
+        }
+
+        [Fact]
+        public void CanSaveAndLoadMultipleIntermediateWithCustomDefinitions()
+        {
+            var sln = new SourceLineNumber("test.wxs", 1);
+
+            // Intermediate #1
+            var fieldDefs = new[]
+            {
+                new IntermediateFieldDefinition("A", IntermediateFieldType.String),
+                new IntermediateFieldDefinition("B", IntermediateFieldType.Number),
+                new IntermediateFieldDefinition("C", IntermediateFieldType.Bool),
+            };
+
+            var tupleDef = new IntermediateTupleDefinition("CustomDef", fieldDefs, null);
+
+            var tuple = tupleDef.CreateTuple(sln, new Identifier(AccessModifier.Public, "customT"));
+            tuple.Set(0, "foo");
+            tuple.Set(1, 2);
+            tuple.Set(2, true);
+
+            var section = new IntermediateSection("test", SectionType.Product, 65001);
+            section.Tuples.Add(tuple);
+
+            var intermediate1 = new Intermediate("TestIntermediate", new[] { section }, null, null);
+
+            // Intermediate #2
+            var fieldDefs2 = new[]
+            {
+                new IntermediateFieldDefinition("A", IntermediateFieldType.String),
+                new IntermediateFieldDefinition("B", IntermediateFieldType.Number),
+                new IntermediateFieldDefinition("C", IntermediateFieldType.Bool),
+                new IntermediateFieldDefinition("D", IntermediateFieldType.String),
+            };
+
+            var tupleDef2 = new IntermediateTupleDefinition("CustomDef", 1, fieldDefs2, null);
+
+            var tuple2 = tupleDef2.CreateTuple(sln, new Identifier(AccessModifier.Public, "customT2"));
+            tuple2.Set(0, "bar");
+            tuple2.Set(1, 3);
+            tuple2.Set(2, false);
+            tuple2.Set(3, "baz");
+
+            var section2 = new IntermediateSection("test2", SectionType.Fragment, 65001);
+            section2.Tuples.Add(tuple2);
+
+            var intermediate2 = new Intermediate("TestIntermediate2", new[] { section2 }, null, null);
+
+            // Save
+            var path1 = Path.GetTempFileName();
+            var path2 = Path.GetTempFileName();
+            try
+            {
+                intermediate1.Save(path1);
+                intermediate2.Save(path2);
+
+                var loaded = Intermediate.Load(new[] { path1, path2 });
+
+                var loaded1 = loaded.First();
+                var loaded2 = loaded.Skip(1).Single();
+
+                var loadedTuple1 = loaded1.Sections.Single().Tuples.Single();
+                var loadedTuple2 = loaded2.Sections.Single().Tuples.Single();
+
+                Assert.Equal("foo", loadedTuple1.AsString(0));
+                Assert.Equal(2, loadedTuple1[1].AsNumber());
+                Assert.True(loadedTuple1[2].AsBool());
+                Assert.Null(loadedTuple1.AsString(3));
+
+                Assert.Equal("bar", loadedTuple2.AsString(0));
+                Assert.Equal(3, loadedTuple2[1].AsNumber());
+                Assert.False(loadedTuple2[2].AsBool());
+                Assert.Equal("baz", loadedTuple2.AsString(3));
+            }
+            finally
+            {
+                File.Delete(path2);
+                File.Delete(path1);
+            }
+        }
+
+        [Fact]
         public void CanSaveAndLoadIntermediateWithLocalization()
         {
             var sln = new SourceLineNumber("test.wxs", 10);