Fix custom tables, small fixes in linker and update latest Data

4 files changed, 352 insertions, 16 deletions

diff --git a/src/WixToolset.Core.WindowsInstaller/Bind/BindDatabaseCommand.cs b/src/WixToolset.Core.WindowsInstaller/Bind/BindDatabaseCommand.cs
index 53451752..411f64bf 100644
--- a/src/WixToolset.Core.WindowsInstaller/Bind/BindDatabaseCommand.cs
+++ b/src/WixToolset.Core.WindowsInstaller/Bind/BindDatabaseCommand.cs
@@ -32,8 +32,6 @@ namespace WixToolset.Core.WindowsInstaller.Bind
 
             this.PathResolver = this.ServiceProvider.GetService<IPathResolver>();
 
-            this.TableDefinitions = WindowsInstallerStandardInternal.GetTableDefinitions();
-
             this.CabbingThreadCount = context.CabbingThreadCount;
             this.CabCachePath = context.CabCachePath;
             this.Codepage = context.Codepage;
@@ -86,8 +84,6 @@ namespace WixToolset.Core.WindowsInstaller.Bind
 
         private bool SuppressLayout { get; }
 
-        private TableDefinitionCollection TableDefinitions { get; }
-
         private string IntermediateFolder { get; }
 
         private Validator Validator { get; }
@@ -111,6 +107,14 @@ namespace WixToolset.Core.WindowsInstaller.Bind
             // If there are any fields to resolve later, create the cache to populate during bind.
             var variableCache = this.DelayedFields.Any() ? new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase) : null;
 
+            TableDefinitionCollection tableDefinitions;
+            {
+                var command = new LoadTableDefinitionsCommand(section);
+                command.Execute();
+
+                tableDefinitions = command.TableDefinitions;
+            }
+
             // Process the summary information table before the other tables.
             bool compressed;
             bool longNames;
@@ -231,7 +235,7 @@ namespace WixToolset.Core.WindowsInstaller.Bind
                 command.FileFacades = fileFacades;
                 command.UpdateFileFacades = fileFacades.Where(f => !f.FromModule);
                 command.OverwriteHash = true;
-                command.TableDefinitions = this.TableDefinitions;
+                command.TableDefinitions = tableDefinitions;
                 command.VariableCache = variableCache;
                 command.Execute();
             }
@@ -308,7 +312,7 @@ namespace WixToolset.Core.WindowsInstaller.Bind
             // Time to create the output object. Try to put as much above here as possible, updating the IR is better.
             Output output;
             {
-                var command = new CreateOutputFromIRCommand(section, this.TableDefinitions, this.BackendExtensions);
+                var command = new CreateOutputFromIRCommand(this.Messaging, section, tableDefinitions, this.BackendExtensions);
                 command.Execute();
 
                 output = command.Output;
@@ -402,7 +406,7 @@ namespace WixToolset.Core.WindowsInstaller.Bind
                 command.Compressed = compressed;
                 command.FileRowsByCabinet = filesByCabinetMedia;
                 command.ResolveMedia = this.ResolveMedia;
-                command.TableDefinitions = this.TableDefinitions;
+                command.TableDefinitions = tableDefinitions;
                 command.TempFilesLocation = this.IntermediateFolder;
                 command.Execute();
 
@@ -429,11 +433,13 @@ namespace WixToolset.Core.WindowsInstaller.Bind
             // Generate database file.
             this.Messaging.Write(VerboseMessages.GeneratingDatabase());
 
-            var trackMsi = this.BackendHelper.TrackFile(this.OutputPath, TrackedFileType.Final);
-            trackedFiles.Add(trackMsi);
+            {
+                var trackMsi = this.BackendHelper.TrackFile(this.OutputPath, TrackedFileType.Final);
+                trackedFiles.Add(trackMsi);
 
-            var temporaryFiles = this.GenerateDatabase(output, trackMsi.Path, false, false);
-            trackedFiles.AddRange(temporaryFiles);
+                var temporaryFiles = this.GenerateDatabase(output, tableDefinitions, trackMsi.Path, false, false);
+                trackedFiles.AddRange(temporaryFiles);
+            }
 
             // Stop processing if an error previously occurred.
             if (this.Messaging.EncounteredError)
@@ -456,7 +462,7 @@ namespace WixToolset.Core.WindowsInstaller.Bind
 
                     if (null == sequenceTable)
                     {
-                        sequenceTable = output.EnsureTable(this.TableDefinitions[sequenceTableName]);
+                        sequenceTable = output.EnsureTable(tableDefinitions[sequenceTableName]);
                     }
 
                     if (0 == sequenceTable.Rows.Count)
@@ -911,7 +917,7 @@ namespace WixToolset.Core.WindowsInstaller.Bind
         /// <param name="databaseFile">The database file to create.</param>
         /// <param name="keepAddedColumns">Whether to keep columns added in a transform.</param>
         /// <param name="useSubdirectory">Whether to use a subdirectory based on the <paramref name="databaseFile"/> file name for intermediate files.</param>
-        private IEnumerable<ITrackedFile> GenerateDatabase(Output output, string databaseFile, bool keepAddedColumns, bool useSubdirectory)
+        private IEnumerable<ITrackedFile> GenerateDatabase(Output output, TableDefinitionCollection tableDefinitions, string databaseFile, bool keepAddedColumns, bool useSubdirectory)
         {
             var command = new GenerateDatabaseCommand();
             command.BackendHelper = this.BackendHelper;
@@ -921,7 +927,7 @@ namespace WixToolset.Core.WindowsInstaller.Bind
             command.KeepAddedColumns = keepAddedColumns;
             command.UseSubDirectory = useSubdirectory;
             command.SuppressAddingValidationRows = this.SuppressAddingValidationRows;
-            command.TableDefinitions = this.TableDefinitions;
+            command.TableDefinitions = tableDefinitions;
             command.IntermediateFolder = this.IntermediateFolder;
             command.Codepage = this.Codepage;
             command.Execute();
diff --git a/src/WixToolset.Core.WindowsInstaller/Bind/CreateOutputFromIRCommand.cs b/src/WixToolset.Core.WindowsInstaller/Bind/CreateOutputFromIRCommand.cs
index ebb494c0..17cac83a 100644
--- a/src/WixToolset.Core.WindowsInstaller/Bind/CreateOutputFromIRCommand.cs
+++ b/src/WixToolset.Core.WindowsInstaller/Bind/CreateOutputFromIRCommand.cs
@@ -11,14 +11,18 @@ namespace WixToolset.Core.WindowsInstaller.Bind
     using WixToolset.Data.WindowsInstaller;
     using WixToolset.Data.WindowsInstaller.Rows;
     using WixToolset.Extensibility;
+    using WixToolset.Extensibility.Services;
 
     internal class CreateOutputFromIRCommand
     {
         private const int DefaultMaximumUncompressedMediaSize = 200; // Default value is 200 MB
         private const int MaxValueOfMaxCabSizeForLargeFileSplitting = 2 * 1024; // 2048 MB (i.e. 2 GB)
 
-        public CreateOutputFromIRCommand(IntermediateSection section, TableDefinitionCollection tableDefinitions, IEnumerable<IWindowsInstallerBackendBinderExtension> backendExtensions)
+        private static readonly char[] ColonCharacter = new[] { ':' };
+
+        public CreateOutputFromIRCommand(IMessaging messaging, IntermediateSection section, TableDefinitionCollection tableDefinitions, IEnumerable<IWindowsInstallerBackendBinderExtension> backendExtensions)
         {
+            this.Messaging = messaging;
             this.Section = section;
             this.TableDefinitions = tableDefinitions;
             this.BackendExtensions = backendExtensions;
@@ -26,6 +30,8 @@ namespace WixToolset.Core.WindowsInstaller.Bind
 
         private IEnumerable<IWindowsInstallerBackendBinderExtension> BackendExtensions { get; }
 
+        private IMessaging Messaging { get; }
+
         private TableDefinitionCollection TableDefinitions { get; }
 
         private IntermediateSection Section { get; }
@@ -49,6 +55,11 @@ namespace WixToolset.Core.WindowsInstaller.Bind
             {
                 switch (tuple.Definition.Type)
                 {
+                case TupleDefinitionType.AppSearch:
+                    this.AddTupleDefaultly(tuple, output);
+                    output.EnsureTable(this.TableDefinitions["Signature"]);
+                    break;
+
                 case TupleDefinitionType.Binary:
                     this.AddTupleDefaultly(tuple, output, idIsPrimaryKey: true);
                     break;
@@ -133,6 +144,11 @@ namespace WixToolset.Core.WindowsInstaller.Bind
                     this.AddMoveFileTuple((MoveFileTuple)tuple, output);
                     break;
 
+                case TupleDefinitionType.ProgId:
+                    this.AddTupleDefaultly(tuple, output);
+                    output.EnsureTable(this.TableDefinitions["Extension"]);
+                    break;
+
                 case TupleDefinitionType.Property:
                     this.AddPropertyTuple((PropertyTuple)tuple, output);
                     break;
@@ -197,6 +213,14 @@ namespace WixToolset.Core.WindowsInstaller.Bind
                     this.AddTupleFromExtension(tuple, output);
                     break;
 
+                case TupleDefinitionType.WixCustomRow:
+                    this.AddWixCustomRowTuple((WixCustomRowTuple)tuple, output);
+                    break;
+
+                case TupleDefinitionType.WixEnsureTable:
+                    this.AddWixEnsureTableTuple((WixEnsureTableTuple)tuple, output);
+                    break;
+
                 // ignored.
                 case TupleDefinitionType.WixFile:
                 case TupleDefinitionType.WixComponentGroup:
@@ -204,6 +228,10 @@ namespace WixToolset.Core.WindowsInstaller.Bind
                 case TupleDefinitionType.WixFeatureGroup:
                     break;
 
+                // Already processed.
+                case TupleDefinitionType.WixCustomTable:
+                    break;
+
                 default:
                     this.AddTupleDefaultly(tuple, output);
                     break;
@@ -382,6 +410,8 @@ namespace WixToolset.Core.WindowsInstaller.Bind
             row[7] = tuple.FirstControlRef;
             row[8] = tuple.DefaultControlRef;
             row[9] = tuple.CancelControlRef;
+
+            output.EnsureTable(this.TableDefinitions["ListBox"]);
         }
 
         private void AddDirectoryTuple(DirectoryTuple tuple, Output output)
@@ -929,6 +959,93 @@ namespace WixToolset.Core.WindowsInstaller.Bind
                 row[2] = tuple.Sequence;
             }
         }
+        
+        private void AddWixCustomRowTuple(WixCustomRowTuple tuple, Output output)
+        {
+            var customTableDefinition = this.TableDefinitions[tuple.Table];
+
+            if (customTableDefinition.Unreal)
+            {
+
+                return;
+            }
+
+            var customTable = output.EnsureTable(customTableDefinition);
+            var customRow = customTable.CreateRow(tuple.SourceLineNumbers);
+
+#if TODO // SectionId seems like a good thing to preserve.
+            customRow.SectionId = tuple.SectionId;
+#endif
+
+            var data = tuple.FieldDataSeparated;
+
+            for (var i = 0; i < data.Length; ++i)
+            {
+                var foundColumn = false;
+                var item = data[i].Split(ColonCharacter, 2);
+
+                for (var j = 0; j < customRow.Fields.Length; ++j)
+                {
+                    if (customRow.Fields[j].Column.Name == item[0])
+                    {
+                        if (0 < item[1].Length)
+                        {
+                            if (ColumnType.Number == customRow.Fields[j].Column.Type)
+                            {
+                                try
+                                {
+                                    customRow.Fields[j].Data = Convert.ToInt32(item[1], CultureInfo.InvariantCulture);
+                                }
+                                catch (FormatException)
+                                {
+                                    this.Messaging.Write(ErrorMessages.IllegalIntegerValue(tuple.SourceLineNumbers, customTableDefinition.Columns[i].Name, customTableDefinition.Name, item[1]));
+                                }
+                                catch (OverflowException)
+                                {
+                                    this.Messaging.Write(ErrorMessages.IllegalIntegerValue(tuple.SourceLineNumbers, customTableDefinition.Columns[i].Name, customTableDefinition.Name, item[1]));
+                                }
+                            }
+                            else if (ColumnCategory.Identifier == customRow.Fields[j].Column.Category)
+                            {
+                                if (Common.IsIdentifier(item[1]) || Common.IsValidBinderVariable(item[1]) || ColumnCategory.Formatted == customRow.Fields[j].Column.Category)
+                                {
+                                    customRow.Fields[j].Data = item[1];
+                                }
+                                else
+                                {
+                                    this.Messaging.Write(ErrorMessages.IllegalIdentifier(tuple.SourceLineNumbers, "Data", item[1]));
+                                }
+                            }
+                            else
+                            {
+                                customRow.Fields[j].Data = item[1];
+                            }
+                        }
+                        foundColumn = true;
+                        break;
+                    }
+                }
+
+                if (!foundColumn)
+                {
+                    this.Messaging.Write(ErrorMessages.UnexpectedCustomTableColumn(tuple.SourceLineNumbers, item[0]));
+                }
+            }
+
+            for (var i = 0; i < customTableDefinition.Columns.Length; ++i)
+            {
+                if (!customTableDefinition.Columns[i].Nullable && (null == customRow.Fields[i].Data || 0 == customRow.Fields[i].Data.ToString().Length))
+                {
+                    this.Messaging.Write(ErrorMessages.NoDataForColumn(tuple.SourceLineNumbers, customTableDefinition.Columns[i].Name, customTableDefinition.Name));
+                }
+            }
+        }
+
+        private void AddWixEnsureTableTuple(WixEnsureTableTuple tuple, Output output)
+        {
+            var tableDefinition = this.TableDefinitions[tuple.Table];
+            output.EnsureTable(tableDefinition);
+        }
 
         private void AddWixMediaTemplateTuple(WixMediaTemplateTuple tuple, Output output)
         {
diff --git a/src/WixToolset.Core.WindowsInstaller/Bind/LoadTableDefinitionsCommand.cs b/src/WixToolset.Core.WindowsInstaller/Bind/LoadTableDefinitionsCommand.cs
new file mode 100644
index 00000000..05f865fa
--- /dev/null
+++ b/src/WixToolset.Core.WindowsInstaller/Bind/LoadTableDefinitionsCommand.cs
@@ -0,0 +1,213 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixToolset.Core.WindowsInstaller.Bind
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Globalization;
+    using System.Linq;
+    using WixToolset.Data;
+    using WixToolset.Data.Tuples;
+    using WixToolset.Data.WindowsInstaller;
+
+    internal class LoadTableDefinitionsCommand
+    {
+        public LoadTableDefinitionsCommand(IntermediateSection section) => this.Section = section;
+
+        public TableDefinitionCollection TableDefinitions { get; private set; }
+
+        private IntermediateSection Section { get; }
+
+        public TableDefinitionCollection Execute()
+        {
+            var tableDefinitions = new TableDefinitionCollection(WindowsInstallerStandardInternal.GetTableDefinitions());
+
+            foreach (var tuple in this.Section.Tuples.OfType<WixCustomTableTuple>())
+            {
+                var customTableDefinition = this.CreateCustomTable(tuple);
+                tableDefinitions.Add(customTableDefinition);
+            }
+
+            this.TableDefinitions = tableDefinitions;
+            return this.TableDefinitions;
+        }
+
+        private TableDefinition CreateCustomTable(WixCustomTableTuple row)
+        {
+            var columnNames = row.ColumnNames.Split('\t');
+            var columnTypes = row.ColumnTypes.Split('\t');
+            var primaryKeys = row.PrimaryKeys.Split('\t');
+            var minValues = row.MinValues?.Split('\t');
+            var maxValues = row.MaxValues?.Split('\t');
+            var keyTables = row.KeyTables?.Split('\t');
+            var keyColumns = row.KeyColumns?.Split('\t');
+            var categories = row.Categories?.Split('\t');
+            var sets = row.Sets?.Split('\t');
+            var descriptions = row.Descriptions?.Split('\t');
+            var modularizations = row.Modularizations?.Split('\t');
+
+            var currentPrimaryKey = 0;
+
+            var columns = new List<ColumnDefinition>(columnNames.Length);
+            for (var i = 0; i < columnNames.Length; ++i)
+            {
+                var name = columnNames[i];
+                var type = ColumnType.Unknown;
+
+                if (columnTypes[i].StartsWith("s", StringComparison.OrdinalIgnoreCase))
+                {
+                    type = ColumnType.String;
+                }
+                else if (columnTypes[i].StartsWith("l", StringComparison.OrdinalIgnoreCase))
+                {
+                    type = ColumnType.Localized;
+                }
+                else if (columnTypes[i].StartsWith("i", StringComparison.OrdinalIgnoreCase))
+                {
+                    type = ColumnType.Number;
+                }
+                else if (columnTypes[i].StartsWith("v", StringComparison.OrdinalIgnoreCase))
+                {
+                    type = ColumnType.Object;
+                }
+
+                var nullable = columnTypes[i].Substring(0, 1) == columnTypes[i].Substring(0, 1).ToUpperInvariant();
+                var length = Convert.ToInt32(columnTypes[i].Substring(1), CultureInfo.InvariantCulture);
+
+                var primaryKey = false;
+                if (currentPrimaryKey < primaryKeys.Length && primaryKeys[currentPrimaryKey] == columnNames[i])
+                {
+                    primaryKey = true;
+                    currentPrimaryKey++;
+                }
+
+                var minValue = String.IsNullOrEmpty(minValues?[i]) ? (int?)null : Convert.ToInt32(minValues[i], CultureInfo.InvariantCulture);
+                var maxValue = String.IsNullOrEmpty(maxValues?[i]) ? (int?)null : Convert.ToInt32(maxValues[i], CultureInfo.InvariantCulture);
+                var keyColumn = String.IsNullOrEmpty(keyColumns?[i]) ? (int?)null : Convert.ToInt32(keyColumns[i], CultureInfo.InvariantCulture);
+
+                var category = ColumnCategory.Unknown;
+                if (null != categories && null != categories[i] && 0 < categories[i].Length)
+                {
+                    switch (categories[i])
+                    {
+                        case "Text":
+                            category = ColumnCategory.Text;
+                            break;
+                        case "UpperCase":
+                            category = ColumnCategory.UpperCase;
+                            break;
+                        case "LowerCase":
+                            category = ColumnCategory.LowerCase;
+                            break;
+                        case "Integer":
+                            category = ColumnCategory.Integer;
+                            break;
+                        case "DoubleInteger":
+                            category = ColumnCategory.DoubleInteger;
+                            break;
+                        case "TimeDate":
+                            category = ColumnCategory.TimeDate;
+                            break;
+                        case "Identifier":
+                            category = ColumnCategory.Identifier;
+                            break;
+                        case "Property":
+                            category = ColumnCategory.Property;
+                            break;
+                        case "Filename":
+                            category = ColumnCategory.Filename;
+                            break;
+                        case "WildCardFilename":
+                            category = ColumnCategory.WildCardFilename;
+                            break;
+                        case "Path":
+                            category = ColumnCategory.Path;
+                            break;
+                        case "Paths":
+                            category = ColumnCategory.Paths;
+                            break;
+                        case "AnyPath":
+                            category = ColumnCategory.AnyPath;
+                            break;
+                        case "DefaultDir":
+                            category = ColumnCategory.DefaultDir;
+                            break;
+                        case "RegPath":
+                            category = ColumnCategory.RegPath;
+                            break;
+                        case "Formatted":
+                            category = ColumnCategory.Formatted;
+                            break;
+                        case "FormattedSddl":
+                            category = ColumnCategory.FormattedSDDLText;
+                            break;
+                        case "Template":
+                            category = ColumnCategory.Template;
+                            break;
+                        case "Condition":
+                            category = ColumnCategory.Condition;
+                            break;
+                        case "Guid":
+                            category = ColumnCategory.Guid;
+                            break;
+                        case "Version":
+                            category = ColumnCategory.Version;
+                            break;
+                        case "Language":
+                            category = ColumnCategory.Language;
+                            break;
+                        case "Binary":
+                            category = ColumnCategory.Binary;
+                            break;
+                        case "CustomSource":
+                            category = ColumnCategory.CustomSource;
+                            break;
+                        case "Cabinet":
+                            category = ColumnCategory.Cabinet;
+                            break;
+                        case "Shortcut":
+                            category = ColumnCategory.Shortcut;
+                            break;
+                        default:
+                            break;
+                    }
+                }
+
+                var keyTable = keyTables?[i];
+                var setValue = sets?[i];
+                var description = descriptions?[i];
+                var modString = modularizations?[i];
+                var modularization = ColumnModularizeType.None;
+
+                switch (modString)
+                {
+                    case null:
+                    case "None":
+                        modularization = ColumnModularizeType.None;
+                        break;
+                    case "Column":
+                        modularization = ColumnModularizeType.Column;
+                        break;
+                    case "Property":
+                        modularization = ColumnModularizeType.Property;
+                        break;
+                    case "Condition":
+                        modularization = ColumnModularizeType.Condition;
+                        break;
+                    case "CompanionFile":
+                        modularization = ColumnModularizeType.CompanionFile;
+                        break;
+                    case "SemicolonDelimited":
+                        modularization = ColumnModularizeType.SemicolonDelimited;
+                        break;
+                }
+
+                var columnDefinition = new ColumnDefinition(name, type, length, primaryKey, nullable, category, minValue, maxValue, keyTable, keyColumn, setValue, description, modularization, ColumnType.Localized == type, true);
+                columns.Add(columnDefinition);
+            }
+
+            var customTable = new TableDefinition(row.Id.Id, columns/*, unreal: bootstrapperApplicationData, bootstrapperApplicationData*/);
+            return customTable;
+        }
+    }
+}
diff --git a/src/WixToolset.Core.WindowsInstaller/Unbind/UnbindDatabaseCommand.cs b/src/WixToolset.Core.WindowsInstaller/Unbind/UnbindDatabaseCommand.cs
index e671f6a1..0699199b 100644
--- a/src/WixToolset.Core.WindowsInstaller/Unbind/UnbindDatabaseCommand.cs
+++ b/src/WixToolset.Core.WindowsInstaller/Unbind/UnbindDatabaseCommand.cs
@@ -271,7 +271,7 @@ namespace WixToolset.Core.WindowsInstaller.Unbind
                                     }
                                 }
 
-                                var tableDefinition = new TableDefinition(tableName, columns, false, false);
+                                var tableDefinition = new TableDefinition(tableName, columns, false);
 
                                 // use our table definitions if core properties are the same; this allows us to take advantage
                                 // of wix concepts like localizable columns which current code assumes