Updated dbtool to support MTA files

Author: jschick04

src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs

@@ -1,4 +1,4 @@
-// // Copyright (c) Microsoft Corporation.
+// // Copyright (c) Microsoft Corporation.
 // // Licensed under the MIT License.
 
 using EventLogExpert.Eventing.EventProviderDatabase;
@@ -20,6 +20,14 @@ public static Command GetCommand()
             Description = "File to create. Must have a .db extension."
         };
 
+        Argument<string?> sourceArgument = new("source")
+        {
+            Description = "Optional provider source: a .db file, an exported .evtx file, or a folder containing " +
+                ".db and/or .evtx files (top-level only). When omitted, local providers on this machine are used. " +
+                "When supplied, ONLY the source is used (no fallback to local providers).",
+            Arity = ArgumentArity.ZeroOrOne
+        };
+
         Option<string> filterOption = new("--filter")
         {
             Description = "Only providers matching specified regex string will be added to the database."
@@ -28,7 +36,8 @@ public static Command GetCommand()
         Option<string> skipProvidersInFileOption = new("--skip-providers-in-file")
         {
             Description =
-                "Any providers found in the specified database file will not be included in the new database. " +
+                "Any providers found in the specified source (a .db file, an exported .evtx file, or a folder " +
+                "containing them, top-level only) will not be included in the new database. " +
                 "For example, when creating a database of event providers for Exchange Server, it may be useful " +
                 "to provide a database of all providers from a fresh OS install with no other products. That way, all the " +
                 "OS providers are skipped, and only providers added by Exchange or other installed products " +
@@ -41,6 +50,7 @@ public static Command GetCommand()
         };
 
         createDatabaseCommand.Arguments.Add(fileArgument);
+        createDatabaseCommand.Arguments.Add(sourceArgument);
         createDatabaseCommand.Options.Add(filterOption);
         createDatabaseCommand.Options.Add(skipProvidersInFileOption);
         createDatabaseCommand.Options.Add(verboseOption);
@@ -51,76 +61,63 @@ public static Command GetCommand()
             new CreateDatabaseCommand(sp.GetRequiredService<ITraceLogger>())
                 .CreateDatabase(
                     result.GetRequiredValue(fileArgument),
+                    result.GetValue(sourceArgument),
                     result.GetValue(filterOption),
                     result.GetValue(skipProvidersInFileOption));
         });
 
         return createDatabaseCommand;
     }
 
-    private void CreateDatabase(string path, string? filter, string? skipProvidersInFile)
+    private void CreateDatabase(string path, string? source, string? filter, string? skipProvidersInFile)
     {
         if (File.Exists(path))
         {
             Logger.Error($"Cannot create database because file already exists: {path}");
             return;
         }
 
-        if (Path.GetExtension(path) != ".db")
+        if (!string.Equals(Path.GetExtension(path), ".db", StringComparison.OrdinalIgnoreCase))
         {
             Logger.Error($"File extension must be .db.");
             return;
         }
 
-        HashSet<string> skipProviderNames = [];
+        if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
+
+        HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
 
         if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
         {
-            if (!File.Exists(skipProvidersInFile))
-            {
-                Logger.Error($"File not found: {skipProvidersInFile}");
-                return;
-            }
-
-            using var skipDbContext = new EventProviderDbContext(skipProvidersInFile, true, Logger);
+            if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
 
-            foreach (var provider in skipDbContext.ProviderDetails)
+            foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
             {
-                skipProviderNames.Add(provider.ProviderName);
+                skipProviderNames.Add(name);
             }
 
-            Logger.Info($"Found {skipProviderNames.Count} providers in file {skipProvidersInFile}. These will not be included in the new database.");
+            Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
         }
 
-        var providerNames = GetLocalProviderNames(filter);
+        IEnumerable<ProviderDetails> providersToAdd = source is null
+            ? LoadLocalProviders(filter, skipProviderNames)
+            : ProviderSource.LoadProviders(source, Logger, filter, skipProviderNames);
 
-        if (!providerNames.Any())
-        {
-            Logger.Warn($"No providers found matching filter {filter}.");
-            return;
-        }
-
-        var providerNamesNotSkipped = providerNames.Where(name => !skipProviderNames.Contains(name)).ToList();
+        var providersNotSkipped = providersToAdd.ToList();
 
-        var numberSkipped = providerNames.Count - providerNamesNotSkipped.Count;
-
-        if (numberSkipped > 0)
+        if (providersNotSkipped.Count == 0)
         {
-            Logger.Info($"{numberSkipped} providers were skipped due to being present in the specified database.");
+            Logger.Warn($"No providers to add to the new database.");
+            return;
         }
 
         using var dbContext = new EventProviderDbContext(path, false, Logger);
 
-        LogProviderDetailHeader(providerNamesNotSkipped);
+        LogProviderDetailHeader(providersNotSkipped.Select(p => p.ProviderName));
 
-        foreach (var providerName in providerNamesNotSkipped)
+        foreach (var details in providersNotSkipped)
         {
-            var provider = new EventMessageProvider(providerName, Logger);
-
-            var details = provider.LoadProviderDetails();
-
             dbContext.ProviderDetails.Add(details);
-
             LogProviderDetails(details);
         }
 
@@ -131,4 +128,5 @@ private void CreateDatabase(string path, string? filter, string? skipProvidersIn
 
         Logger.Info($"Done!");
     }
+
 }

src/EventLogExpert.EventDbTool/DbToolCommand.cs

@@ -26,6 +26,18 @@ protected static List<string> GetLocalProviderNames(string? filter)
         return providers;
     }
 
+    protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
+    {
+        foreach (var providerName in GetLocalProviderNames(filter))
+        {
+            // Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we
+            // are about to discard (e.g. when --skip-providers-in-file lists most local providers).
+            if (skipProviderNames is not null && skipProviderNames.Contains(providerName)) { continue; }
+
+            yield return new EventMessageProvider(providerName, Logger).LoadProviderDetails();
+        }
+    }
+
     protected void LogProviderDetailHeader(IEnumerable<string> providerNames)
     {
         var maxNameLength = providerNames.Any() ? providerNames.Max(p => p.Length) : 14;

src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs

@@ -15,31 +15,31 @@ public static Command GetCommand()
     {
         Command diffDatabaseCommand = new(
             "diff",
-            "Given two databases, produces a third database containing all providers " +
-            "from the second database which are not in the first database.");
+            "Given two provider sources (each may be a .db, an exported .evtx, or a folder containing them), " +
+            "produces a database containing all providers from the second source which are not in the first source.");
 
-        Argument<string> dbOneArgument = new("first db")
+        Argument<string> firstArgument = new("first source")
         {
-            Description = "The first database to compare."
+            Description = "The first source to compare: a .db, an exported .evtx, or a folder containing .db and/or .evtx files (top-level only)."
         };
 
-        Argument<string> dbTwoArgument = new("second db")
+        Argument<string> secondArgument = new("second source")
         {
-            Description = "The second database to compare."
+            Description = "The second source to compare: a .db, an exported .evtx, or a folder containing .db and/or .evtx files (top-level only)."
         };
 
         Argument<string> newDbArgument = new("new db")
         {
-            Description = "The new database containing only the providers in the second db which are not in the first db. Must have a .db extension."
+            Description = "The new database containing only the providers in the second source which are not in the first source. Must have a .db extension."
         };
 
         Option<bool> verboseOption = new("--verbose")
         {
             Description = "Verbose logging. May be useful for troubleshooting."
         };
 
-        diffDatabaseCommand.Arguments.Add(dbOneArgument);
-        diffDatabaseCommand.Arguments.Add(dbTwoArgument);
+        diffDatabaseCommand.Arguments.Add(firstArgument);
+        diffDatabaseCommand.Arguments.Add(secondArgument);
         diffDatabaseCommand.Arguments.Add(newDbArgument);
         diffDatabaseCommand.Options.Add(verboseOption);
 
@@ -48,76 +48,67 @@ public static Command GetCommand()
             using var sp = Program.BuildServiceProvider(action.GetValue(verboseOption));
             new DiffDatabaseCommand(sp.GetRequiredService<ITraceLogger>())
                 .DiffDatabase(
-                    action.GetRequiredValue(dbOneArgument),
-                    action.GetRequiredValue(dbTwoArgument),
+                    action.GetRequiredValue(firstArgument),
+                    action.GetRequiredValue(secondArgument),
                     action.GetRequiredValue(newDbArgument));
         });
 
         return diffDatabaseCommand;
     }
 
-    private void DiffDatabase(string dbOne, string dbTwo, string newDb)
+    private void DiffDatabase(string firstSource, string secondSource, string newDb)
     {
-        foreach (var path in new[] { dbOne, dbTwo })
-        {
-            if (File.Exists(path)) { continue; }
-
-            Logger.Error($"File not found: {path}");
-            return;
-        }
+        if (!ProviderSource.TryValidate(firstSource, Logger)) { return; }
+        if (!ProviderSource.TryValidate(secondSource, Logger)) { return; }
 
         if (File.Exists(newDb))
         {
             Logger.Error($"File already exists: {newDb}");
             return;
         }
 
-        if (Path.GetExtension(newDb) != ".db")
+        if (!string.Equals(Path.GetExtension(newDb), ".db", StringComparison.OrdinalIgnoreCase))
         {
             Logger.Error($"New db path must have a .db extension.");
             return;
         }
 
-        var dbOneProviderNames = new HashSet<string>();
-
-        using (var dbOneContext = new EventProviderDbContext(dbOne, true, Logger))
-        {
-            dbOneContext.ProviderDetails.Select(p => p.ProviderName).ToList()
-                .ForEach(name => dbOneProviderNames.Add(name));
-        }
+        var firstProviderNames = new HashSet<string>(
+            ProviderSource.LoadProviderNames(firstSource, Logger),
+            StringComparer.OrdinalIgnoreCase);
 
         var providersCopied = new List<ProviderDetails>();
 
-        using var dbTwoContext = new EventProviderDbContext(dbTwo, true, Logger);
         using var newDbContext = new EventProviderDbContext(newDb, false, Logger);
 
-        foreach (var details in dbTwoContext.ProviderDetails)
+        foreach (var details in ProviderSource.LoadProviders(secondSource, Logger))
         {
-            if (dbOneProviderNames.Contains(details.ProviderName))
+            if (firstProviderNames.Contains(details.ProviderName))
             {
-                Logger.Info($"Skipping {details.ProviderName} because it is present in both databases.");
+                Logger.Info($"Skipping {details.ProviderName} because it is present in both sources.");
+                continue;
             }
-            else
+
+            Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
+
+            newDbContext.ProviderDetails.Add(new ProviderDetails
             {
-                Logger.Info($"Copying {details.ProviderName} because it is present in second db but not first db.");
-                newDbContext.ProviderDetails.Add(new ProviderDetails
-                {
-                    ProviderName = details.ProviderName,
-                    Events = details.Events,
-                    Keywords = details.Keywords,
-                    Messages = details.Messages,
-                    Opcodes = details.Opcodes,
-                    Tasks = details.Tasks
-                });
-
-                providersCopied.Add(details);
-            }
+                ProviderName = details.ProviderName,
+                Events = details.Events,
+                Parameters = details.Parameters,
+                Keywords = details.Keywords,
+                Messages = details.Messages,
+                Opcodes = details.Opcodes,
+                Tasks = details.Tasks
+            });
+
+            providersCopied.Add(details);
         }
 
         newDbContext.SaveChanges();
 
         if (providersCopied.Count <= 0) { return; }
-        
+
         Logger.Info($"Providers copied to new database:");
         Logger.Info($"");
         LogProviderDetailHeader(providersCopied.Select(p => p.ProviderName));