Skip to content

Commit a80dc86

Browse files
jschick04jschick04
committed
Fixed CI issue related to needs upgrade checks
1 parent 64dc8ce commit a80dc86

File tree

12 files changed

+853
-272
lines changed

12 files changed

+853
-272
lines changed

‎src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs‎

Lines changed: 73 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using EventLogExpert.Eventing.Providers;
77
using Microsoft.Extensions.DependencyInjection;
88
using System.CommandLine;
9+
using System.Text.RegularExpressions;
910

1011
namespace EventLogExpert.EventDbTool;
1112

@@ -83,50 +84,94 @@ private void CreateDatabase(string path, string? source, string? filter, string?
8384
return;
8485
}
8586

86-
if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
87+
if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { return; }
8788

88-
HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
89+
if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
8990

90-
if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
91+
try
9192
{
92-
if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
93+
HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
9394

94-
foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
95+
if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
9596
{
96-
skipProviderNames.Add(name);
97+
if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
98+
99+
foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
100+
{
101+
skipProviderNames.Add(name);
102+
}
103+
104+
Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
97105
}
98106

99-
Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
100-
}
107+
// Load provider names first (cheap string-only query) for the empty check and header
108+
// formatting. This avoids materializing all ProviderDetails (with large compressed
109+
// payloads) just to compute the column widths.
110+
IReadOnlyList<string> providerNames = source is null
111+
? GetLocalProviderNames(regex)
112+
.Where(n => !skipProviderNames.Contains(n)).ToList()
113+
: ProviderSource.LoadProviderNames(source, Logger, regex)
114+
.Where(n => !skipProviderNames.Contains(n)).ToList();
101115

102-
IEnumerable<ProviderDetails> providersToAdd = source is null
103-
? LoadLocalProviders(filter, skipProviderNames)
104-
: ProviderSource.LoadProviders(source, Logger, filter, skipProviderNames);
116+
if (providerNames.Count == 0)
117+
{
118+
Logger.Warn($"No providers to add to the new database.");
119+
return;
120+
}
105121

106-
var providersNotSkipped = providersToAdd.ToList();
122+
LogProviderDetailHeader(providerNames);
107123

108-
if (providersNotSkipped.Count == 0)
109-
{
110-
Logger.Warn($"No providers to add to the new database.");
111-
return;
112-
}
124+
// Defer creating the DbContext (and therefore the .db file on disk) until we have
125+
// at least one provider to persist. This prevents leaving an empty database behind
126+
// when no provider details could be resolved (e.g., .evtx without LocaleMetaData).
127+
EventProviderDbContext? dbContext = null;
128+
129+
try
130+
{
131+
// Stream details directly into the DbContext. Batch saves prevent the change tracker
132+
// from accumulating all entities in memory at once.
133+
const int batchSize = 100;
134+
var count = 0;
113135

114-
using var dbContext = new EventProviderDbContext(path, false, Logger);
136+
IEnumerable<ProviderDetails> providersToAdd = source is null
137+
? LoadLocalProviders(regex, skipProviderNames)
138+
: ProviderSource.LoadProviders(source, Logger, regex, skipProviderNames);
115139

116-
LogProviderDetailHeader(providersNotSkipped.Select(p => p.ProviderName));
140+
foreach (var details in providersToAdd)
141+
{
142+
dbContext ??= new EventProviderDbContext(path, false, Logger);
143+
dbContext.ProviderDetails.Add(details);
144+
LogProviderDetails(details);
145+
count++;
117146

118-
foreach (var details in providersNotSkipped)
119-
{
120-
dbContext.ProviderDetails.Add(details);
121-
LogProviderDetails(details);
122-
}
147+
if (count % batchSize != 0) { continue; }
148+
dbContext.SaveChanges();
149+
dbContext.ChangeTracker.Clear();
150+
}
123151

124-
Logger.Info($"");
125-
Logger.Info($"Saving database. Please wait...");
152+
if (dbContext is null)
153+
{
154+
Logger.Warn($"No provider details could be resolved from the source. Database was not created.");
126155

127-
dbContext.SaveChanges();
156+
return;
157+
}
128158

129-
Logger.Info($"Done!");
159+
Logger.Info($"");
160+
Logger.Info($"Saving database. Please wait...");
161+
162+
dbContext.SaveChanges();
163+
164+
Logger.Info($"Done!");
165+
}
166+
finally
167+
{
168+
dbContext?.Dispose();
169+
}
170+
}
171+
catch (RegexMatchTimeoutException)
172+
{
173+
Logger.Error($"The --filter regex timed out. The pattern may cause catastrophic backtracking.");
174+
}
130175
}
131176

132177
}

‎src/EventLogExpert.EventDbTool/DbToolCommand.cs‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,29 @@ public class DbToolCommand(ITraceLogger logger)
1414

1515
protected ITraceLogger Logger => logger;
1616

17-
protected static List<string> GetLocalProviderNames(string? filter)
17+
protected static List<string> GetLocalProviderNames(string? filter, ITraceLogger logger) =>
18+
!RegexHelper.TryCreate(filter, logger, out var regex) ? [] : GetLocalProviderNames(regex);
19+
20+
protected static List<string> GetLocalProviderNames(Regex? regex)
1821
{
1922
var providers = new List<string>(EventLogSession.GlobalSession.GetProviderNames().Distinct().OrderBy(name => name));
2023

21-
if (string.IsNullOrEmpty(filter)) { return providers; }
24+
return regex is null ? providers : providers.Where(p => regex.IsMatch(p)).ToList();
25+
}
2226

23-
var regex = new Regex(filter, RegexOptions.IgnoreCase);
24-
providers = providers.Where(p => regex.IsMatch(p)).ToList();
27+
protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
28+
{
29+
if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { yield break; }
2530

26-
return providers;
31+
foreach (var details in LoadLocalProviders(regex, skipProviderNames))
32+
{
33+
yield return details;
34+
}
2735
}
2836

29-
protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
37+
protected IEnumerable<ProviderDetails> LoadLocalProviders(Regex? regex, IReadOnlySet<string>? skipProviderNames = null)
3038
{
31-
foreach (var providerName in GetLocalProviderNames(filter))
39+
foreach (var providerName in GetLocalProviderNames(regex))
3240
{
3341
// Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we
3442
// are about to discard (e.g. when --skip-providers-in-file lists most local providers).

‎src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs‎

Lines changed: 42 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -79,43 +79,58 @@ private void DiffDatabase(string firstSource, string secondSource, string newDb)
7979

8080
var providersCopied = new List<ProviderDetails>();
8181

82-
using var newDbContext = new EventProviderDbContext(newDb, false, Logger);
82+
// Pass firstProviderNames as the skip set so providers present in the first source are
83+
// never resolved from the second source's metadata path. This is especially important when
84+
// the second source is .evtx+MTA, where each provider triggers an expensive load.
85+
Logger.Info($"Skipping up to {firstProviderNames.Count} provider name(s) from the second source that also appear in the first source.");
8386

84-
foreach (var details in ProviderSource.LoadProviders(secondSource, Logger))
87+
// Defer creating the DbContext (and therefore the .db file on disk) until at least one
88+
// provider is actually about to be persisted. This prevents leaving an empty database
89+
// behind when the second source yields no new providers.
90+
EventProviderDbContext? newDbContext = null;
91+
92+
try
8593
{
86-
if (firstProviderNames.Contains(details.ProviderName))
94+
foreach (var details in ProviderSource.LoadProviders(secondSource, Logger, filter: null, skipProviderNames: firstProviderNames))
8795
{
88-
Logger.Info($"Skipping {details.ProviderName} because it is present in both sources.");
89-
continue;
96+
Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
97+
98+
newDbContext ??= new EventProviderDbContext(newDb, false, Logger);
99+
100+
newDbContext.ProviderDetails.Add(new ProviderDetails
101+
{
102+
ProviderName = details.ProviderName,
103+
Events = details.Events,
104+
Parameters = details.Parameters,
105+
Keywords = details.Keywords,
106+
Messages = details.Messages,
107+
Opcodes = details.Opcodes,
108+
Tasks = details.Tasks
109+
});
110+
111+
providersCopied.Add(details);
90112
}
91113

92-
Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
93-
94-
newDbContext.ProviderDetails.Add(new ProviderDetails
114+
if (newDbContext is null)
95115
{
96-
ProviderName = details.ProviderName,
97-
Events = details.Events,
98-
Parameters = details.Parameters,
99-
Keywords = details.Keywords,
100-
Messages = details.Messages,
101-
Opcodes = details.Opcodes,
102-
Tasks = details.Tasks
103-
});
104-
105-
providersCopied.Add(details);
106-
}
107-
108-
newDbContext.SaveChanges();
116+
Logger.Warn($"No providers in the second source are missing from the first. Database was not created.");
117+
return;
118+
}
109119

110-
if (providersCopied.Count <= 0) { return; }
120+
newDbContext.SaveChanges();
111121

112-
Logger.Info($"Providers copied to new database:");
113-
Logger.Info($"");
114-
LogProviderDetailHeader(providersCopied.Select(p => p.ProviderName));
122+
Logger.Info($"Providers copied to new database:");
123+
Logger.Info($"");
124+
LogProviderDetailHeader(providersCopied.Select(p => p.ProviderName));
115125

116-
foreach (var provider in providersCopied)
126+
foreach (var provider in providersCopied)
127+
{
128+
LogProviderDetails(provider);
129+
}
130+
}
131+
finally
117132
{
118-
LogProviderDetails(provider);
133+
newDbContext?.Dispose();
119134
}
120135
}
121136
}

‎src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs‎

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -84,12 +84,25 @@ private void MergeDatabase(string source, string targetFile, bool overwriteProvi
8484
sourceProviders.Select(p => p.ProviderName),
8585
StringComparer.OrdinalIgnoreCase);
8686

87-
var targetMatchingNames = targetContext.ProviderDetails
88-
.AsNoTracking()
89-
.Select(p => p.ProviderName)
90-
.ToList()
91-
.Where(n => sourceNames.Contains(n))
92-
.ToList();
87+
// Query the overlap in the database by chunking sourceNames into IN-clause batches,
88+
// rather than pulling every target ProviderName into memory. Same chunk size as the
89+
// delete loop below to stay below SQLite's default parameter limit (999).
90+
var sourceNamesList = sourceNames.ToList();
91+
var targetMatchingNames = new List<string>();
92+
93+
for (var offset = 0; offset < sourceNamesList.Count; offset += ProviderSource.MaxInClauseParameters)
94+
{
95+
var chunk = sourceNamesList
96+
.Skip(offset)
97+
.Take(ProviderSource.MaxInClauseParameters)
98+
.ToList();
99+
100+
targetMatchingNames.AddRange(
101+
targetContext.ProviderDetails
102+
.AsNoTracking()
103+
.Where(p => chunk.Contains(p.ProviderName))
104+
.Select(p => p.ProviderName));
105+
}
93106

94107
// Track the source-side provider names whose case-insensitive equivalent exists in target.
95108
// ProviderName is the primary key in the target DB, so case-sensitive uniqueness identifies
@@ -104,15 +117,24 @@ private void MergeDatabase(string source, string targetFile, bool overwriteProvi
104117
{
105118
Logger.Info($"Removing these providers from the target database...");
106119

107-
// Single round-trip to load just the rows we need to remove. Since these names came
108-
// from the same DB, exact (binary) matching here is correct.
109-
var toRemove = targetContext.ProviderDetails
110-
.Where(p => targetMatchingNames.Contains(p.ProviderName))
111-
.ToList();
120+
// Chunk the IN-clause to stay below SQLite's parameter limit (default 999). Without
121+
// chunking, an --overwrite of a large overlap could throw at runtime.
122+
// ExecuteDelete() issues a SQL DELETE directly, avoiding change-tracker overhead.
123+
var removed = 0;
124+
125+
for (var offset = 0; offset < targetMatchingNames.Count; offset += ProviderSource.MaxInClauseParameters)
126+
{
127+
var chunk = targetMatchingNames
128+
.Skip(offset)
129+
.Take(ProviderSource.MaxInClauseParameters)
130+
.ToList();
131+
132+
removed += targetContext.ProviderDetails
133+
.Where(p => chunk.Contains(p.ProviderName))
134+
.ExecuteDelete();
135+
}
112136

113-
targetContext.RemoveRange(toRemove);
114-
targetContext.SaveChanges();
115-
Logger.Info($"Removal of {toRemove.Count} provider row(s) completed.");
137+
Logger.Info($"Removal of {removed} provider row(s) completed.");
116138
}
117139
else
118140
{

0 commit comments

Comments
 (0)