Skip to content

Commit 361cc0d

Browse files
jschick04jschick04
committed
Fixed CI issue related to needs upgrade checks
1 parent 64dc8ce commit 361cc0d

File tree

12 files changed

+955
-296
lines changed

12 files changed

+955
-296
lines changed

‎src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs‎

Lines changed: 95 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using EventLogExpert.Eventing.Providers;
77
using Microsoft.Extensions.DependencyInjection;
88
using System.CommandLine;
9+
using System.Text.RegularExpressions;
910

1011
namespace EventLogExpert.EventDbTool;
1112

@@ -83,50 +84,120 @@ private void CreateDatabase(string path, string? source, string? filter, string?
8384
return;
8485
}
8586

86-
if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
87+
if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { return; }
8788

88-
HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
89+
if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
8990

90-
if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
91+
try
9192
{
92-
if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
93+
HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
9394

94-
foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
95+
if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
9596
{
96-
skipProviderNames.Add(name);
97+
if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
98+
99+
foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
100+
{
101+
skipProviderNames.Add(name);
102+
}
103+
104+
Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
97105
}
98106

99-
Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
100-
}
107+
// Stream details directly into the DbContext. For .evtx sources, scanning the file
108+
// is expensive, so we deliberately do NOT pre-scan provider names just to size the
109+
// log header — that would cause a second full pass over the .evtx. Instead we
110+
// buffer the first batch of resolved details, derive the header column width from
111+
// those names, then log the header + buffered rows and continue streaming.
112+
const int batchSize = 100;
113+
var count = 0;
114+
var headerLogged = false;
115+
var pendingForHeader = new List<ProviderDetails>(batchSize);
116+
117+
// Defer creating the DbContext (and therefore the .db file on disk) until we have
118+
// at least one provider to persist. This prevents leaving an empty database behind
119+
// when no provider details could be resolved (e.g., .evtx without LocaleMetaData).
120+
EventProviderDbContext? dbContext = null;
121+
122+
try
123+
{
124+
IEnumerable<ProviderDetails> providersToAdd = source is null
125+
? LoadLocalProviders(regex, skipProviderNames)
126+
: ProviderSource.LoadProviders(source, Logger, regex, skipProviderNames);
127+
128+
foreach (var details in providersToAdd)
129+
{
130+
if (!headerLogged)
131+
{
132+
pendingForHeader.Add(details);
133+
134+
if (pendingForHeader.Count < batchSize) { continue; }
135+
136+
FlushHeaderAndBuffer(ref dbContext, path, pendingForHeader, ref count);
137+
headerLogged = true;
138+
continue;
139+
}
140+
141+
dbContext ??= new EventProviderDbContext(path, false, Logger);
142+
dbContext.ProviderDetails.Add(details);
143+
LogProviderDetails(details);
144+
count++;
145+
146+
if (count % batchSize != 0) { continue; }
147+
dbContext.SaveChanges();
148+
dbContext.ChangeTracker.Clear();
149+
}
150+
151+
// Flush any buffered providers when the stream ended before the buffer filled.
152+
if (!headerLogged && pendingForHeader.Count > 0)
153+
{
154+
FlushHeaderAndBuffer(ref dbContext, path, pendingForHeader, ref count);
155+
}
101156

102-
IEnumerable<ProviderDetails> providersToAdd = source is null
103-
? LoadLocalProviders(filter, skipProviderNames)
104-
: ProviderSource.LoadProviders(source, Logger, filter, skipProviderNames);
157+
if (dbContext is null)
158+
{
159+
Logger.Warn($"No provider details could be resolved from the source. Database was not created.");
105160

106-
var providersNotSkipped = providersToAdd.ToList();
161+
return;
162+
}
107163

108-
if (providersNotSkipped.Count == 0)
164+
Logger.Info($"");
165+
Logger.Info($"Saving database. Please wait...");
166+
167+
dbContext.SaveChanges();
168+
169+
Logger.Info($"Done!");
170+
}
171+
finally
172+
{
173+
dbContext?.Dispose();
174+
}
175+
}
176+
catch (RegexMatchTimeoutException)
109177
{
110-
Logger.Warn($"No providers to add to the new database.");
111-
return;
178+
Logger.Error($"The --filter regex timed out. The pattern may cause catastrophic backtracking.");
112179
}
180+
}
113181

114-
using var dbContext = new EventProviderDbContext(path, false, Logger);
182+
private void FlushHeaderAndBuffer(
183+
ref EventProviderDbContext? dbContext,
184+
string path,
185+
List<ProviderDetails> buffer,
186+
ref int count)
187+
{
188+
LogProviderDetailHeader(buffer.Select(p => p.ProviderName));
115189

116-
LogProviderDetailHeader(providersNotSkipped.Select(p => p.ProviderName));
190+
dbContext ??= new EventProviderDbContext(path, false, Logger);
117191

118-
foreach (var details in providersNotSkipped)
192+
foreach (var details in buffer)
119193
{
120194
dbContext.ProviderDetails.Add(details);
121195
LogProviderDetails(details);
196+
count++;
122197
}
123198

124-
Logger.Info($"");
125-
Logger.Info($"Saving database. Please wait...");
126-
127199
dbContext.SaveChanges();
128-
129-
Logger.Info($"Done!");
200+
dbContext.ChangeTracker.Clear();
201+
buffer.Clear();
130202
}
131-
132203
}

‎src/EventLogExpert.EventDbTool/DbToolCommand.cs‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,29 @@ public class DbToolCommand(ITraceLogger logger)
1414

1515
protected ITraceLogger Logger => logger;
1616

17-
protected static List<string> GetLocalProviderNames(string? filter)
17+
protected static List<string> GetLocalProviderNames(string? filter, ITraceLogger logger) =>
18+
!RegexHelper.TryCreate(filter, logger, out var regex) ? [] : GetLocalProviderNames(regex);
19+
20+
protected static List<string> GetLocalProviderNames(Regex? regex)
1821
{
1922
var providers = new List<string>(EventLogSession.GlobalSession.GetProviderNames().Distinct().OrderBy(name => name));
2023

21-
if (string.IsNullOrEmpty(filter)) { return providers; }
24+
return regex is null ? providers : providers.Where(p => regex.IsMatch(p)).ToList();
25+
}
2226

23-
var regex = new Regex(filter, RegexOptions.IgnoreCase);
24-
providers = providers.Where(p => regex.IsMatch(p)).ToList();
27+
protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
28+
{
29+
if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { yield break; }
2530

26-
return providers;
31+
foreach (var details in LoadLocalProviders(regex, skipProviderNames))
32+
{
33+
yield return details;
34+
}
2735
}
2836

29-
protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
37+
protected IEnumerable<ProviderDetails> LoadLocalProviders(Regex? regex, IReadOnlySet<string>? skipProviderNames = null)
3038
{
31-
foreach (var providerName in GetLocalProviderNames(filter))
39+
foreach (var providerName in GetLocalProviderNames(regex))
3240
{
3341
// Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we
3442
// are about to discard (e.g. when --skip-providers-in-file lists most local providers).

‎src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs‎

Lines changed: 42 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -79,43 +79,58 @@ private void DiffDatabase(string firstSource, string secondSource, string newDb)
7979

8080
var providersCopied = new List<ProviderDetails>();
8181

82-
using var newDbContext = new EventProviderDbContext(newDb, false, Logger);
82+
// Pass firstProviderNames as the skip set so providers present in the first source are
83+
// never resolved from the second source's metadata path. This is especially important when
84+
// the second source is .evtx+MTA, where each provider triggers an expensive load.
85+
Logger.Info($"Skipping up to {firstProviderNames.Count} provider name(s) from the second source that also appear in the first source.");
8386

84-
foreach (var details in ProviderSource.LoadProviders(secondSource, Logger))
87+
// Defer creating the DbContext (and therefore the .db file on disk) until at least one
88+
// provider is actually about to be persisted. This prevents leaving an empty database
89+
// behind when the second source yields no new providers.
90+
EventProviderDbContext? newDbContext = null;
91+
92+
try
8593
{
86-
if (firstProviderNames.Contains(details.ProviderName))
94+
foreach (var details in ProviderSource.LoadProviders(secondSource, Logger, filter: null, skipProviderNames: firstProviderNames))
8795
{
88-
Logger.Info($"Skipping {details.ProviderName} because it is present in both sources.");
89-
continue;
96+
Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
97+
98+
newDbContext ??= new EventProviderDbContext(newDb, false, Logger);
99+
100+
newDbContext.ProviderDetails.Add(new ProviderDetails
101+
{
102+
ProviderName = details.ProviderName,
103+
Events = details.Events,
104+
Parameters = details.Parameters,
105+
Keywords = details.Keywords,
106+
Messages = details.Messages,
107+
Opcodes = details.Opcodes,
108+
Tasks = details.Tasks
109+
});
110+
111+
providersCopied.Add(details);
90112
}
91113

92-
Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
93-
94-
newDbContext.ProviderDetails.Add(new ProviderDetails
114+
if (newDbContext is null)
95115
{
96-
ProviderName = details.ProviderName,
97-
Events = details.Events,
98-
Parameters = details.Parameters,
99-
Keywords = details.Keywords,
100-
Messages = details.Messages,
101-
Opcodes = details.Opcodes,
102-
Tasks = details.Tasks
103-
});
104-
105-
providersCopied.Add(details);
106-
}
107-
108-
newDbContext.SaveChanges();
116+
Logger.Warn($"No providers in the second source are missing from the first. Database was not created.");
117+
return;
118+
}
109119

110-
if (providersCopied.Count <= 0) { return; }
120+
newDbContext.SaveChanges();
111121

112-
Logger.Info($"Providers copied to new database:");
113-
Logger.Info($"");
114-
LogProviderDetailHeader(providersCopied.Select(p => p.ProviderName));
122+
Logger.Info($"Providers copied to new database:");
123+
Logger.Info($"");
124+
LogProviderDetailHeader(providersCopied.Select(p => p.ProviderName));
115125

116-
foreach (var provider in providersCopied)
126+
foreach (var provider in providersCopied)
127+
{
128+
LogProviderDetails(provider);
129+
}
130+
}
131+
finally
117132
{
118-
LogProviderDetails(provider);
133+
newDbContext?.Dispose();
119134
}
120135
}
121136
}

0 commit comments

Comments
 (0)