This commit is contained in:
2026-02-13 15:46:30 +01:00
parent 85d42c0641
commit e13f7217ab
16 changed files with 759 additions and 1170 deletions

View File

@@ -51,8 +51,6 @@ namespace EonaCat.Logger.LogClient
} }
} }
public void Log(LogEntry entry) => LogAsync(entry).GetAwaiter().GetResult();
public async Task LogExceptionAsync(Exception ex, string message = "", public async Task LogExceptionAsync(Exception ex, string message = "",
Dictionary<string, object>? properties = null) Dictionary<string, object>? properties = null)
{ {

View File

@@ -37,7 +37,7 @@ namespace EonaCat.Logger.LogClient
entry.StackTrace = e.Exception.StackTrace; entry.StackTrace = e.Exception.StackTrace;
} }
_client.Log(entry); _client.LogAsync(entry).ConfigureAwait(false);
} }
private static LogLevel MapLogLevel(ELogType logType) private static LogLevel MapLogLevel(ELogType logType)

View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
<PackageReference Include="Serilog.Sinks.Async" Version="2.1.0" />
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
<PackageReference Include="Serilog.Sinks.Network" Version="3.0.0" />
<PackageReference Include="Serilog.Sinks.Seq" Version="9.0.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,197 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.DataProtection;
using Microsoft.Extensions.Hosting;
using Serilog;
using Serilog.Events;
using Serilog.Formatting.Json;
using System.Net.Sockets;
using System.Runtime.InteropServices;
using System.Text;
var builder = WebApplication.CreateBuilder(args);
//
// LOGGER CONFIGURATION (Equivalent to LoggerSettings)
//
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Verbose()
.Enrich.WithProperty("Id", "TEST")
.Enrich.WithProperty("AppName", "[JIJ BENT EEN BRASSER!]")
.WriteTo.Async(a => a.Console())
.WriteTo.Async(a => a.File(
path: "logs/web-.log",
rollingInterval: RollingInterval.Day,
fileSizeLimitBytes: 1_000_000,
rollOnFileSizeLimit: true,
retainedFileCountLimit: 5,
shared: true))
.WriteTo.Async(a => a.File(
new JsonFormatter(),
path: "logs/test.json",
rollingInterval: RollingInterval.Day))
//.WriteTo.Seq("http://localhost:5341") // central logging
.CreateLogger();
builder.Services.AddDataProtection()
.PersistKeysToFileSystem(new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), "keys")))
.SetApplicationName("SerilogStressTest");
builder.Services.AddRazorPages();
builder.WebHost.ConfigureKestrel(options =>
{
options.ListenAnyIP(6000);
});
var app = builder.Build();
app.UseHttpsRedirection();
app.UseStaticFiles();
app.UseRouting();
app.UseAuthorization();
app.MapRazorPages();
//
// ==============================
// 🔥 TESTS START HERE
// ==============================
//
_ = Task.Run(RunLoggingTestsAsync);
_ = Task.Run(RunWebLoggingTestsAsync);
_ = Task.Run(RunLoggingExceptionTests);
_ = Task.Run(RunWebLoggingExceptionTests);
//_ = Task.Run(RunMemoryLeakTest);
_ = Task.Run(RunTcpLoggerTest);
app.Run();
// =======================================================
// 1⃣ EXACT HIGH-SPEED FILE LOGGING LOOP
// =======================================================
async Task RunLoggingTestsAsync()
{
for (var i = 0; i < 9_000_000; i++)
{
Log.Information("test to file {i} INFO", i);
Log.Fatal("test to file {i} CRITICAL", i);
Log.Debug("test to file {i} DEBUG", i);
Log.Error("test to file {i} ERROR", i);
Log.Verbose("test to file {i} TRACE", i);
Log.Warning("test to file {i} WARNING", i);
Console.WriteLine($"Logged: {i}");
await Task.Delay(1);
}
}
// =======================================================
// 2⃣ WEB LOGGER STRESS TEST
// =======================================================
async Task RunWebLoggingTestsAsync()
{
int i = 0;
while (true)
{
i++;
Log.Information("web-test {i}", i);
Log.Debug("web-test {i}", i);
Log.Warning("web-test {i}", i);
Log.Error("web-test {i}", i);
Log.Verbose("web-test {i}", i);
await Task.Delay(1);
}
}
// =======================================================
// 3⃣ EXCEPTION TEST (FILE LOGGER)
// =======================================================
void RunLoggingExceptionTests()
{
for (int i = 0; i < 10; i++)
{
try
{
throw new Exception($"Normal Exception {i}");
}
catch (Exception ex)
{
Log.Error(ex, "Exception {Index}", i);
Console.WriteLine($"Normal ExceptionLogged: {i}");
}
}
}
// =======================================================
// 4⃣ WEB EXCEPTION TEST
// =======================================================
void RunWebLoggingExceptionTests()
{
for (int i = 0; i < 10; i++)
{
try
{
throw new Exception($"WebException {i}");
}
catch (Exception ex)
{
Log.Fatal(ex, "CRITICAL");
Log.Debug(ex, "DEBUG");
Log.Error(ex, "ERROR");
Log.Verbose(ex, "TRACE");
Log.Warning(ex, "WARNING");
Log.Information(ex, "INFORMATION");
Console.WriteLine($"WebExceptionLogged: {i}");
}
}
}
// =======================================================
// 6⃣ MEMORY LEAK TEST (IDENTICAL BEHAVIOR)
// =======================================================
async Task RunMemoryLeakTest()
{
var managedLeak = new List<byte[]>();
while (true)
{
managedLeak.Add(new byte[5_000_000]); // 5MB
Marshal.AllocHGlobal(10_000_000); // 10MB unmanaged
await Task.Delay(500);
}
}
// =======================================================
// 7⃣ TCP LOGGER TEST
// =======================================================
async Task RunTcpLoggerTest()
{
using var client = new TcpClient();
try
{
await client.ConnectAsync("192.168.1.1", 12345);
int i = 0;
while (true)
{
var message = Encoding.UTF8.GetBytes($"TCP log {++i}\n");
await client.GetStream().WriteAsync(message);
await Task.Delay(1000);
}
}
catch
{
Log.Warning("TCP server not reachable");
}
}

View File

@@ -0,0 +1,12 @@
{
"profiles": {
"EonaCat.Logger.SerilogTest": {
"commandName": "Project",
"launchBrowser": true,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"applicationUrl": "https://localhost:56815;http://localhost:56816"
}
}
}

View File

@@ -11,6 +11,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EonaCat.Logger.LogClient",
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EonaCat.Logger.Server", "EonaCat.Logger.Server\EonaCat.Logger.Server.csproj", "{E2AC1343-1D08-8FBF-0194-270CB862675F}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EonaCat.Logger.Server", "EonaCat.Logger.Server\EonaCat.Logger.Server.csproj", "{E2AC1343-1D08-8FBF-0194-270CB862675F}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EonaCat.Logger.SerilogTest", "EonaCat.Logger.SerilogTest\EonaCat.Logger.SerilogTest.csproj", "{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@@ -69,6 +71,18 @@ Global
{E2AC1343-1D08-8FBF-0194-270CB862675F}.Release|x64.Build.0 = Release|Any CPU {E2AC1343-1D08-8FBF-0194-270CB862675F}.Release|x64.Build.0 = Release|Any CPU
{E2AC1343-1D08-8FBF-0194-270CB862675F}.Release|x86.ActiveCfg = Release|Any CPU {E2AC1343-1D08-8FBF-0194-270CB862675F}.Release|x86.ActiveCfg = Release|Any CPU
{E2AC1343-1D08-8FBF-0194-270CB862675F}.Release|x86.Build.0 = Release|Any CPU {E2AC1343-1D08-8FBF-0194-270CB862675F}.Release|x86.Build.0 = Release|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Debug|x64.ActiveCfg = Debug|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Debug|x64.Build.0 = Debug|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Debug|x86.ActiveCfg = Debug|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Debug|x86.Build.0 = Debug|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Release|Any CPU.Build.0 = Release|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Release|x64.ActiveCfg = Release|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Release|x64.Build.0 = Release|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Release|x86.ActiveCfg = Release|Any CPU
{1EAB9752-2FDF-49CC-A9E3-7591375B4BF6}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE

View File

@@ -13,8 +13,8 @@
<Copyright>EonaCat (Jeroen Saey)</Copyright> <Copyright>EonaCat (Jeroen Saey)</Copyright>
<PackageTags>EonaCat;Logger;EonaCatLogger;Log;Writer;Jeroen;Saey</PackageTags> <PackageTags>EonaCat;Logger;EonaCatLogger;Log;Writer;Jeroen;Saey</PackageTags>
<PackageIconUrl /> <PackageIconUrl />
<Version>1.7.3</Version> <Version>1.7.4</Version>
<FileVersion>1.7.3</FileVersion> <FileVersion>1.7.4</FileVersion>
<PackageReadmeFile>README.md</PackageReadmeFile> <PackageReadmeFile>README.md</PackageReadmeFile>
<GenerateDocumentationFile>True</GenerateDocumentationFile> <GenerateDocumentationFile>True</GenerateDocumentationFile>
<PackageLicenseFile>LICENSE</PackageLicenseFile> <PackageLicenseFile>LICENSE</PackageLicenseFile>
@@ -25,7 +25,7 @@
</PropertyGroup> </PropertyGroup>
<PropertyGroup> <PropertyGroup>
<EVRevisionFormat>1.7.3+{chash:10}.{c:ymd}</EVRevisionFormat> <EVRevisionFormat>1.7.4+{chash:10}.{c:ymd}</EVRevisionFormat>
<EVDefault>true</EVDefault> <EVDefault>true</EVDefault>
<EVInfo>true</EVInfo> <EVInfo>true</EVInfo>
<EVTagMatch>v[0-9]*</EVTagMatch> <EVTagMatch>v[0-9]*</EVTagMatch>

View File

@@ -111,44 +111,6 @@ public class FileLoggerOptions : BatchingLoggerOptions
/// <returns></returns> /// <returns></returns>
public string LogDirectory { get; set; } = Path.Combine(DefaultPath, "logs"); public string LogDirectory { get; set; } = Path.Combine(DefaultPath, "logs");
/// <summary>
/// Determines if we need to mask certain keywords
/// </summary>
public bool UseMask { get; set; }
/// <summary>
/// Determines the keywords to mask
/// </summary>
public List<string> MaskedKeywords { get; set; } = new List<string>();
public string Mask { get; set; } = "***MASKED***";
/// <summary>
/// Determines that if masking is enabled we also need to use the default masking options:
/// IP addresses
/// MAC addresses
/// Emails
/// Passwords
/// Credit card numbers
/// Social security numbers (SSN) and BSN (Dutch Citizen Service Number)
/// API keys/tokens
/// Phone numbers (generic and Dutch specific)
/// Dates of birth (DOB) or other date formats
/// IBAN/Bank account numbers (generic and Dutch specific)
/// JWT tokens
/// URLs with sensitive query strings
/// License keys
/// Public and private keys (e.g., PEM format)
/// Dutch KVK number (8 or 12 digits)
/// Dutch BTW-nummer (VAT number)
/// Dutch driving license number (10-12 characters)
/// Dutch health insurance number (Zorgnummer)
/// Other Dutch Bank Account numbers (9-10 digits)
/// Dutch Passport Numbers (9 alphanumeric characters
/// Dutch Identification Document Numbers (varying formats)
/// Custom keywords specified in LoggerSettings
/// </summary>
public bool UseDefaultMasking { get; set; } = true;
/// <summary> /// <summary>
/// Determines if we need to include the correlation ID in the log (default: false) /// Determines if we need to include the correlation ID in the log (default: false)
/// </summary> /// </summary>

File diff suppressed because it is too large Load Diff

View File

@@ -18,7 +18,7 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
private readonly CancellationTokenSource _cts = new(); private readonly CancellationTokenSource _cts = new();
private readonly int _batchSize; private readonly int _batchSize;
private readonly long _maxQueueBytes = 512L * 1024 * 1024; // 512MB safety private readonly long _maxQueueBytes = 13L * 1024 * 1024; // 13MB safety - forces frequent flushes to disk
private long _currentQueueBytes; private long _currentQueueBytes;
private long _dropped; private long _dropped;
@@ -38,7 +38,6 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
if (o is FileLoggerOptions file) if (o is FileLoggerOptions file)
{ {
UseLocalTime = file.UseLocalTime; UseLocalTime = file.UseLocalTime;
UseMask = file.UseMask;
LoggerSettings = file.LoggerSettings; LoggerSettings = file.LoggerSettings;
} }
@@ -51,14 +50,13 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
} }
protected bool UseLocalTime { get; } protected bool UseLocalTime { get; }
protected bool UseMask { get; }
protected DateTimeOffset NowOffset => protected DateTimeOffset NowOffset =>
UseLocalTime ? DateTimeOffset.Now : DateTimeOffset.UtcNow; UseLocalTime ? DateTimeOffset.Now : DateTimeOffset.UtcNow;
protected LoggerSettings LoggerSettings protected LoggerSettings LoggerSettings
{ {
get => _loggerSettings ??= new LoggerSettings { UseLocalTime = UseLocalTime, UseMask = UseMask }; get => _loggerSettings ??= new LoggerSettings { UseLocalTime = UseLocalTime };
set => _loggerSettings = value; set => _loggerSettings = value;
} }
@@ -89,16 +87,10 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
} }
private LogMessage CreateLogMessage(string message, DateTimeOffset ts, string category, ELogType logLevel, string[] tags = null, LoggerSettings? settings = null) private LogMessage CreateLogMessage(string message, DateTimeOffset ts, string category, ELogType logLevel, string[] tags = null, LoggerSettings? settings = null)
{ {
var effectiveSettings = settings ?? LoggerSettings; var effectiveSettings = settings ?? LoggerSettings;
if (effectiveSettings.UseMask)
{
var masker = new SensitiveDataMasker(effectiveSettings);
message = masker.MaskSensitiveInformation(message);
}
return new LogMessage return new LogMessage
{ {
Message = message, Message = message,
@@ -175,6 +167,7 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
public void Dispose() public void Dispose()
{ {
// Make Dispose idempotent and ensure derived shutdown completes synchronously.
if (_disposed) if (_disposed)
{ {
return; return;
@@ -182,17 +175,33 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
_disposed = true; _disposed = true;
_cts.Cancel(); try
_worker.Join(TimeSpan.FromSeconds(5)); {
// Signal cancellation and wake the worker thread.
_cts.Cancel();
_signal.Set();
OnShutdownFlushAsync().ConfigureAwait(false); // Wait briefly for the worker to finish; thread is background so process won't hang indefinitely.
_worker.Join(TimeSpan.FromSeconds(5));
_cts.Dispose(); // Ensure derived classes flush synchronously before we dispose native resources.
_signal.Dispose(); OnShutdownFlushAsync().GetAwaiter().GetResult();
}
catch (Exception ex)
{
// Surface errors via event if available.
try { OnError?.Invoke(this, ex); } catch { }
}
finally
{
try { _cts.Dispose(); } catch { }
try { _signal.Dispose(); } catch { }
GC.SuppressFinalize(this);
}
} }
protected virtual async Task OnShutdownFlushAsync() protected virtual async Task OnShutdownFlushAsync()
{ {
// default: Do nothing // default: Do nothing
} }
} }

View File

@@ -1,100 +0,0 @@
using global::EonaCat.Logger.Managers;
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
namespace EonaCat.Logger.EonaCatCoreLogger.Internal
{
internal class SensitiveDataMasker
{
internal LoggerSettings LoggerSettings { get; }
internal SensitiveDataMasker(LoggerSettings settings)
{
LoggerSettings = settings;
}
// Precompiled Regexes
private static readonly Regex IpRegex = new Regex(@"\b(?:\d{1,3}\.){3}\d{1,3}\b(?!\d)", RegexOptions.Compiled);
private static readonly Regex MacRegex = new Regex(@"\b(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}\b", RegexOptions.Compiled);
private static readonly Regex EmailRegex = new Regex(@"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex PasswordRegex = new Regex(@"(?i)(password\s*[:= ]\s*|pwd\s*[:= ]\s*)[^\s]+", RegexOptions.Compiled);
private static readonly Regex DutchPasswordRegex = new Regex(@"(?i)(wachtwoord\s*[:= ]\s*|ww\s*=\s*)[^\s]+", RegexOptions.Compiled);
private static readonly Regex CreditCardRegex = new Regex(@"\b(?:\d{4}[ -]?){3}\d{4}\b", RegexOptions.Compiled);
private static readonly Regex SsnBsnRegex = new Regex(@"\b\d{3}-\d{2}-\d{4}\b|\b\d{9}\b", RegexOptions.Compiled);
private static readonly Regex ApiKeyRegex = new Regex(@"\b[A-Za-z0-9-_]{20,}\b", RegexOptions.Compiled);
private static readonly Regex PhoneGenericRegex = new Regex(@"\b(\+?\d{1,4}[-.\s]?)?(\(?\d{3}\)?[-.\s]?)?\d{3}[-.\s]?\d{4}\b", RegexOptions.Compiled);
private static readonly Regex PhoneDutchRegex = new Regex(@"\b(\+31|0031|0|06)[-\s]?\d{8}\b", RegexOptions.Compiled);
private static readonly Regex PhoneBelgiumRegex = new Regex(@"\b(\+32|0032|0|06)[-\s]?\d{8}\b", RegexOptions.Compiled);
private static readonly Regex DobRegex = new Regex(@"\b\d{2}[/-]\d{2}[/-]\d{4}\b", RegexOptions.Compiled);
private static readonly Regex PostalCodeDutchRegex = new Regex(@"\b\d{4}\s?[A-Z]{2}\b", RegexOptions.Compiled);
private static readonly Regex IbanGenericRegex = new Regex(@"\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b", RegexOptions.Compiled);
private static readonly Regex IbanDutchRegex = new Regex(@"\bNL\d{2}[A-Z]{4}\d{10}\b", RegexOptions.Compiled);
private static readonly Regex JwtRegex = new Regex(@"\b[A-Za-z0-9-_]{16,}\.[A-Za-z0-9-_]{16,}\.[A-Za-z0-9-_]{16,}\b", RegexOptions.Compiled);
private static readonly Regex UrlSensitiveRegex = new Regex(@"\bhttps?:\/\/[^\s?]+(\?[^\s]+?[\&=](password|key|token|wachtwoord|sleutel))[^&\s]*", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LicenseKeyRegex = new Regex(@"\b[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}\b", RegexOptions.Compiled);
private static readonly Regex PemKeyRegex = new Regex(@"-----BEGIN [A-Z ]+KEY-----[\s\S]+?-----END [A-Z ]+KEY-----", RegexOptions.Compiled);
private static readonly Regex KvkRegex = new Regex(@"\b\d{8}|\d{12}\b", RegexOptions.Compiled);
private static readonly Regex BtwRegex = new Regex(@"\bNL\d{9}B\d{2}\b", RegexOptions.Compiled);
private static readonly Regex DutchDrivingLicenseRegex = new Regex(@"\b[A-Z0-9]{10,12}\b", RegexOptions.Compiled);
private static readonly Regex ZorgnummerRegex = new Regex(@"\b\d{9}\b", RegexOptions.Compiled);
private static readonly Regex DutchBankAccountRegex = new Regex(@"\b\d{9,10}\b", RegexOptions.Compiled);
private static readonly Regex DutchPassportRegex = new Regex(@"\b[A-Z0-9]{9}\b", RegexOptions.Compiled);
private static readonly Regex DutchIdDocRegex = new Regex(@"\b[A-Z]{2}\d{6,7}\b", RegexOptions.Compiled);
internal virtual string MaskSensitiveInformation(string message)
{
if (string.IsNullOrEmpty(message))
{
return message;
}
if (LoggerSettings != null && LoggerSettings.UseDefaultMasking)
{
string mask = LoggerSettings.Mask ?? "***";
message = IpRegex.Replace(message, mask);
message = MacRegex.Replace(message, mask);
message = EmailRegex.Replace(message, mask);
message = PasswordRegex.Replace(message, $"password={mask}");
message = DutchPasswordRegex.Replace(message, $"wachtwoord={mask}");
message = CreditCardRegex.Replace(message, mask);
message = SsnBsnRegex.Replace(message, mask);
message = ApiKeyRegex.Replace(message, mask);
message = PhoneGenericRegex.Replace(message, mask);
message = PhoneDutchRegex.Replace(message, mask);
message = PhoneBelgiumRegex.Replace(message, mask);
message = DobRegex.Replace(message, mask);
message = PostalCodeDutchRegex.Replace(message, mask);
message = IbanGenericRegex.Replace(message, mask);
message = IbanDutchRegex.Replace(message, mask);
message = JwtRegex.Replace(message, mask);
message = UrlSensitiveRegex.Replace(message, mask);
message = LicenseKeyRegex.Replace(message, mask);
message = PemKeyRegex.Replace(message, mask);
message = KvkRegex.Replace(message, mask);
message = BtwRegex.Replace(message, mask);
message = DutchDrivingLicenseRegex.Replace(message, mask);
message = ZorgnummerRegex.Replace(message, mask);
message = DutchBankAccountRegex.Replace(message, mask);
message = DutchPassportRegex.Replace(message, mask);
message = DutchIdDocRegex.Replace(message, mask);
}
// Mask custom keywords
if (LoggerSettings?.MaskedKeywords != null)
{
foreach (var keyword in LoggerSettings.MaskedKeywords)
{
if (!string.IsNullOrWhiteSpace(keyword))
{
message = message.Replace(keyword, LoggerSettings.Mask);
}
}
}
return message;
}
}
}

View File

@@ -27,10 +27,6 @@ namespace EonaCat.Logger
target.IsEnabled = source.IsEnabled; target.IsEnabled = source.IsEnabled;
target.MaxRolloverFiles = source.MaxRolloverFiles; target.MaxRolloverFiles = source.MaxRolloverFiles;
target.UseLocalTime = source.UseLocalTime; target.UseLocalTime = source.UseLocalTime;
target.UseMask = source.UseMask;
target.Mask = source.Mask;
target.UseDefaultMasking = source.UseDefaultMasking;
target.MaskedKeywords = source.MaskedKeywords;
target.IncludeCorrelationId = source.IncludeCorrelationId; target.IncludeCorrelationId = source.IncludeCorrelationId;
target.EnableCategoryRouting = source.EnableCategoryRouting; target.EnableCategoryRouting = source.EnableCategoryRouting;
target.Category = source.Category; target.Category = source.Category;

View File

@@ -192,44 +192,6 @@ public class LoggerSettings
/// </summary> /// </summary>
public string LogOrigin { get; set; } public string LogOrigin { get; set; }
/// <summary>
/// Determines if we need to mask certain keywords
/// </summary>
public bool UseMask { get; set; }
/// <summary>
/// Determines the keywords to mask
/// </summary>
public List<string> MaskedKeywords { get; set; } = new List<string>();
public string Mask { get; set; } = "***MASKED***";
/// <summary>
/// Determines that if masking is enabled we also need to use the default masking options:
/// IP addresses
/// MAC addresses
/// Emails
/// Passwords
/// Credit card numbers
/// Social security numbers (SSN) and BSN (Dutch Citizen Service Number)
/// API keys/tokens
/// Phone numbers (generic and Dutch specific)
/// Dates of birth (DOB) or other date formats
/// IBAN/Bank account numbers (generic and Dutch specific)
/// JWT tokens
/// URLs with sensitive query strings
/// License keys
/// Public and private keys (e.g., PEM format)
/// Dutch KVK number (8 or 12 digits)
/// Dutch BTW-nummer (VAT number)
/// Dutch driving license number (10-12 characters)
/// Dutch health insurance number (Zorgnummer)
/// Other Dutch Bank Account numbers (9-10 digits)
/// Dutch Passport Numbers (9 alphanumeric characters
/// Dutch Identification Document Numbers (varying formats)
/// Custom keywords specified in LoggerSettings
/// </summary>
public bool UseDefaultMasking { get; set; } = true;
public event LogDelegate OnLog; public event LogDelegate OnLog;
public event LogDelegate OnError; public event LogDelegate OnError;

View File

@@ -260,42 +260,6 @@ namespace EonaCat.Logger.Advanced
} }
``` ```
**Code for enabling keyword masking (default: false):**
```csharp
Logger.LoggerSettings.UseMask = true;
```
Sensitive information will be replaced with ****MASKED****
You can add additional masking keywords using the MaskedKeywords property on the LoggerSettings object
You can also override the MaskSensitiveInformation method of the batchLoggingProvider to add your own masking
Default maskings (when the masking is enabled):
IP addresses
MAC addresses
Emails
Passwords
Credit card numbers
Social security numbers (SSN) and BSN (Dutch Citizen Service Number)
API keys/tokens
Phone numbers (generic and Dutch specific)
Dates of birth (DOB) or other date formats
IBAN/Bank account numbers (generic and Dutch specific)
JWT tokens
URLs with sensitive query strings
License keys
Public and private keys (e.g., PEM format)
Dutch KVK number (8 or 12 digits)
Dutch BTW-nummer (VAT number)
Dutch driving license number (10-12 characters)
Dutch health insurance number (Zorgnummer)
Other Dutch Bank Account numbers (9-10 digits)
Dutch Passport Numbers (9 alphanumeric characters
Dutch Identification Document Numbers (varying formats)
Custom keywords specified in LoggerSettings
Header tokens: Header tokens:
{date} {time} {ts} {tz} {unix} {ticks}{newline} {date} {time} {ts} {tz} {unix} {ticks}{newline}

View File

@@ -7,8 +7,6 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="EonaCat.MemoryGuard" Version="1.1.1" />
<PackageReference Include="EonaCat.MemoryGuard.Generator" Version="1.1.0" />
<PackageReference Include="EonaCat.Versioning" Version="1.2.8"> <PackageReference Include="EonaCat.Versioning" Version="1.2.8">
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

View File

@@ -6,7 +6,7 @@
using EonaCat.Logger.EonaCatCoreLogger.Models; using EonaCat.Logger.EonaCatCoreLogger.Models;
using EonaCat.Logger.LogClient; using EonaCat.Logger.LogClient;
using EonaCat.Logger.Managers; using EonaCat.Logger.Managers;
using EonaCat.MemoryGuard; //using EonaCat.MemoryGuard;
using EonaCat.Versioning.Helpers; using EonaCat.Versioning.Helpers;
using EonaCat.Web.RateLimiter; using EonaCat.Web.RateLimiter;
using EonaCat.Web.RateLimiter.Endpoints.Extensions; using EonaCat.Web.RateLimiter.Endpoints.Extensions;
@@ -42,22 +42,22 @@
}); });
Console.ReadKey(); Console.ReadKey();
var _config = new MemoryGuardConfiguration //var _config = new MemoryGuardConfiguration
{ //{
MonitoringInterval = TimeSpan.FromSeconds(5), // MonitoringInterval = TimeSpan.FromSeconds(5),
AnalysisInterval = TimeSpan.FromSeconds(10), // AnalysisInterval = TimeSpan.FromSeconds(10),
PredictionInterval = TimeSpan.FromSeconds(15), // PredictionInterval = TimeSpan.FromSeconds(15),
LeakDetectionThreshold = TimeSpan.FromSeconds(5), // LeakDetectionThreshold = TimeSpan.FromSeconds(5),
SuspiciousObjectThreshold = TimeSpan.FromSeconds(3), // SuspiciousObjectThreshold = TimeSpan.FromSeconds(3),
BackgroundReportingInterval = TimeSpan.FromMinutes(1.5), // BackgroundReportingInterval = TimeSpan.FromMinutes(1.5),
CaptureStackTraces = true, // CaptureStackTraces = true,
EnableAutoRemediation = true, // EnableAutoRemediation = true,
AutoSaveReports = true, // AutoSaveReports = true,
MemoryPressureThreshold = 500 * 1024 * 1024, // 500MB // MemoryPressureThreshold = 500 * 1024 * 1024, // 500MB
BackgroundAnalysisInterval = TimeSpan.FromMinutes(1), // BackgroundAnalysisInterval = TimeSpan.FromMinutes(1),
OptimizationInterval = TimeSpan.FromMinutes(10), // OptimizationInterval = TimeSpan.FromMinutes(10),
PatternDetectionInterval = TimeSpan.FromMinutes(3) // PatternDetectionInterval = TimeSpan.FromMinutes(3)
}; //};
//MemoryGuard.Start(_config); //MemoryGuard.Start(_config);
@@ -112,7 +112,6 @@
logger.LoggerSettings.LogTrace(); logger.LoggerSettings.LogTrace();
logger.LoggerSettings.LogTraffic(); logger.LoggerSettings.LogTraffic();
logger.LoggerSettings.OnLog += LoggerSettings_OnLog; logger.LoggerSettings.OnLog += LoggerSettings_OnLog;
logger.LoggerSettings.UseMask = true;
//LoggerSettings.CustomHeaderFormatter = ctx => //LoggerSettings.CustomHeaderFormatter = ctx =>
//{ //{
@@ -168,7 +167,6 @@
options.MaxRolloverFiles = 5; options.MaxRolloverFiles = 5;
//options.FileSizeLimit = 1 * 1024 * 1024 / 4; //options.FileSizeLimit = 1 * 1024 * 1024 / 4;
options.UseLocalTime = true; options.UseLocalTime = true;
options.UseMask = true;
builder.Logging.AddEonaCatFileLogger(fileLoggerOptions: options, filenamePrefix: "web"); builder.Logging.AddEonaCatFileLogger(fileLoggerOptions: options, filenamePrefix: "web");
builder.Logging.AddEonaCatConsoleLogger(); builder.Logging.AddEonaCatConsoleLogger();
@@ -287,23 +285,22 @@
} }
//MemoryLeakTester.Start(logger); //MemoryLeakTester.Start(logger);
_ = Task.Run(RunMemoryReportTask).ConfigureAwait(false); //_ = Task.Run(RunMemoryReportTask).ConfigureAwait(false);
_ = Task.Run(RunMaskTest).ConfigureAwait(false);
_ = Task.Run(RunWebLoggerTestsAsync).ConfigureAwait(false); _ = Task.Run(RunWebLoggerTestsAsync).ConfigureAwait(false);
_ = Task.Run(RunWebLoggingTests).ConfigureAwait(false); _ = Task.Run(RunWebLoggingTests).ConfigureAwait(false);
_ = Task.Run(RunLoggingTestsAsync).ConfigureAwait(false); _ = Task.Run(RunLoggingTestsAsync).ConfigureAwait(false);
_ = Task.Run(RunLoggingExceptionTests).ConfigureAwait(false); _ = Task.Run(RunLoggingExceptionTests).ConfigureAwait(false);
_ = Task.Run(RunWebLoggingExceptionTests).ConfigureAwait(false); _ = Task.Run(RunWebLoggingExceptionTests).ConfigureAwait(false);
async Task RunMemoryReportTask() //async Task RunMemoryReportTask()
{ //{
while (true) // while (true)
{ // {
await MemoryGuard.PrintReportAsync().ConfigureAwait(false); // await MemoryGuard.PrintReportAsync().ConfigureAwait(false);
await Task.Delay(60000).ConfigureAwait(false); // await Task.Delay(60000).ConfigureAwait(false);
Console.ReadKey(); // Console.ReadKey();
} // }
} //}
void RunWebLoggingExceptionTests() void RunWebLoggingExceptionTests()
{ {
@@ -347,29 +344,6 @@
} }
} }
async Task RunMaskTest()
{
if (!Directory.Exists(logger.LogFolder))
{
Directory.CreateDirectory(logger.LogFolder);
}
for (var i = 0; i < 9000000; i++)
{
var message = $"mask-test {i}";
app.Logger.LogInformation("password: test");
app.Logger.LogInformation("JWT Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c");
app.Logger.LogInformation("JWT Token2: eyJhbGciOiJQUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMn0.iOeNU4dAFFeBwNj6qdhdvm-IvDQrTa6R22lQVJVuWJxorJfeQww5Nwsra0PjaOYhAMj9jNMO5YLmud8U7iQ5gJK2zYyepeSuXhfSi8yjFZfRiSkelqSkU19I-Ja8aQBDbqXf2SAWA8mHF8VS3F08rgEaLCyv98fLLH4vSvsJGf6ueZSLKDVXz24rZRXGWtYYk_OYYTVgR1cg0BLCsuCvqZvHleImJKiWmtS0-CymMO4MMjCy_FIl6I56NqLE9C87tUVpo1mT-kbg5cHDD8I7MjCW5Iii5dethB4Vid3mZ6emKjVYgXrtkOQ-JyGMh6fnQxEFN1ft33GX2eRHluK9eg");
using (var file = new StreamWriter(Path.Combine(logger.LogFolder, "testmask.log"), true))
{
await file.WriteAsync(message);
}
Console.WriteLine($"Masked: {i}");
await Task.Delay(1);
}
}
async Task RunLoggingTestsAsync() async Task RunLoggingTestsAsync()
{ {
var loggerSettings = new LoggerSettings(); var loggerSettings = new LoggerSettings();
@@ -430,10 +404,10 @@
app.Run(); app.Run();
} }
private static void Instance_LeakDetected(object? sender, EonaCat.MemoryGuard.EventArguments.MemoryLeakDetectedEventArgs e) //private static void Instance_LeakDetected(object? sender, EonaCat.MemoryGuard.EventArguments.MemoryLeakDetectedEventArgs e)
{ //{
// Leak detected // // Leak detected
} //}
} }
static class MemoryLeakTester static class MemoryLeakTester