Added masking

2024-08-15 21:48:41 +02:00
parent 7e41f8108d
commit 5f26cb9180
11 changed files with 344 additions and 32 deletions
--- a/EonaCat.Logger/EonaCatCoreLogger/Extensions/FileLoggerFactoryExtensions.cs
+++ b/EonaCat.Logger/EonaCatCoreLogger/Extensions/FileLoggerFactoryExtensions.cs
@@ -49,6 +49,10 @@ public static class FileLoggerFactoryExtensions
                options.IsEnabled = fileLoggerOptions.IsEnabled;
                options.MaxRolloverFiles = fileLoggerOptions.MaxRolloverFiles;
                options.UseLocalTime = fileLoggerOptions.UseLocalTime;
+                options.UseMask = fileLoggerOptions.UseMask;
+                options.Mask = fileLoggerOptions.Mask;
+                options.UseDefaultMasking = fileLoggerOptions.UseDefaultMasking;
+                options.MaskedKeywords = fileLoggerOptions.MaskedKeywords;
            }
        );
        return builder;
--- a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerOptions.cs
+++ b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerOptions.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using EonaCat.Logger.EonaCatCoreLogger.Internal;

@@ -100,4 +101,42 @@ public class FileLoggerOptions : BatchingLoggerOptions
    /// </summary>
    /// <returns></returns>
    public string LogDirectory { get; set; } = Path.Combine(DefaultPath, "logs");
+
+    /// <summary>
+    /// Determines if we need to mask certain keywords
+    /// </summary>
+    public bool UseMask { get; set; }
+
+    /// <summary>
+    /// Determines the keywords to mask
+    /// </summary>
+    public List<string> MaskedKeywords { get; set; } = new List<string>();
+    public string Mask { get; set; } = "***MASKED***";
+
+    /// <summary>
+    /// Determines that if masking is enabled we also need to use the default masking options:
+    /// IP addresses
+    /// MAC addresses
+    /// Emails
+    /// Passwords
+    /// Credit card numbers
+    /// Social security numbers (SSN) and BSN (Dutch Citizen Service Number)
+    /// API keys/tokens
+    /// Phone numbers (generic and Dutch specific)
+    /// Dates of birth (DOB) or other date formats
+    /// IBAN/Bank account numbers (generic and Dutch specific)
+    /// JWT tokens
+    /// URLs with sensitive query strings
+    /// License keys
+    /// Public and private keys (e.g., PEM format)
+    /// Dutch KVK number (8 or 12 digits)
+    /// Dutch BTW-nummer (VAT number)
+    /// Dutch driving license number (10-12 characters)
+    /// Dutch health insurance number (Zorgnummer)
+    /// Other Dutch Bank Account numbers (9-10 digits)
+    /// Dutch Passport Numbers (9 alphanumeric characters
+    /// Dutch Identification Document Numbers (varying formats)
+    /// Custom keywords specified in LoggerSettings
+    /// </summary>
+    public bool UseDefaultMasking { get; set; } = true;
 }
--- a/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLogger.cs
+++ b/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLogger.cs
@@ -57,12 +57,10 @@ namespace EonaCat.Logger.EonaCatCoreLogger.Internal
            message = LogHelper.FormatMessageWithHeader(_loggerSettings, logLevel.FromLogLevel(), message, timestamp.DateTime, category)
                      + Environment.NewLine;

-            _provider.AddMessage(timestamp, message);
-
            var currentMessage = new EonaCatLogMessage
            {
                DateTime = timestamp.DateTime,
-                Message = message,
+                Message = _provider.AddMessage(timestamp, message),
                LogType = logLevel.FromLogLevel(),
                Origin = string.IsNullOrWhiteSpace(_loggerSettings.LogOrigin) ? "BatchingLogger" : _loggerSettings.LogOrigin
            };
--- a/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLoggerProvider.cs
+++ b/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLoggerProvider.cs
@@ -38,6 +38,7 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
        if (options.Value is FileLoggerOptions fileLoggerOptions)
        {
            UseLocalTime = fileLoggerOptions.UseLocalTime;
+            UseMask = fileLoggerOptions.UseMask;
        }

        _batchSize = loggerOptions.BatchSize;
@@ -61,6 +62,7 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable

            _loggerSettings = new LoggerSettings();
            _loggerSettings.UseLocalTime = UseLocalTime;
+            _loggerSettings.UseMask = UseMask;
            return _loggerSettings;
        }

@@ -68,6 +70,7 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
    }

    public bool IsStarted { get; set; }
+    public bool UseMask { get; set; }

    public void Dispose()
    {
@@ -126,18 +129,198 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable
    protected async Task WriteStartMessage()
    {
        var message = LogHelper.GetStartupMessage();
-        await WriteMessagesAsync(new List<LogMessage> { new LogMessage() { Message = message, Timestamp = CurrentDateTimeOffset } }, _cancellationTokenSource.Token).ConfigureAwait(false);
+        await WriteMessagesAsync(new List<LogMessage> { CreateLoggerMessage(message, CurrentDateTimeOffset) }, _cancellationTokenSource.Token).ConfigureAwait(false);
+    }
+
+    private LogMessage CreateLoggerMessage(string message, DateTimeOffset currentDateTimeOffset)
+    {
+        var result = new LogMessage() { Message = message, Timestamp = currentDateTimeOffset };
+
+        if (LoggerSettings != null && LoggerSettings.UseMask)
+        {
+            // Masking sensitive information
+            result.Message = MaskSensitiveInformation(result.Message);
+        }
+
+        return result;
+    }
+
+    /// <summary>
+    /// Masks sensitive information within the provided message string.
+    /// This method is virtual and can be overridden to customize masking behavior.
+    /// </summary>
+    /// <param name="message">The log message potentially containing sensitive information.</param>
+    /// <returns>The masked log message.</returns>
+    protected virtual string MaskSensitiveInformation(string message)
+    {
+        if (string.IsNullOrEmpty(message))
+        {
+            return message;
+        }
+
+        if (LoggerSettings != null && LoggerSettings.UseDefaultMasking)
+        {
+            // Mask IP addresses
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b(?:\d{1,3}\.){3}\d{1,3}\b(?!\d)",
+                LoggerSettings.Mask);
+
+            // Mask MAC addresses
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}\b",
+                LoggerSettings.Mask);
+
+            // Mask emails
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b",
+                LoggerSettings.Mask,
+                System.Text.RegularExpressions.RegexOptions.IgnoreCase);
+
+            // Mask passwords
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"(?i)(password\s*[:= ]\s*|pwd\s*[:= ]\s*)[^\s]+",
+                $"password={LoggerSettings.Mask}");
+
+            // Mask credit card numbers
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b(?:\d{4}[ -]?){3}\d{4}\b",
+                LoggerSettings.Mask);
+
+            // Mask social security numbers (SSN) and BSN (Dutch Citizen Service Number)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b\d{3}-\d{2}-\d{4}\b|\b\d{9}\b",
+                LoggerSettings.Mask); // SSN (USA)
+
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b\d{9}\b",
+                LoggerSettings.Mask); // BSN (Dutch)
+
+            // Mask passwords (Dutch)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"(?i)(wachtwoord\s*[:= ]\s*|ww\s*=\s*)[^\s]+",
+                $"wachtwoord={LoggerSettings.Mask}");
+
+            // Mask API keys/tokens
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b[A-Za-z0-9-_]{20,}\b",
+                LoggerSettings.Mask);
+
+            // Mask phone numbers (generic and Dutch specific)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b(\+?\d{1,4}[-.\s]?)?(\(?\d{3}\)?[-.\s]?)?\d{3}[-.\s]?\d{4}\b",
+                LoggerSettings.Mask); // Generic
+
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b(\+31|0031|0|06)[-\s]?\d{8}\b",
+                LoggerSettings.Mask); // Dutch
+
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b(\+32|0032|0|06)[-\s]?\d{8}\b",
+                LoggerSettings.Mask); // Belgium
+
+            // Mask dates of birth (DOB) or other date formats
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b\d{2}[/-]\d{2}[/-]\d{4}\b",
+                LoggerSettings.Mask);
+
+            // Mask Dutch postal codes
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b\d{4}\s?[A-Z]{2}\b",
+                LoggerSettings.Mask);
+
+            // Mask IBAN/Bank account numbers (generic and Dutch specific)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b",
+                LoggerSettings.Mask); // Generic for EU and other country IBANs
+
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\bNL\d{2}[A-Z]{4}\d{10}\b",
+                LoggerSettings.Mask); // Dutch IBAN
+
+            // Mask JWT Tokens
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+             @"\b[A-Za-z0-9-_]{16,}\.[A-Za-z0-9-_]{16,}\.[A-Za-z0-9-_]{16,}\b",
+             LoggerSettings.Mask);
+
+            // Mask URLs with sensitive query strings
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+            @"\bhttps?:\/\/[^\s?]+(\?[^\s]+?[\&=](password|key|token|wachtwoord|sleutel))[^&\s]*",
+            LoggerSettings.Mask,
+            System.Text.RegularExpressions.RegexOptions.IgnoreCase);
+
+            // Mask license keys
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}\b",
+                LoggerSettings.Mask);
+
+            // Mask public and private keys (e.g., PEM format)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"-----BEGIN [A-Z ]+KEY-----[\s\S]+?-----END [A-Z ]+KEY-----",
+                LoggerSettings.Mask);
+
+            // Mask Dutch KVK number (8 or 12 digits)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b\d{8}|\d{12}\b",
+                LoggerSettings.Mask);
+
+            // Mask Dutch BTW-nummer (VAT number)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\bNL\d{9}B\d{2}\b",
+                LoggerSettings.Mask);
+
+            // Mask Dutch driving license number (10-12 characters)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b[A-Z0-9]{10,12}\b",
+                LoggerSettings.Mask);
+
+            // Mask Dutch health insurance number (Zorgnummer)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b\d{9}\b",
+                LoggerSettings.Mask);
+
+            // Mask other Dutch Bank Account numbers (9-10 digits)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b\d{9,10}\b",
+                LoggerSettings.Mask);
+
+            // Mask Dutch Passport Numbers (9 alphanumeric characters)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b[A-Z0-9]{9}\b",
+                LoggerSettings.Mask);
+
+            // Mask Dutch Identification Document Numbers (varying formats)
+            message = System.Text.RegularExpressions.Regex.Replace(message,
+                @"\b[A-Z]{2}\d{6,7}\b",
+                LoggerSettings.Mask);
+        }
+
+        // Mask custom keywords specified in LoggerSettings
+        if (LoggerSettings?.MaskedKeywords != null)
+        {
+            foreach (var keyword in LoggerSettings.MaskedKeywords)
+            {
+                if (string.IsNullOrWhiteSpace(keyword))
+                {
+                    continue;
+                }
+                message = message.Replace(keyword, LoggerSettings.Mask);
+            }
+        }
+
+        return message;
    }

    private async Task WriteStopMessage()
    {
        var message = LogHelper.GetStopMessage();
-        await WriteMessagesAsync(new List<LogMessage> { new LogMessage() { Message = message, Timestamp = CurrentDateTimeOffset } }, _cancellationTokenSource.Token).ConfigureAwait(false);
+        await WriteMessagesAsync(new List<LogMessage> { CreateLoggerMessage(message, CurrentDateTimeOffset) }, _cancellationTokenSource.Token).ConfigureAwait(false);
    }

-    internal void AddMessage(DateTimeOffset timestamp, string message)
+    internal string AddMessage(DateTimeOffset timestamp, string message)
    {
-        _messageQueue.Enqueue(new LogMessage { Message = message, Timestamp = timestamp });
+        var result = CreateLoggerMessage(message, timestamp);
+        _messageQueue.Enqueue(result);
+        return result.Message;
    }

    private Task StartAsync()