diff --git a/EonaCat.Logger/EonaCat.Logger.csproj b/EonaCat.Logger/EonaCat.Logger.csproj index 3f75c3e..afd2933 100644 --- a/EonaCat.Logger/EonaCat.Logger.csproj +++ b/EonaCat.Logger/EonaCat.Logger.csproj @@ -13,8 +13,8 @@ EonaCat (Jeroen Saey) EonaCat;Logger;EonaCatLogger;Log;Writer;Jeroen;Saey - 1.6.4 - 1.6.4 + 1.6.5 + 1.6.5 README.md True LICENSE @@ -25,7 +25,7 @@ - 1.6.4+{chash:10}.{c:ymd} + 1.6.5+{chash:10}.{c:ymd} true true v[0-9]* diff --git a/EonaCat.Logger/EonaCatCoreLogger/Extensions/FileLoggerFactoryExtensions.cs b/EonaCat.Logger/EonaCatCoreLogger/Extensions/FileLoggerFactoryExtensions.cs index 23596eb..31d7230 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/Extensions/FileLoggerFactoryExtensions.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/Extensions/FileLoggerFactoryExtensions.cs @@ -43,26 +43,11 @@ public static class FileLoggerFactoryExtensions fileLoggerOptions.FileNamePrefix = filenamePrefix; } - builder.AddEonaCatFileLogger(options => - { - options.FileNamePrefix = fileLoggerOptions.FileNamePrefix; - options.FlushPeriod = fileLoggerOptions.FlushPeriod; - options.RetainedFileCountLimit = fileLoggerOptions.RetainedFileCountLimit; - options.MaxWriteTries = fileLoggerOptions.MaxWriteTries; - options.FileSizeLimit = fileLoggerOptions.FileSizeLimit; - options.LogDirectory = fileLoggerOptions.LogDirectory; - options.BatchSize = fileLoggerOptions.BatchSize; - options.FileSizeLimit = fileLoggerOptions.FileSizeLimit; - options.IsEnabled = fileLoggerOptions.IsEnabled; - options.MaxRolloverFiles = fileLoggerOptions.MaxRolloverFiles; - options.UseLocalTime = fileLoggerOptions.UseLocalTime; - options.UseMask = fileLoggerOptions.UseMask; - options.Mask = fileLoggerOptions.Mask; - options.UseDefaultMasking = fileLoggerOptions.UseDefaultMasking; - options.MaskedKeywords = fileLoggerOptions.MaskedKeywords; - options.IncludeCorrelationId = fileLoggerOptions.IncludeCorrelationId; - } - ); + builder = builder.AddEonaCatFileLogger(options => + { + LoggerConfigurator.ApplyFileLoggerSettings(options, fileLoggerOptions); + }); + return builder; } diff --git a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerOptions.cs b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerOptions.cs index 42c9ebe..9a02ce1 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerOptions.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerOptions.cs @@ -16,6 +16,8 @@ public class FileLoggerOptions : BatchingLoggerOptions private int _fileSizeLimit = 200 * 1024 * 1024; private int _maxRolloverFiles = 10; private int _retainedFileCountLimit = 50; + public bool EnableCategoryRouting { get; set; } + public string Category { get; set; } public static string DefaultPath => AppDomain.CurrentDomain.RelativeSearchPath ?? AppDomain.CurrentDomain.BaseDirectory; diff --git a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs index 0c88033..0fdba8c 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs @@ -1,4 +1,9 @@ -using System; +using EonaCat.Logger.EonaCatCoreLogger; +using EonaCat.Logger.EonaCatCoreLogger.Internal; +using EonaCat.Logger.Managers; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; @@ -6,12 +11,6 @@ using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; -using EonaCat.Logger.EonaCatCoreLogger.Internal; -using EonaCat.Logger.Managers; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace EonaCat.Logger.EonaCatCoreLogger; [ProviderAlias("EonaCatFileLogger")] public sealed class FileLoggerProvider : BatchingLoggerProvider @@ -22,212 +21,371 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider private readonly int _maxRetainedFiles; private readonly int _maxRolloverFiles; - private string _logFile; - private long _currentFileSize; - private Timer _flushTimer; - private readonly ConcurrentDictionary _buffers = new(); - private readonly ConcurrentDictionary _fileSizes = new(); - private readonly SemaphoreSlim _writeLock = new(1, 1); - private readonly SemaphoreSlim _rolloverLock = new(1, 1); + private readonly LoggerScopedContext _context = new LoggerScopedContext(); + private readonly Dictionary _files = new Dictionary(); + private static readonly ConcurrentDictionary _fileLocks = new ConcurrentDictionary(); - private readonly LoggerScopedContext _context = new(); + // High-performance buffer pool + private static readonly ConcurrentBag _bufferPool = new ConcurrentBag(); + private const int BufferSize = 256 * 1024; // 256KB buffers + private const int MaxPoolSize = 100; - public string LogFile => _logFile ?? string.Empty; + // Pre-allocated StringBuilder pool for low memory + private static readonly ConcurrentBag _stringBuilderPool = new ConcurrentBag(); + private const int MaxStringBuilderPoolSize = 50; + + public bool IncludeCorrelationId { get; } + public bool EnableCategoryRouting { get; } public event EventHandler OnError; public event EventHandler OnRollOver; - public FileLoggerProvider(IOptions options) : base(options) + public string LogFile + { + get + { + FileState state; + return _files.TryGetValue(string.Empty, out state) ? state.FilePath : string.Empty; + } + } + + private sealed class FileState + { + public string FilePath; + public long Size; + public DateTime Date; + public readonly SemaphoreSlim WriteLock = new SemaphoreSlim(1, 1); + + // Buffered writing for high throughput + public byte[] Buffer; + public int BufferPosition; + public FileStream Stream; + } + + public FileLoggerProvider(IOptions options) + : base(options) { var o = options.Value; + if (o == null) + { + throw new ArgumentNullException("options"); + } + _path = o.LogDirectory; _fileNamePrefix = o.FileNamePrefix; _maxFileSize = o.FileSizeLimit; _maxRetainedFiles = o.RetainedFileCountLimit; _maxRolloverFiles = o.MaxRolloverFiles; IncludeCorrelationId = o.IncludeCorrelationId; - - Directory.CreateDirectory(_path); - InitializeCurrentFile(); - _flushTimer = new Timer(_ => FlushAllAsync().GetAwaiter().GetResult(), null, 5000, 5000); - } - - private async Task FlushAllAsync() - { - foreach (var kv in _buffers) - { - await FlushAsync(kv.Key, kv.Value, CancellationToken.None); - } - } - - public bool IncludeCorrelationId { get; } - - internal override async Task WriteMessagesAsync( - IReadOnlyList messages, - CancellationToken token) - { - if (messages.Count == 0) - { - return; - } + EnableCategoryRouting = o.EnableCategoryRouting; Directory.CreateDirectory(_path); - // Group messages by date - foreach (var group in messages.GroupBy(m => (m.Timestamp.Year, m.Timestamp.Month, m.Timestamp.Day))) - { - var file = GetFullName(group.Key); - InitializeFile(file); - - var stringBuilder = _buffers.GetOrAdd(file, _ => new StringBuilder(4096)); - - lock (stringBuilder) - { - foreach (var message in group) - { - AppendMessage(stringBuilder, message); - } - } - - await FlushAsync(file, stringBuilder, token).ConfigureAwait(false); - DeleteOldLogFiles(); - } + // Initialize + var defaultState = CreateFileState(DateTime.UtcNow.Date, options.Value.Category); + _files[string.Empty] = defaultState; } - private void AppendMessage(StringBuilder sb, LogMessage msg) + internal override async Task WriteMessagesAsync(IReadOnlyList messages, CancellationToken token) { - // Ensure correlation id exists (once per async context) - if (IncludeCorrelationId) + // Group messages by category for batch processing + if (EnableCategoryRouting) { - var cid = _context.Get("CorrelationId") ?? Guid.NewGuid().ToString(); - _context.Set("CorrelationId", cid); - } - - sb.Append(msg.Message); - - var ctx = _context.GetAll(); - if (ctx.Count > 0) - { - sb.Append(" ["); - bool first = true; - foreach (var kv in ctx) + var grouped = messages.GroupBy(m => SanitizeCategory(m.Category)); + foreach (var group in grouped) { - if (!first) + var categoryKey = group.Key; + + FileState state; + if (!_files.TryGetValue(categoryKey, out state)) { - sb.Append(' '); + var date = DateTime.UtcNow.Date; + state = CreateFileState(date, categoryKey); + _files[categoryKey] = state; } - sb.Append(kv.Key).Append('=').Append(kv.Value); - first = false; + await WriteBatchAsync(state, group, categoryKey, token); } - sb.Append(']'); + } + else + { + var categoryKey = string.Empty; + + FileState state; + if (!_files.TryGetValue(categoryKey, out state)) + { + var date = DateTime.UtcNow.Date; + state = CreateFileState(date, categoryKey); + _files[categoryKey] = state; + } + + await WriteBatchAsync(state, messages, categoryKey, token); } - sb.AppendLine(); + DeleteOldLogFiles(); } - public void InitializeCurrentFile() + private async Task WriteBatchAsync(FileState state, IEnumerable messages, string categoryKey, CancellationToken token) { - if (!string.IsNullOrEmpty(_logFile)) - { - return; - } - - _logFile = GetFullName((DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day)); - _currentFileSize = File.Exists(_logFile) ? new FileInfo(_logFile).Length : 0; - } - - private async Task FlushAsync(string file, StringBuilder stringBuilder, CancellationToken token) - { - if (stringBuilder.Length == 0) - { - return; - } - - await _writeLock.WaitAsync(token).ConfigureAwait(false); + await state.WriteLock.WaitAsync(token); try { - var text = stringBuilder.ToString(); - byte[] bytesToWrite = Encoding.UTF8.GetBytes(text); - - using (var fileStream = new FileStream( - file, - FileMode.Append, - FileAccess.Write, - FileShare.Read, - 64 * 1024, - useAsync: true)) + foreach (var msg in messages) { - await fileStream.WriteAsync(bytesToWrite, 0, bytesToWrite.Length, token); + if (token.IsCancellationRequested) + break; + + var date = msg.Timestamp.UtcDateTime.Date; + + // Rotate file by date + if (state.Date != date) + { + await FlushBufferAsync(state); + RotateByDate(state, date, categoryKey); + } + + WriteMessageToBuffer(state, msg); + + // Flush buffer if nearly full or file size limit reached + if (state.BufferPosition >= BufferSize - 1024 || state.Size >= _maxFileSize) + { + await FlushBufferAsync(state); + + if (state.Size >= _maxFileSize) + { + RollOver(state, categoryKey); + } + } } - // Update current file size correctly - _fileSizes.AddOrUpdate(file, bytesToWrite.Length, (_, old) => old + bytesToWrite.Length); - _currentFileSize = _fileSizes[file]; - - stringBuilder.Clear(); - - if (_fileSizes[file] >= _maxFileSize) + // Final flush for this batch + await FlushBufferAsync(state); + } + catch (Exception ex) + { + if (OnError != null) { - await RollOverAsync(file).ConfigureAwait(false); + OnError.Invoke(this, new ErrorMessage { Exception = ex, Message = ex.Message }); } } finally { - _writeLock.Release(); + state.WriteLock.Release(); } } - private async Task RollOverAsync(string file) + private FileState CreateFileState(DateTime date, string category) + { + var path = GetFullName(date, category); + + var buffer = GetBuffer(); + var stream = new FileStream( + path, + FileMode.Append, + FileAccess.Write, + FileShare.ReadWrite, + BufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan); + + var state = new FileState + { + Stream = stream, + FilePath = path, + Size = GetFileSize(path), + Date = date, + Buffer = buffer, + BufferPosition = 0 + }; + + return state; + } + + private static byte[] GetBuffer() + { + byte[] buffer; + if (!_bufferPool.TryTake(out buffer)) + { + buffer = new byte[BufferSize]; + } + return buffer; + } + + private static void ReturnBuffer(byte[] buffer) + { + if (_bufferPool.Count < MaxPoolSize) + { + Array.Clear(buffer, 0, buffer.Length); + _bufferPool.Add(buffer); + } + } + + private static StringBuilder GetStringBuilder() + { + StringBuilder sb; + if (!_stringBuilderPool.TryTake(out sb)) + { + sb = new StringBuilder(512); + } + else + { + sb.Clear(); + } + return sb; + } + + private static void ReturnStringBuilder(StringBuilder sb) + { + if (_stringBuilderPool.Count < MaxStringBuilderPoolSize && sb.Capacity <= 2048) + { + sb.Clear(); + _stringBuilderPool.Add(sb); + } + } + + private static long GetFileSize(string path) { - await _rolloverLock.WaitAsync().ConfigureAwait(false); try { - var dir = Path.GetDirectoryName(file); - var name = Path.GetFileNameWithoutExtension(file); - var ext = Path.GetExtension(file); + return File.Exists(path) ? new FileInfo(path).Length : 0; + } + catch + { + return 0; + } + } - for (int i = _maxRolloverFiles - 1; i >= 1; i--) + private void RotateByDate(FileState state, DateTime newDate, string category) + { + if (state.Stream != null) + { + state.Stream.Dispose(); + } + + state.Date = newDate; + state.FilePath = GetFullName(newDate, category); + state.Size = GetFileSize(state.FilePath); + state.BufferPosition = 0; + + state.Stream = new FileStream( + state.FilePath, + FileMode.Append, + FileAccess.Write, + FileShare.ReadWrite, + BufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan); + } + + private void RollOver(FileState state, string category) + { + if (state.Stream != null) + { + state.Stream.Dispose(); + state.Stream = null; + } + + var dir = Path.GetDirectoryName(state.FilePath); + var name = Path.GetFileNameWithoutExtension(state.FilePath); + var ext = Path.GetExtension(state.FilePath); + + for (int i = _maxRolloverFiles - 1; i >= 1; i--) + { + var src = Path.Combine(dir, string.Format("{0}.{1}{2}", name, i, ext)); + var dst = Path.Combine(dir, string.Format("{0}.{1}{2}", name, i + 1, ext)); + + if (File.Exists(dst)) { - var source = Path.Combine(dir, $"{name}.{i}{ext}"); - var destination = Path.Combine(dir, $"{name}.{i + 1}{ext}"); + File.Delete(dst); + } - if (File.Exists(destination)) - { - File.Delete(destination); - } + if (File.Exists(src)) + { + File.Move(src, dst); + } + } - if (File.Exists(source)) + var first = Path.Combine(dir, string.Format("{0}.1{1}", name, ext)); + if (File.Exists(state.FilePath)) + { + File.Move(state.FilePath, first); + } + + if (OnRollOver != null) + { + OnRollOver.Invoke(this, state.FilePath); + } + + state.Size = 0; + state.BufferPosition = 0; + + state.Stream = new FileStream( + state.FilePath, + FileMode.Append, + FileAccess.Write, + FileShare.ReadWrite, + BufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan); + } + + private void WriteMessageToBuffer(FileState state, LogMessage msg) + { + var sb = GetStringBuilder(); + + try + { + sb.Append(msg.Message); + + if (IncludeCorrelationId) + { + var ctx = _context.GetAll(); + if (ctx.Count > 0) { - File.Move(source, destination); + sb.Append(" ["); + bool first = true; + foreach (var kv in ctx) + { + if (!first) + { + sb.Append(' '); + } + + sb.Append(kv.Key).Append('=').Append(kv.Value); + first = false; + } + sb.Append(']'); } } - var firstRoll = Path.Combine(dir, $"{name}.1{ext}"); - if (File.Exists(file)) + sb.AppendLine(); + + var text = sb.ToString(); + var byteCount = Encoding.UTF8.GetByteCount(text); + + // If message won't fit in buffer, flush first + if (state.BufferPosition + byteCount > BufferSize) { - File.Move(file, firstRoll); + FlushBufferAsync(state).Wait(); } - _fileSizes[file] = 0; + // Write to buffer + var written = Encoding.UTF8.GetBytes(text, 0, text.Length, state.Buffer, state.BufferPosition); + state.BufferPosition += written; + state.Size += written; } finally { - _rolloverLock.Release(); - OnRollOver?.Invoke(this, file); + ReturnStringBuilder(sb); } } - private void InitializeFile(string file) + private async Task FlushBufferAsync(FileState state) { - _fileSizes.TryAdd(file, File.Exists(file) ? new FileInfo(file).Length : 0); - _buffers.TryAdd(file, new StringBuilder(4096)); - } + if (state.BufferPosition == 0 || state.Stream == null) + return; - private string GetFullName((int Year, int Month, int Day) g) => - string.IsNullOrWhiteSpace(_fileNamePrefix) - ? Path.Combine(_path, $"{Environment.MachineName}_{g.Year:0000}{g.Month:00}{g.Day:00}.log") - : Path.Combine(_path, $"{_fileNamePrefix}_{Environment.MachineName}_{g.Year:0000}{g.Month:00}{g.Day:00}.log"); + await state.Stream.WriteAsync(state.Buffer, 0, state.BufferPosition); + await state.Stream.FlushAsync(); + state.BufferPosition = 0; + } private void DeleteOldLogFiles() { @@ -237,19 +395,17 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider } var files = new DirectoryInfo(_path) - .GetFiles($"{_fileNamePrefix}*") + .GetFiles(string.Format("{0}*", _fileNamePrefix)) .OrderByDescending(f => { - // Parse date from filename instead of CreationTimeUtc var name = Path.GetFileNameWithoutExtension(f.Name); var parts = name.Split('_'); - var datePart = parts.Length > 1 ? parts[1] : parts[0]; - if (DateTime.TryParseExact(datePart, "yyyyMMdd", null, System.Globalization.DateTimeStyles.None, out var dt)) - { - return dt; - } - - return DateTime.MinValue; + var datePart = parts.LastOrDefault(); + DateTime dt; + return DateTime.TryParseExact(datePart, "yyyyMMdd", null, + System.Globalization.DateTimeStyles.None, out dt) + ? dt + : DateTime.MinValue; }) .Skip(_maxRetainedFiles); @@ -258,4 +414,61 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider try { f.Delete(); } catch { } } } -} + + private string GetFullName(DateTime date, string category) + { + var datePart = date.ToString("yyyyMMdd"); + var machine = Environment.MachineName; + + if (!EnableCategoryRouting || string.IsNullOrWhiteSpace(category)) + { + return string.IsNullOrWhiteSpace(_fileNamePrefix) + ? Path.Combine(_path, string.Format("{0}_{1}.log", machine, datePart)) + : Path.Combine(_path, string.Format("{0}_{1}_{2}.log", _fileNamePrefix, machine, datePart)); + } + + var safeCategory = SanitizeCategory(category); + + return string.IsNullOrWhiteSpace(_fileNamePrefix) + ? Path.Combine(_path, string.Format("{0}_{1}_{2}.log", machine, safeCategory, datePart)) + : Path.Combine(_path, string.Format("{0}_{1}_{2}_{3}.log", _fileNamePrefix, machine, safeCategory, datePart)); + } + + private static string SanitizeCategory(string category) + { + foreach (var c in Path.GetInvalidFileNameChars()) + { + category = category.Replace(c, '_'); + } + + return category.Replace('.', '_'); + } + + protected override void OnShutdownFlush() + { + foreach (var kvp in _files) + { + var state = kvp.Value; + if (state != null) + { + try + { + FlushBufferAsync(state).Wait(); + if (state.Stream != null) + { + state.Stream.Dispose(); + } + if (state.Buffer != null) + { + ReturnBuffer(state.Buffer); + } + } + catch + { + // Ignore errors during shutdown + } + } + } + _files.Clear(); + } +} \ No newline at end of file diff --git a/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLogger.cs b/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLogger.cs index dc8356d..768de82 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLogger.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLogger.cs @@ -63,7 +63,7 @@ namespace EonaCat.Logger.EonaCatCoreLogger.Internal var writtenMessage = _provider.AddMessage(timestamp, formatted, _category); - _settings.RaiseOnLog(new EonaCatLogMessage + _settings?.RaiseOnLog(new EonaCatLogMessage { DateTime = timestamp.DateTime, Message = writtenMessage, @@ -79,7 +79,17 @@ namespace EonaCat.Logger.EonaCatCoreLogger.Internal } catch (Exception ex) { - Console.WriteLine($"Logging error: {ex.Message}"); + _settings?.RaiseOnLogError(new EonaCatLogMessage + { + DateTime = Now.DateTime, + Message = $"Failed to log message: {ex.Message}", + LogType = ELogType.ERROR, + Category = _category, + Exception = ex, + Origin = string.IsNullOrWhiteSpace(_settings.LogOrigin) + ? "BatchingLogger" + : _settings.LogOrigin + }); } } diff --git a/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLoggerProvider.cs b/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLoggerProvider.cs index 9377341..7c55e65 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLoggerProvider.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/Internal/BatchingLoggerProvider.cs @@ -1,76 +1,67 @@ -using EonaCat.Logger.Managers; +using EonaCat.Logger.EonaCatCoreLogger; +using EonaCat.Logger.EonaCatCoreLogger.Internal; +using EonaCat.Logger.Extensions; +using EonaCat.Logger.Managers; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -namespace EonaCat.Logger.EonaCatCoreLogger.Internal; - public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable { - private readonly int _batchSize; - private readonly BlockingCollection _queue; + private readonly ConcurrentQueue _queue = new(); + private readonly Thread _worker; private readonly CancellationTokenSource _cts = new(); - private readonly Task _worker; + + private readonly int _batchSize; + private readonly long _maxQueueBytes = 512L * 1024 * 1024; // 512MB safety + private long _currentQueueBytes; + private long _dropped; private bool _disposed; private LoggerSettings _loggerSettings; + private readonly AutoResetEvent _signal = new(false); + + public event EventHandler? OnLogDropped; + public event EventHandler? OnError; + protected BatchingLoggerProvider(IOptions options) { - var currentOptions = options.Value ?? throw new ArgumentNullException(nameof(options)); + var o = options.Value ?? throw new ArgumentNullException(nameof(options)); + _batchSize = o.BatchSize > 0 ? o.BatchSize : 100; - if (currentOptions.FlushPeriod <= TimeSpan.Zero) - { - throw new ArgumentOutOfRangeException(nameof(currentOptions.FlushPeriod)); - } - - _batchSize = currentOptions.BatchSize > 0 ? currentOptions.BatchSize : 100; - _queue = new BlockingCollection(Math.Max(1, _batchSize * 2)); - - if (currentOptions is FileLoggerOptions file) + if (o is FileLoggerOptions file) { UseLocalTime = file.UseLocalTime; UseMask = file.UseMask; LoggerSettings = file.LoggerSettings; } - _worker = Task.Factory.StartNew( - ProcessLoop, - _cts.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default); + _worker = new Thread(ProcessLoop) + { + IsBackground = true, + Name = "EonaCat-LoggerWriter" + }; + _worker.Start(); } protected bool UseLocalTime { get; } - public bool UseMask { get; } + protected bool UseMask { get; } protected DateTimeOffset NowOffset => UseLocalTime ? DateTimeOffset.Now : DateTimeOffset.UtcNow; protected LoggerSettings LoggerSettings { - get - { - if (_loggerSettings == null) - { - _loggerSettings = new LoggerSettings - { - UseLocalTime = UseLocalTime, - UseMask = UseMask - }; - } - return _loggerSettings; - } + get => _loggerSettings ??= new LoggerSettings { UseLocalTime = UseLocalTime, UseMask = UseMask }; set => _loggerSettings = value; } - public ILogger CreateLogger(string categoryName) - => new BatchingLogger(this, categoryName, LoggerSettings); + public ILogger CreateLogger(string categoryName) => new BatchingLogger(this, categoryName, LoggerSettings); internal abstract Task WriteMessagesAsync( IReadOnlyList messages, @@ -79,16 +70,30 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable internal string AddMessage(DateTimeOffset timestamp, string message, string category) { var log = CreateLogMessage(message, timestamp, category); - _queue.Add(log); + var size = log.EstimatedSize; + + var newSize = Interlocked.Add(ref _currentQueueBytes, size); + if (newSize > _maxQueueBytes) + { + Interlocked.Add(ref _currentQueueBytes, -size); + var dropped = Interlocked.Increment(ref _dropped); + OnLogDropped?.Invoke(this, dropped); + return log.Message; + } + + _queue.Enqueue(log); + _signal.Set(); + return log.Message; } + private LogMessage CreateLogMessage(string message, DateTimeOffset ts, string category) { if (LoggerSettings.UseMask) { - SensitiveDataMasker sensitiveDataMasker = new SensitiveDataMasker(LoggerSettings); - message = sensitiveDataMasker.MaskSensitiveInformation(message); + var masker = new SensitiveDataMasker(LoggerSettings); + message = masker.MaskSensitiveInformation(message); } return new LogMessage @@ -99,53 +104,58 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable }; } - private async Task ProcessLoop() + private void ProcessLoop() { var batch = new List(_batchSize); - var flushInterval = LoggerSettings.FileLoggerOptions.FlushPeriod; - var timeoutMs = (int)Math.Min(flushInterval.TotalMilliseconds, int.MaxValue); - try - { - while (!_cts.Token.IsCancellationRequested) - { - if (_queue.TryTake(out var item, timeoutMs, _cts.Token)) - { - batch.Add(item); - } - - if (batch.Count >= _batchSize || - (batch.Count > 0 && !_queue.TryTake(out _, 0))) - { - await FlushBatchAsync(batch); - } - } - } - catch (OperationCanceledException) - { - if (batch.Count > 0) - { - await FlushBatchAsync(batch); - } - } - } - - private async Task FlushBatchAsync(List batch) - { - try - { - await WriteMessagesAsync(batch, _cts.Token).ConfigureAwait(false); - } - catch (Exception ex) - { - Console.WriteLine($"An error occurred while processing log batches. {ex.Message}"); - } - finally + while (!_cts.IsCancellationRequested) { batch.Clear(); + + // Dequeue up to _batchSize messages + while (batch.Count < _batchSize && _queue.TryDequeue(out var msg)) + { + batch.Add(msg); + Interlocked.Add(ref _currentQueueBytes, -msg.EstimatedSize); + } + + if (batch.Count == 0) + { + // Wait until a new log arrives or cancellation is requested + WaitHandle.WaitAny(new WaitHandle[] { _signal, _cts.Token.WaitHandle }); + continue; + } + + try + { + WriteMessagesAsync(batch, _cts.Token).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + OnError?.Invoke(this, ex); + } + } + + // Process any remaining messages before exit + batch.Clear(); + while (_queue.TryDequeue(out var msg)) + { + batch.Add(msg); + Interlocked.Add(ref _currentQueueBytes, -msg.EstimatedSize); + if (batch.Count >= _batchSize) + { + WriteMessagesAsync(batch, CancellationToken.None).GetAwaiter().GetResult(); + batch.Clear(); + } + } + + if (batch.Count > 0) + { + WriteMessagesAsync(batch, CancellationToken.None).GetAwaiter().GetResult(); } } + public void Dispose() { if (_disposed) @@ -156,10 +166,15 @@ public abstract class BatchingLoggerProvider : ILoggerProvider, IDisposable _disposed = true; _cts.Cancel(); - _queue.CompleteAdding(); + _worker.Join(); + + OnShutdownFlush(); - try { _worker.Wait(); } catch { } _cts.Dispose(); - _queue.Dispose(); + } + + protected virtual void OnShutdownFlush() + { + // default: Do nothing } } diff --git a/EonaCat.Logger/EonaCatCoreLogger/Internal/LogMessage.cs b/EonaCat.Logger/EonaCatCoreLogger/Internal/LogMessage.cs index 25dcb4e..76d964d 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/Internal/LogMessage.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/Internal/LogMessage.cs @@ -9,4 +9,5 @@ public struct LogMessage public DateTimeOffset Timestamp { get; set; } public string Message { get; set; } public string Category { get; set; } + public int EstimatedSize { get; set; } } \ No newline at end of file diff --git a/EonaCat.Logger/EonaCatCoreLogger/TcpLogger.cs b/EonaCat.Logger/EonaCatCoreLogger/TcpLogger.cs index d349d36..3da2f0a 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/TcpLogger.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/TcpLogger.cs @@ -54,7 +54,9 @@ namespace EonaCat.Logger.EonaCatCoreLogger Exception exception, Func formatter) { if (!IsEnabled(logLevel) || formatter == null) + { return; + } try { @@ -77,14 +79,20 @@ namespace EonaCat.Logger.EonaCatCoreLogger { sb.Append(" | Context: "); foreach (var kvp in contextData) + { sb.Append(kvp.Key).Append("=").Append(kvp.Value).Append("; "); + } } if (exception != null) + { sb.Append(" | Exception: ").Append(exception); + } if (!_logChannel.Writer.TryWrite(sb.ToString())) + { OnLogDropped?.Invoke(this, sb.ToString()); // notify if log is dropped + } } catch (Exception ex) { @@ -136,7 +144,9 @@ namespace EonaCat.Logger.EonaCatCoreLogger // Flush any remaining logs if (batch.Count > 0) + { await SafeWriteBatch(writer, batch, client, token); + } } catch (OperationCanceledException) { diff --git a/EonaCat.Logger/LoggerConfigurator.cs b/EonaCat.Logger/LoggerConfigurator.cs new file mode 100644 index 0000000..a16b9d2 --- /dev/null +++ b/EonaCat.Logger/LoggerConfigurator.cs @@ -0,0 +1,38 @@ +using EonaCat.Logger.EonaCatCoreLogger; + +namespace EonaCat.Logger +{ + public static class LoggerConfigurator + { + /// + /// Copies file logger settings from the specified source options to the target options instance. + /// + /// Only properties defined in FileLoggerOptions are copied. The method does not perform + /// a deep clone; reference-type properties are assigned directly. If either parameter is null, the method + /// returns without making changes. + /// The FileLoggerOptions instance to which settings will be applied. If null, no changes are made. + /// The FileLoggerOptions instance from which settings are copied. If null, no changes are made. + public static void ApplyFileLoggerSettings(FileLoggerOptions target, FileLoggerOptions source) + { + if (target == null || source == null) + { + return; + } + + target.FileNamePrefix = source.FileNamePrefix; + target.FlushPeriod = source.FlushPeriod; + target.RetainedFileCountLimit = source.RetainedFileCountLimit; + target.MaxWriteTries = source.MaxWriteTries; + target.FileSizeLimit = source.FileSizeLimit; + target.IsEnabled = source.IsEnabled; + target.MaxRolloverFiles = source.MaxRolloverFiles; + target.UseLocalTime = source.UseLocalTime; + target.UseMask = source.UseMask; + target.Mask = source.Mask; + target.UseDefaultMasking = source.UseDefaultMasking; + target.MaskedKeywords = source.MaskedKeywords; + target.IncludeCorrelationId = source.IncludeCorrelationId; + target.EnableCategoryRouting = source.EnableCategoryRouting; + } + } +} diff --git a/EonaCat.Logger/Managers/LogHelper.cs b/EonaCat.Logger/Managers/LogHelper.cs index 27250b7..7c89a26 100644 --- a/EonaCat.Logger/Managers/LogHelper.cs +++ b/EonaCat.Logger/Managers/LogHelper.cs @@ -1,6 +1,4 @@ using EonaCat.Json; -using EonaCat.Logger.EonaCatCoreLogger; -using EonaCat.Logger.EonaCatCoreLogger.Internal; using EonaCat.Logger.Extensions; using EonaCat.Logger.Servers.GrayLog; using EonaCat.Logger.Servers.Splunk.Models; @@ -8,15 +6,11 @@ using Microsoft.Extensions.Logging; using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Net; -using System.Runtime.InteropServices; using System.Text; using System.Text.RegularExpressions; -using System.Threading; using System.Threading.Tasks; -using static EonaCat.Logger.Managers.LogHelper; // This file is part of the EonaCat project(s) which is released under the Apache License. // See the LICENSE file or go to https://EonaCat.com/License for full license details. diff --git a/EonaCat.Logger/Managers/LogManager.cs b/EonaCat.Logger/Managers/LogManager.cs index e850ebe..12fab58 100644 --- a/EonaCat.Logger/Managers/LogManager.cs +++ b/EonaCat.Logger/Managers/LogManager.cs @@ -31,15 +31,19 @@ namespace EonaCat.Logger.Managers private bool _isDisposing; private string _category; - public LogManager(LoggerSettings settings) + public LogManager(LoggerSettings settings, string category = null) { Settings = settings ?? CreateDefaultSettings(); - SetupLogManager(); - } + _category = string.IsNullOrWhiteSpace(category) ? null : category; + _category = settings.FileLoggerOptions.Category; - public LogManager(LoggerSettings settings, string category = null) : this(settings) - { - _category = string.IsNullOrWhiteSpace(category) ? "General" : category; + if (string.IsNullOrEmpty(_category)) + { + _category = category; + settings.FileLoggerOptions.Category = _category; + } + + SetupLogManager(); SetupFileLogger(settings); // Subscribe to static events @@ -61,14 +65,8 @@ namespace EonaCat.Logger.Managers { if (LoggerProvider is FileLoggerProvider fileLoggerProvider) { - // Ensure log file is initialized - if (string.IsNullOrEmpty(fileLoggerProvider.LogFile)) - { - fileLoggerProvider.InitializeCurrentFile(); - } return fileLoggerProvider.LogFile; } - return string.Empty; } } @@ -128,7 +126,9 @@ namespace EonaCat.Logger.Managers public async Task StartNewLogAsync() { if (_isDisposing || _tokenSource.IsCancellationRequested) + { return; + } await _startLock.WaitAsync().ConfigureAwait(false); try @@ -140,13 +140,8 @@ namespace EonaCat.Logger.Managers if (!IsRunning) { - CreateLogger(); + CreateLogger(_category); Directory.CreateDirectory(Settings.FileLoggerOptions.LogDirectory); - if (LoggerProvider is FileLoggerProvider fileProvider) - { - fileProvider.InitializeCurrentFile(); - } - _logDate = CurrentDateTime; IsRunning = true; } @@ -175,7 +170,7 @@ namespace EonaCat.Logger.Managers LogHelper.SendToFile(Logger, Settings, ELogType.INFO, stopMessage); } - private void CreateLogger() + private void CreateLogger(string categoryName = null) { if (Logger != null) { @@ -204,31 +199,15 @@ namespace EonaCat.Logger.Managers serviceCollection.AddLogging(builder => builder.SetMinimumLevel(Settings.TypesToLog.Max().ToLogLevel()) - .AddEonaCatFileLogger(config => - { - var options = Settings.FileLoggerOptions; - config.LoggerSettings = Settings; - config.MaxWriteTries = options.MaxWriteTries; - config.RetainedFileCountLimit = options.RetainedFileCountLimit; - config.FlushPeriod = options.FlushPeriod; - config.IsEnabled = options.IsEnabled; - config.BatchSize = options.BatchSize; - config.FileSizeLimit = options.FileSizeLimit; - config.LogDirectory = options.LogDirectory; - config.FileNamePrefix = options.FileNamePrefix; - config.MaxRolloverFiles = options.MaxRolloverFiles; - config.UseLocalTime = Settings.UseLocalTime; - config.UseMask = Settings.UseMask; - config.Mask = options.Mask; - config.UseDefaultMasking = Settings.UseDefaultMasking; - config.MaskedKeywords = options.MaskedKeywords; - config.Settings = Settings; - })); + .AddEonaCatFileLogger(options => + { + LoggerConfigurator.ApplyFileLoggerSettings(options, Settings.FileLoggerOptions); + })); var serviceProvider = serviceCollection.BuildServiceProvider(); - LoggerProvider = serviceProvider.GetService(); + LoggerProvider = serviceProvider.GetService(); LoggerFactory = serviceProvider.GetService(); - Logger = LoggerFactory.CreateLogger(Settings.Id); + Logger = LoggerFactory.CreateLogger(_category ?? Settings.Id); LogHelper.SendToFile(Logger, Settings, ELogType.INFO, LogHelper.GetStartupMessage()); } diff --git a/EonaCat.Logger/Managers/LoggerSettings.cs b/EonaCat.Logger/Managers/LoggerSettings.cs index 5934b41..a83718c 100644 --- a/EonaCat.Logger/Managers/LoggerSettings.cs +++ b/EonaCat.Logger/Managers/LoggerSettings.cs @@ -229,6 +229,7 @@ public class LoggerSettings public bool UseDefaultMasking { get; set; } = true; public event LogDelegate OnLog; + public event LogDelegate OnError; private static FileLoggerOptions CreateDefaultFileLoggerOptions() { @@ -360,4 +361,14 @@ public class LoggerSettings { OnLogEvent(eonaCatLogMessage); } + + internal void RaiseOnLogError(EonaCatLogMessage eonaCatLogMessage) + { + OnLogErrorEvent(eonaCatLogMessage); + } + + private void OnLogErrorEvent(EonaCatLogMessage eonaCatLogMessage) + { + OnError?.Invoke(eonaCatLogMessage); + } } \ No newline at end of file diff --git a/Testers/EonaCat.Logger.Test.Web/Program.cs b/Testers/EonaCat.Logger.Test.Web/Program.cs index 9d2170f..63c7d32 100644 --- a/Testers/EonaCat.Logger.Test.Web/Program.cs +++ b/Testers/EonaCat.Logger.Test.Web/Program.cs @@ -16,6 +16,32 @@ { public static async Task Main(string[] args) { + _ = Task.Run(async () => + { + var loggerSettings = new LoggerSettings(); + loggerSettings.Id = "SPEEDTEST"; + loggerSettings.UseLocalTime = true; + loggerSettings.FileLoggerOptions.UseLocalTime = true; + loggerSettings.EnableConsole = false; + loggerSettings.FileLoggerOptions.Category = "SpeedTests"; + loggerSettings.FileLoggerOptions.EnableCategoryRouting = true; + loggerSettings.AllowAllLogTypes(); + var logger = new LogManager(loggerSettings); + + var i = 0; + while (true) + { + i++; + await logger.WriteAsync($"test to file {i} INFO").ConfigureAwait(false); + await logger.WriteAsync($"test to file {i} CRITICAL", ELogType.CRITICAL).ConfigureAwait(false); + await logger.WriteAsync($"test to file {i} DEBUG", ELogType.DEBUG).ConfigureAwait(false); + await logger.WriteAsync($"test to file {i} ERROR", ELogType.ERROR).ConfigureAwait(false); + await logger.WriteAsync($"test to file {i} TRACE", ELogType.TRACE).ConfigureAwait(false); + await Task.Delay(1).ConfigureAwait(false); + } + }); + Console.ReadKey(); + var _config = new MemoryGuardConfiguration { MonitoringInterval = TimeSpan.FromSeconds(5), @@ -35,6 +61,30 @@ //MemoryGuard.Start(_config); + _ = Task.Run(async () => + { + var loggerSettings = new LoggerSettings(); + loggerSettings.Id = "SPEEDTEST"; + loggerSettings.UseLocalTime = true; + loggerSettings.FileLoggerOptions.UseLocalTime = true; + loggerSettings.FileLoggerOptions.Category = "SpeedTests"; + loggerSettings.FileLoggerOptions.EnableCategoryRouting = true; + loggerSettings.AllowAllLogTypes(); + var logger = new LogManager(loggerSettings); + + var i = 0; + while (true) + { + i++; + await logger.WriteAsync($"test to file {i} INFO").ConfigureAwait(false); + await logger.WriteAsync($"test to file {i} CRITICAL", ELogType.CRITICAL).ConfigureAwait(false); + await logger.WriteAsync($"test to file {i} DEBUG", ELogType.DEBUG).ConfigureAwait(false); + await logger.WriteAsync($"test to file {i} ERROR", ELogType.ERROR).ConfigureAwait(false); + await logger.WriteAsync($"test to file {i} TRACE", ELogType.TRACE).ConfigureAwait(false); + await Task.Delay(1).ConfigureAwait(false); + } + }); + var builder = WebApplication.CreateBuilder(args); int onLogCounter = 0; var defaultColor = Console.ForegroundColor; @@ -215,6 +265,8 @@ loggerSettings.FileLoggerOptions.UseLocalTime = true; loggerSettings.UseLocalTime = true; loggerSettings.Id = "TEST"; + loggerSettings.FileLoggerOptions.Category = "ExceptionTests"; + loggerSettings.FileLoggerOptions.EnableCategoryRouting = true; loggerSettings.TypesToLog.Add(ELogType.INFO); var logger = new LogManager(loggerSettings); @@ -234,7 +286,7 @@ } } - MemoryLeakTester.Start(logger); + //MemoryLeakTester.Start(logger); _ = Task.Run(RunMemoryReportTask).ConfigureAwait(false); _ = Task.Run(RunMaskTest).ConfigureAwait(false); _ = Task.Run(RunWebLoggerTestsAsync).ConfigureAwait(false);