Updated

EonaCat.Logger/EonaCat.Logger.csproj

@@ -13,8 +13,8 @@
     <Copyright>EonaCat (Jeroen Saey)</Copyright>
     <PackageTags>EonaCat;Logger;EonaCatLogger;Log;Writer;Jeroen;Saey</PackageTags>
     <PackageIconUrl />
-    <Version>1.6.8</Version>
+    <Version>1.6.9</Version>
-    <FileVersion>1.6.8</FileVersion>
+    <FileVersion>1.6.9</FileVersion>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <GenerateDocumentationFile>True</GenerateDocumentationFile>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
@@ -25,7 +25,7 @@
   </PropertyGroup>
 
 	<PropertyGroup>
-		<EVRevisionFormat>1.6.8+{chash:10}.{c:ymd}</EVRevisionFormat>
+		<EVRevisionFormat>1.6.9+{chash:10}.{c:ymd}</EVRevisionFormat>
 		<EVDefault>true</EVDefault>
 		<EVInfo>true</EVInfo>
 		<EVTagMatch>v[0-9]*</EVTagMatch>

EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs

@@ -4,47 +4,53 @@ using EonaCat.Logger.Managers;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using System;
-using System.Buffers;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading;
-using System.Threading.Channels;
 using System.Threading.Tasks;
 
 [ProviderAlias("EonaCatFileLogger")]
-public sealed class FileLoggerProvider : ILoggerProvider
+public sealed class FileLoggerProvider : BatchingLoggerProvider
 {
     private readonly string _path;
     private readonly string _fileNamePrefix;
     private readonly int _maxFileSize;
     private readonly int _maxRetainedFiles;
-    private readonly LoggerScopedContext _context = new LoggerScopedContext();
+    private readonly int _maxRolloverFiles;
-    private readonly Channel<LogMessage> _channel;
-    private readonly CancellationTokenSource _cts = new();
-    private readonly Task _writerTask;
-    private readonly ConcurrentDictionary<string, FileState> _files = new();
 
-    private const int BufferSize = 4 * 1024 * 1024; // 4 MB buffer for large messages
+    private readonly LoggerScopedContext _context = new LoggerScopedContext();
+    private readonly ConcurrentDictionary<string, FileState> _files = new();
+    private readonly ConcurrentDictionary<string, ConcurrentQueue<LogMessage>> _messageQueues = new();
+
+    private const int BufferSize = 1024 * 1024; // 1 MB buffer for large JSON logs
     private static readonly Encoding Utf8 = new UTF8Encoding(false);
 
     public bool IncludeCorrelationId { get; }
     public bool EnableCategoryRouting { get; }
 
-    public string LogFile => _files.TryGetValue(string.Empty, out var state) ? state.FilePath : string.Empty;
+    public string LogFile => _files.TryGetValue(string.Empty, out var state) ? state.FilePath : null;
+
+    public event EventHandler<ErrorMessage> OnError;
+    public event EventHandler<string> OnRollOver;
+
+    private readonly Timer _flushTimer;
+    private readonly TimeSpan _flushInterval = TimeSpan.FromMilliseconds(500);
 
     private sealed class FileState
     {
-        public FileStream Stream;
+        public string FilePath;
-        public byte[] Buffer = ArrayPool<byte>.Shared.Rent(BufferSize);
-        public int BufferPosition;
         public long Size;
         public DateTime Date;
-        public string FilePath;
+        public byte[] Buffer = new byte[BufferSize];
+        public int BufferPosition;
+        public FileStream Stream;
+        public SemaphoreSlim WriteLock = new(1, 1); // async safe
     }
 
-    public FileLoggerProvider(IOptions<FileLoggerOptions> options)
+    public FileLoggerProvider(IOptions<FileLoggerOptions> options) : base(options)
     {
         var o = options.Value ?? throw new ArgumentNullException(nameof(options));
 
@@ -52,111 +58,193 @@ public sealed class FileLoggerProvider : ILoggerProvider
         _fileNamePrefix = o.FileNamePrefix;
         _maxFileSize = o.FileSizeLimit;
         _maxRetainedFiles = o.RetainedFileCountLimit;
+        _maxRolloverFiles = o.MaxRolloverFiles;
         IncludeCorrelationId = o.IncludeCorrelationId;
         EnableCategoryRouting = o.EnableCategoryRouting;
 
         Directory.CreateDirectory(_path);
 
-        _channel = Channel.CreateUnbounded<LogMessage>(new UnboundedChannelOptions
+        var defaultState = CreateFileState(DateTime.UtcNow.Date, o.Category);
-        {
+        _files[string.Empty] = defaultState;
-            SingleReader = true,
-            SingleWriter = false
-        });
 
-        // Start writer task
+        // Periodic flush
-        _writerTask = Task.Run(ProcessQueueAsync);
+        _flushTimer = new Timer(_ => PeriodicFlushAsync().ConfigureAwait(false), null, _flushInterval, _flushInterval);
-
-        // Start background cleanup task
-        Task.Run(BackgroundCleanupAsync);
     }
 
-    public ILogger CreateLogger(string category) => new Logger(_channel, category);
+    internal override Task WriteMessagesAsync(IReadOnlyList<LogMessage> messages, CancellationToken token)
-
-    public void Dispose()
     {
-        _cts.Cancel();
+        try
-        _writerTask.Wait();
-
-        foreach (var state in _files.Values)
         {
-            FlushAndDispose(state);
+            if (EnableCategoryRouting)
-        }
-    }
-
-    private async Task ProcessQueueAsync()
-    {
-        await foreach (var msg in _channel.Reader.ReadAllAsync(_cts.Token))
-        {
-            var categoryKey = EnableCategoryRouting ? SanitizeCategory(msg.Category) : string.Empty;
-            var state = _files.GetOrAdd(categoryKey, _ => CreateFileState(categoryKey));
-
-            var text = BuildMessage(msg);
-            var bytes = Utf8.GetBytes(text);
-
-            if (bytes.Length > BufferSize)
             {
-                await state.Stream.WriteAsync(bytes, 0, bytes.Length, _cts.Token);
+                var grouped = messages.GroupBy(m => SanitizeCategory(m.Category));
-                state.Size += bytes.Length;
+                foreach (var group in grouped)
+                {
+                    var key = group.Key;
+                    var queue = _messageQueues.GetOrAdd(key, _ => new ConcurrentQueue<LogMessage>());
+                    foreach (var msg in group)
+                    {
+                        queue.Enqueue(msg);
+                    }
+                }
             }
             else
             {
-                if (state.BufferPosition + bytes.Length > BufferSize)
+                var queue = _messageQueues.GetOrAdd(string.Empty, _ => new ConcurrentQueue<LogMessage>());
+                foreach (var msg in messages)
                 {
-                    await FlushBufferAsync(state);
+                    queue.Enqueue(msg);
                 }
-
-                Array.Copy(bytes, 0, state.Buffer, state.BufferPosition, bytes.Length);
-                state.BufferPosition += bytes.Length;
-                state.Size += bytes.Length;
-            }
-
-            if (state.Size >= _maxFileSize)
-            {
-                await RollOverAsync(state, categoryKey);
             }
         }
+        catch (Exception ex)
+        {
+            OnError?.Invoke(this, new ErrorMessage { Exception = ex, Message = ex.Message });
+        }
+
+        return Task.CompletedTask;
     }
 
-    private FileState CreateFileState(string category)
+    private async Task PeriodicFlushAsync()
     {
-        var path = GetFullPath(category, DateTime.UtcNow);
+        foreach (var kv in _messageQueues)
-        var stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete,
+        {
-            4096, useAsync: true);
+            var key = kv.Key;
+            var queue = kv.Value;
+
+            if (!_files.TryGetValue(key, out var state))
+            {
+                state = CreateFileState(DateTime.UtcNow.Date, key);
+                _files[key] = state;
+            }
+
+            var messagesToWrite = new List<LogMessage>();
+            while (queue.TryDequeue(out var msg))
+            {
+                messagesToWrite.Add(msg);
+            }
+
+            if (messagesToWrite.Count > 0)
+            {
+                try
+                {
+                    await state.WriteLock.WaitAsync();
+                    WriteBatch(state, messagesToWrite, key);
+                }
+                catch (Exception ex)
+                {
+                    OnError?.Invoke(this, new ErrorMessage { Exception = ex, Message = ex.Message });
+                }
+                finally
+                {
+                    state.WriteLock.Release();
+                }
+            }
+        }
+
+        DeleteOldLogFiles();
+    }
+
+    private void WriteBatch(FileState state, IEnumerable<LogMessage> messages, string categoryKey)
+    {
+        if (!File.Exists(state.FilePath))
+        {
+            RecreateFile(state, categoryKey);
+        }
+
+        foreach (var msg in messages)
+        {
+            var date = msg.Timestamp.UtcDateTime.Date;
+
+            if (state.Date != date)
+            {
+                FlushBufferAsync(state).GetAwaiter().GetResult();
+                RotateByDate(state, date, categoryKey);
+            }
+
+            WriteMessageToBuffer(state, msg);
+
+            if (state.BufferPosition >= BufferSize - 1024 || state.Size >= _maxFileSize)
+            {
+                FlushBufferAsync(state).GetAwaiter().GetResult();
+
+                if (state.Size >= _maxFileSize)
+                {
+                    RollOver(state, categoryKey);
+                }
+            }
+        }
+
+        FlushBufferAsync(state).GetAwaiter().GetResult();
+    }
+
+    private FileState CreateFileState(DateTime date, string category)
+    {
+        var path = GetFullName(date, category);
 
         return new FileState
         {
-            Stream = stream,
-            Date = DateTime.UtcNow.Date,
             FilePath = path,
-            Size = stream.Length
+            Date = date,
+            Size = GetFileSize(path),
+            Stream = OpenFileWithRetryAsync(path).GetAwaiter().GetResult()
         };
     }
 
-    private async Task FlushBufferAsync(FileState state)
+    private static async Task<FileStream> OpenFileWithRetryAsync(string path)
     {
-        if (state.BufferPosition == 0)
+        const int retries = 3;
+        for (int i = 0; i < retries; i++)
         {
-            return;
+            try
+            {
+                return new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
+            }
+            catch
+            {
+                await Task.Delay(5);
+            }
         }
 
-        await state.Stream.WriteAsync(state.Buffer, 0, state.BufferPosition, _cts.Token);
+        throw new IOException("Unable to open log file.");
-        state.BufferPosition = 0;
-        await state.Stream.FlushAsync(_cts.Token);
     }
 
-    private async Task RollOverAsync(FileState state, string category)
+    private void RecreateFile(FileState state, string category)
     {
-        await FlushBufferAsync(state);
+        FlushBufferAsync(state).GetAwaiter().GetResult();
-        state.Stream.Dispose();
+        state.Stream?.Dispose();
+
+        state.FilePath = GetFullName(DateTime.UtcNow.Date, category);
+        state.Size = 0;
+        state.BufferPosition = 0;
+        state.Stream = OpenFileWithRetryAsync(state.FilePath).GetAwaiter().GetResult();
+    }
+
+    private void RotateByDate(FileState state, DateTime newDate, string category)
+    {
+        state.Stream?.Dispose();
+
+        state.Date = newDate;
+        state.FilePath = GetFullName(newDate, category);
+        state.Size = GetFileSize(state.FilePath);
+        state.BufferPosition = 0;
+        state.Stream = OpenFileWithRetryAsync(state.FilePath).GetAwaiter().GetResult();
+    }
+
+    private void RollOver(FileState state, string category)
+    {
+        FlushBufferAsync(state).GetAwaiter().GetResult();
+        state.Stream?.Dispose();
 
         var dir = Path.GetDirectoryName(state.FilePath);
         var name = Path.GetFileNameWithoutExtension(state.FilePath);
         var ext = Path.GetExtension(state.FilePath);
 
-        for (int i = _maxRetainedFiles - 1; i >= 1; i--)
+        for (int i = _maxRolloverFiles - 1; i >= 1; i--)
         {
             var src = Path.Combine(dir, $"{name}.{i}{ext}");
             var dst = Path.Combine(dir, $"{name}.{i + 1}{ext}");
+
             if (File.Exists(dst))
             {
                 File.Delete(dst);
@@ -174,14 +262,95 @@ public sealed class FileLoggerProvider : ILoggerProvider
             File.Move(state.FilePath, first);
         }
 
-        state.FilePath = GetFullPath(category, DateTime.UtcNow);
+        OnRollOver?.Invoke(this, state.FilePath);
-        state.Stream = new FileStream(state.FilePath, FileMode.Append, FileAccess.Write,
+
-            FileShare.ReadWrite | FileShare.Delete, 4096, useAsync: true);
         state.Size = 0;
         state.BufferPosition = 0;
+        state.Stream = OpenFileWithRetryAsync(state.FilePath).GetAwaiter().GetResult();
+    }
+
+    private void WriteMessageToBuffer(FileState state, LogMessage msg)
+    {
+        var text = BuildMessage(msg);
+        var byteCount = Utf8.GetByteCount(text);
+
+        if (state.BufferPosition + byteCount > BufferSize)
+        {
+            FlushBufferAsync(state).GetAwaiter().GetResult();
+        }
+
+        var written = Utf8.GetBytes(text, 0, text.Length, state.Buffer, state.BufferPosition);
+        state.BufferPosition += written;
+        state.Size += written;
+    }
+
+    private string BuildMessage(LogMessage msg)
+    {
+        var settings = msg.Settings ?? LoggerSettings;
+
+        if (!IncludeCorrelationId)
+        {
+            return msg.Message + Environment.NewLine;
+        }
+
+        var ctx = _context.GetAll();
+        if (ctx.Count == 0)
+        {
+            return msg.Message + Environment.NewLine;
+        }
+
+        var sb = new StringBuilder(256);
+        sb.Append(msg.Message).Append(" [");
+
+        bool first = true;
+        foreach (var kv in ctx)
+        {
+            if (!first)
+            {
+                sb.Append(' ');
+            }
+
+            sb.Append(kv.Key).Append('=').Append(kv.Value);
+            first = false;
+        }
+
+        sb.Append(']').AppendLine();
+        return sb.ToString();
+    }
+
+    private async Task FlushBufferAsync(FileState state)
+    {
+        if (state.BufferPosition == 0 || state.Stream == null)
+        {
+            return;
+        }
+
+        await state.Stream.WriteAsync(state.Buffer, 0, state.BufferPosition);
+        await state.Stream.FlushAsync();
+        state.BufferPosition = 0;
     }
 
-    private string GetFullPath(string category, DateTime date)
+    private static long GetFileSize(string path) => File.Exists(path) ? new FileInfo(path).Length : 0;
+
+    private void DeleteOldLogFiles()
+    {
+        if (_maxRetainedFiles <= 0)
+        {
+            return;
+        }
+
+        var files = new DirectoryInfo(_path)
+            .GetFiles($"{_fileNamePrefix}*")
+            .OrderByDescending(f => f.LastWriteTimeUtc)
+            .Skip(_maxRetainedFiles);
+
+        foreach (var f in files)
+        {
+            try { f.Delete(); } catch { }
+        }
+    }
+
+    private string GetFullName(DateTime date, string category)
     {
         var datePart = date.ToString("yyyyMMdd");
         var machine = Environment.MachineName;
@@ -195,109 +364,34 @@ public sealed class FileLoggerProvider : ILoggerProvider
         return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{safeCategory}_{datePart}.log");
     }
 
-    private string BuildMessage(LogMessage msg)
-    {
-        if (!IncludeCorrelationId)
-        {
-            return msg.Message + Environment.NewLine;
-        }
-
-        var ctx = _context.GetAll();
-        if (ctx.Count == 0)
-        {
-            return msg.Message + Environment.NewLine;
-        }
-
-        var sb = new StringBuilder(msg.Message.Length + 128);
-        sb.Append(msg.Message).Append(" [");
-        bool first = true;
-        foreach (var kv in ctx)
-        {
-            if (!first)
-            {
-                sb.Append(' ');
-            }
-
-            sb.Append(kv.Key).Append('=').Append(kv.Value);
-            first = false;
-        }
-        sb.Append(']').AppendLine();
-        return sb.ToString();
-    }
-
     private static string SanitizeCategory(string category)
     {
         foreach (var c in Path.GetInvalidFileNameChars())
         {
             category = category.Replace(c, '_');
         }
-
         return category.Replace('.', '_');
     }
 
-    private void FlushAndDispose(FileState state)
+    protected override void OnShutdownFlush()
     {
-        try
+        _flushTimer?.Dispose();
-        {
+        PeriodicFlushAsync().GetAwaiter().GetResult();
-            if (state.BufferPosition > 0)
-            {
-                state.Stream.Write(state.Buffer, 0, state.BufferPosition);
-                state.Stream.Flush();
-            }
-            state.Stream.Dispose();
-            ArrayPool<byte>.Shared.Return(state.Buffer);
-        }
-        catch { }
-    }
 
-    private async Task BackgroundCleanupAsync()
+        foreach (var state in _files.Values)
-    {
-        while (!_cts.Token.IsCancellationRequested)
         {
             try
             {
-                foreach (var file in new DirectoryInfo(_path).GetFiles($"{_fileNamePrefix}*"))
+                FlushBufferAsync(state).GetAwaiter().GetResult();
-                {
+                state.Stream?.Dispose();
-                    var files = new DirectoryInfo(_path).GetFiles($"{_fileNamePrefix}*");
+            }
-                    Array.Sort(files, (a, b) => b.LastWriteTimeUtc.CompareTo(a.LastWriteTimeUtc));
+            catch 
-
+            {
-                    for (int i = _maxRetainedFiles; i < files.Length; i++)
+                // Do nothing
-                    {
-                        try { files[i].Delete(); } catch { }
-                    }
-                }
             }
-            catch { }
-
-            await Task.Delay(TimeSpan.FromSeconds(30), _cts.Token); // cleanup every 30s
-        }
-    }
-
-    private sealed class Logger : ILogger
-    {
-        private readonly Channel<LogMessage> _channel;
-        private readonly string _category;
-
-        public Logger(Channel<LogMessage> channel, string category)
-        {
-            _channel = channel;
-            _category = category;
         }
 
-        public IDisposable BeginScope<TState>(TState state) => null;
+        _files.Clear();
-        public bool IsEnabled(LogLevel logLevel) => true;
+        _messageQueues.Clear();
-
-        public void Log<TState>(LogLevel logLevel, EventId eventId, TState state,
-            Exception exception, Func<TState, Exception, string> formatter)
-        {
-            var msg = formatter(state, exception);
-            _channel.Writer.TryWrite(new LogMessage { Message = msg, Category = _category });
-        }
-    }
-
-    private sealed class LogMessage
-    {
-        public string Message;
-        public string Category;
     }
 }