Updated

EonaCat.Logger/EonaCat.Logger.csproj

@@ -13,8 +13,8 @@
     <Copyright>EonaCat (Jeroen Saey)</Copyright>
     <PackageTags>EonaCat;Logger;EonaCatLogger;Log;Writer;Jeroen;Saey</PackageTags>
     <PackageIconUrl />
-    <Version>1.6.7</Version>
+    <Version>1.6.8</Version>
-    <FileVersion>1.6.7</FileVersion>
+    <FileVersion>1.6.8</FileVersion>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <GenerateDocumentationFile>True</GenerateDocumentationFile>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
@@ -25,7 +25,7 @@
   </PropertyGroup>
 
 	<PropertyGroup>
-		<EVRevisionFormat>1.6.7+{chash:10}.{c:ymd}</EVRevisionFormat>
+		<EVRevisionFormat>1.6.8+{chash:10}.{c:ymd}</EVRevisionFormat>
 		<EVDefault>true</EVDefault>
 		<EVInfo>true</EVInfo>
 		<EVTagMatch>v[0-9]*</EVTagMatch>

EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs

@@ -4,48 +4,45 @@ using EonaCat.Logger.Managers;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using System;
+using System.Buffers;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Text;
 using System.Threading;
+using System.Threading.Channels;
 using System.Threading.Tasks;
 
 [ProviderAlias("EonaCatFileLogger")]
-public sealed class FileLoggerProvider : BatchingLoggerProvider
+public sealed class AsyncFileLoggerProvider : ILoggerProvider
 {
     private readonly string _path;
     private readonly string _fileNamePrefix;
     private readonly int _maxFileSize;
     private readonly int _maxRetainedFiles;
-    private readonly int _maxRolloverFiles;
-
     private readonly LoggerScopedContext _context = new LoggerScopedContext();
-    private readonly ConcurrentDictionary<string, FileState> _files = new ConcurrentDictionary<string, FileState>();
+    private readonly Channel<LogMessage> _channel;
+    private readonly CancellationTokenSource _cts = new();
+    private readonly Task _writerTask;
+    private readonly ConcurrentDictionary<string, FileState> _files = new();
 
-    private const int BufferSize = 256 * 1024;
+    private const int BufferSize = 4 * 1024 * 1024; // 4 MB buffer for large messages
     private static readonly Encoding Utf8 = new UTF8Encoding(false);
 
     public bool IncludeCorrelationId { get; }
     public bool EnableCategoryRouting { get; }
 
-    public string LogFile => _files.TryGetValue(string.Empty, out var state) ? state.FilePath : null;
-
-    public event EventHandler<ErrorMessage> OnError;
-    public event EventHandler<string> OnRollOver;
-
     private sealed class FileState
     {
-        public string FilePath;
+        public FileStream Stream;
+        public byte[] Buffer = ArrayPool<byte>.Shared.Rent(BufferSize);
+        public int BufferPosition;
         public long Size;
         public DateTime Date;
-        public byte[] Buffer = new byte[BufferSize];
+        public string FilePath;
-        public int BufferPosition;
-        public FileStream Stream;
     }
 
-    public FileLoggerProvider(IOptions<FileLoggerOptions> options) : base(options)
+    public AsyncFileLoggerProvider(IOptions<FileLoggerOptions> options)
     {
         var o = options.Value ?? throw new ArgumentNullException(nameof(options));
 
@@ -53,155 +50,111 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider
         _fileNamePrefix = o.FileNamePrefix;
         _maxFileSize = o.FileSizeLimit;
         _maxRetainedFiles = o.RetainedFileCountLimit;
-        _maxRolloverFiles = o.MaxRolloverFiles;
         IncludeCorrelationId = o.IncludeCorrelationId;
         EnableCategoryRouting = o.EnableCategoryRouting;
 
         Directory.CreateDirectory(_path);
 
-        var defaultState = CreateFileState(DateTime.UtcNow.Date, o.Category);
+        _channel = Channel.CreateUnbounded<LogMessage>(new UnboundedChannelOptions
-        _files[string.Empty] = defaultState;
+        {
+            SingleReader = true,
+            SingleWriter = false
+        });
+
+        // Start writer task
+        _writerTask = Task.Run(ProcessQueueAsync);
+
+        // Start background cleanup task
+        Task.Run(BackgroundCleanupAsync);
     }
 
-    internal override Task WriteMessagesAsync(IReadOnlyList<LogMessage> messages, CancellationToken token)
+    public ILogger CreateLogger(string category) => new Logger(_channel, category);
+
+    public void Dispose()
     {
-        try
+        _cts.Cancel();
+        _writerTask.Wait();
+
+        foreach (var state in _files.Values)
         {
-            if (EnableCategoryRouting)
+            FlushAndDispose(state);
+        }
+    }
+
+    private async Task ProcessQueueAsync()
+    {
+        await foreach (var msg in _channel.Reader.ReadAllAsync(_cts.Token))
+        {
+            var categoryKey = EnableCategoryRouting ? SanitizeCategory(msg.Category) : string.Empty;
+            var state = _files.GetOrAdd(categoryKey, _ => CreateFileState(categoryKey));
+
+            var text = BuildMessage(msg);
+            var bytes = Utf8.GetBytes(text);
+
+            if (bytes.Length > BufferSize)
             {
-                var grouped = messages.GroupBy(m => SanitizeCategory(m.Category));
+                await state.Stream.WriteAsync(bytes, 0, bytes.Length, _cts.Token);
-
+                state.Size += bytes.Length;
-                foreach (var group in grouped)
-                {
-                    var categoryKey = group.Key;
-
-                    var state = _files.GetOrAdd(categoryKey,
-                        _ => CreateFileState(DateTime.UtcNow.Date, categoryKey));
-
-                    WriteBatch(state, group, categoryKey);
-                }
             }
             else
             {
-                var state = _files.GetOrAdd(string.Empty,
+                if (state.BufferPosition + bytes.Length > BufferSize)
-                    _ => CreateFileState(DateTime.UtcNow.Date, string.Empty));
-
-                WriteBatch(state, messages, string.Empty);
-            }
-
-            DeleteOldLogFiles();
-        }
-        catch (Exception ex)
-        {
-            OnError?.Invoke(this, new ErrorMessage { Exception = ex, Message = ex.Message });
-        }
-
-        return Task.CompletedTask;
-    }
-
-    private void WriteBatch(FileState state, IEnumerable<LogMessage> messages, string categoryKey)
-    {
-        if (!File.Exists(state.FilePath))
-        {
-            RecreateFile(state, categoryKey);
-        }
-
-        foreach (var msg in messages)
-        {
-            var date = msg.Timestamp.UtcDateTime.Date;
-
-            if (state.Date != date)
-            {
-                FlushBuffer(state);
-                RotateByDate(state, date, categoryKey);
-            }
-
-            WriteMessageToBuffer(state, msg);
-
-            if (state.BufferPosition >= BufferSize - 1024 || state.Size >= _maxFileSize)
-            {
-                FlushBuffer(state);
-
-                if (state.Size >= _maxFileSize)
                 {
-                    RollOver(state, categoryKey);
+                    await FlushBufferAsync(state);
                 }
+
+                Array.Copy(bytes, 0, state.Buffer, state.BufferPosition, bytes.Length);
+                state.BufferPosition += bytes.Length;
+                state.Size += bytes.Length;
+            }
+
+            if (state.Size >= _maxFileSize)
+            {
+                await RollOverAsync(state, categoryKey);
             }
         }
-
-        FlushBuffer(state);
     }
 
-    private FileState CreateFileState(DateTime date, string category)
+    private FileState CreateFileState(string category)
     {
-        var path = GetFullName(date, category);
+        var path = GetFullPath(category, DateTime.UtcNow);
+        var stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete,
+            4096, useAsync: true);
 
         return new FileState
         {
+            Stream = stream,
+            Date = DateTime.UtcNow.Date,
             FilePath = path,
-            Date = date,
+            Size = stream.Length
-            Size = GetFileSize(path),
-            Stream = OpenFileWithRetry(path)
         };
     }
 
-    private static FileStream OpenFileWithRetry(string path)
+    private async Task FlushBufferAsync(FileState state)
     {
-        const int retries = 3;
+        if (state.BufferPosition == 0)
-
-        for (int i = 0; i < retries; i++)
         {
-            try
+            return;
-            {
-                return new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.SequentialScan);
-            }
-            catch
-            {
-                Thread.Sleep(5);
-            }
         }
 
-        throw new IOException("Unable to open log file.");
+        await state.Stream.WriteAsync(state.Buffer, 0, state.BufferPosition, _cts.Token);
-    }
-
-    private void RecreateFile(FileState state, string category)
-    {
-        FlushBuffer(state);
-        state.Stream?.Dispose();
-
-        state.FilePath = GetFullName(DateTime.UtcNow.Date, category);
-        state.Size = 0;
         state.BufferPosition = 0;
-
+        await state.Stream.FlushAsync(_cts.Token);
-        state.Stream = OpenFileWithRetry(state.FilePath);
     }
 
-    private void RotateByDate(FileState state, DateTime newDate, string category)
+    private async Task RollOverAsync(FileState state, string category)
     {
-        state.Stream?.Dispose();
+        await FlushBufferAsync(state);
-
+        state.Stream.Dispose();
-        state.Date = newDate;
-        state.FilePath = GetFullName(newDate, category);
-        state.Size = GetFileSize(state.FilePath);
-        state.BufferPosition = 0;
-
-        state.Stream = OpenFileWithRetry(state.FilePath);
-    }
-
-    private void RollOver(FileState state, string category)
-    {
-        FlushBuffer(state);
-        state.Stream?.Dispose();
 
         var dir = Path.GetDirectoryName(state.FilePath);
         var name = Path.GetFileNameWithoutExtension(state.FilePath);
         var ext = Path.GetExtension(state.FilePath);
 
-        for (int i = _maxRolloverFiles - 1; i >= 1; i--)
+        for (int i = _maxRetainedFiles - 1; i >= 1; i--)
         {
             var src = Path.Combine(dir, $"{name}.{i}{ext}");
             var dst = Path.Combine(dir, $"{name}.{i + 1}{ext}");
-
             if (File.Exists(dst))
             {
                 File.Delete(dst);
@@ -219,32 +172,29 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider
             File.Move(state.FilePath, first);
         }
 
-        OnRollOver?.Invoke(this, state.FilePath);
+        state.FilePath = GetFullPath(category, DateTime.UtcNow);
-
+        state.Stream = new FileStream(state.FilePath, FileMode.Append, FileAccess.Write,
+            FileShare.ReadWrite | FileShare.Delete, 4096, useAsync: true);
         state.Size = 0;
         state.BufferPosition = 0;
-        state.Stream = OpenFileWithRetry(state.FilePath);
     }
 
-    private void WriteMessageToBuffer(FileState state, LogMessage msg)
+    private string GetFullPath(string category, DateTime date)
     {
-        var text = BuildMessage(msg);
+        var datePart = date.ToString("yyyyMMdd");
-        var byteCount = Utf8.GetByteCount(text);
+        var machine = Environment.MachineName;
 
-        if (state.BufferPosition + byteCount > BufferSize)
+        if (!EnableCategoryRouting || string.IsNullOrWhiteSpace(category))
         {
-            FlushBuffer(state);
+            return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{datePart}.log");
         }
 
-        var written = Utf8.GetBytes(text, 0, text.Length, state.Buffer, state.BufferPosition);
+        var safeCategory = SanitizeCategory(category);
-        state.BufferPosition += written;
+        return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{safeCategory}_{datePart}.log");
-        state.Size += written;
     }
 
     private string BuildMessage(LogMessage msg)
     {
-        var settings = msg.Settings ?? LoggerSettings;
-
         if (!IncludeCorrelationId)
         {
             return msg.Message + Environment.NewLine;
@@ -256,9 +206,8 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider
             return msg.Message + Environment.NewLine;
         }
 
-        var sb = new StringBuilder(256);
+        var sb = new StringBuilder(msg.Message.Length + 128);
         sb.Append(msg.Message).Append(" [");
-
         bool first = true;
         foreach (var kv in ctx)
         {
@@ -270,59 +219,10 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider
             sb.Append(kv.Key).Append('=').Append(kv.Value);
             first = false;
         }
-
         sb.Append(']').AppendLine();
         return sb.ToString();
     }
 
-    private static void FlushBuffer(FileState state)
-    {
-        if (state.BufferPosition == 0 || state.Stream == null)
-        {
-            return;
-        }
-
-        state.Stream.Write(state.Buffer, 0, state.BufferPosition);
-        state.Stream.Flush();
-        state.BufferPosition = 0;
-    }
-
-    private static long GetFileSize(string path)
-        => File.Exists(path) ? new FileInfo(path).Length : 0;
-
-    private void DeleteOldLogFiles()
-    {
-        if (_maxRetainedFiles <= 0)
-        {
-            return;
-        }
-
-        var files = new DirectoryInfo(_path)
-            .GetFiles($"{_fileNamePrefix}*")
-            .OrderByDescending(f => f.LastWriteTimeUtc)
-            .Skip(_maxRetainedFiles);
-
-        foreach (var f in files)
-        {
-            try { f.Delete(); } catch { }
-        }
-    }
-
-    private string GetFullName(DateTime date, string category)
-    {
-        var datePart = date.ToString("yyyyMMdd");
-        var machine = Environment.MachineName;
-
-        if (!EnableCategoryRouting || string.IsNullOrWhiteSpace(category))
-        {
-            return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{datePart}.log");
-        }
-
-        var safeCategory = SanitizeCategory(category);
-
-        return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{safeCategory}_{datePart}.log");
-    }
-
     private static string SanitizeCategory(string category)
     {
         foreach (var c in Path.GetInvalidFileNameChars())
@@ -333,21 +233,69 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider
         return category.Replace('.', '_');
     }
 
-    protected override void OnShutdownFlush()
+    private void FlushAndDispose(FileState state)
     {
-        foreach (var state in _files.Values)
+        try
+        {
+            if (state.BufferPosition > 0)
+            {
+                state.Stream.Write(state.Buffer, 0, state.BufferPosition);
+                state.Stream.Flush();
+            }
+            state.Stream.Dispose();
+            ArrayPool<byte>.Shared.Return(state.Buffer);
+        }
+        catch { }
+    }
+
+    private async Task BackgroundCleanupAsync()
+    {
+        while (!_cts.Token.IsCancellationRequested)
         {
             try
             {
-                FlushBuffer(state);
+                foreach (var file in new DirectoryInfo(_path).GetFiles($"{_fileNamePrefix}*"))
-                state.Stream?.Dispose();
+                {
-            }
+                    var files = new DirectoryInfo(_path).GetFiles($"{_fileNamePrefix}*");
-            catch
+                    Array.Sort(files, (a, b) => b.LastWriteTimeUtc.CompareTo(a.LastWriteTimeUtc));
-            {
+
-                // Do nothing during shutdown flush
+                    for (int i = _maxRetainedFiles; i < files.Length; i++)
+                    {
+                        try { files[i].Delete(); } catch { }
+                    }
+                }
             }
+            catch { }
+
+            await Task.Delay(TimeSpan.FromSeconds(30), _cts.Token); // cleanup every 30s
+        }
+    }
+
+    private sealed class Logger : ILogger
+    {
+        private readonly Channel<LogMessage> _channel;
+        private readonly string _category;
+
+        public Logger(Channel<LogMessage> channel, string category)
+        {
+            _channel = channel;
+            _category = category;
         }
 
-        _files.Clear();
+        public IDisposable BeginScope<TState>(TState state) => null;
+        public bool IsEnabled(LogLevel logLevel) => true;
+
+        public void Log<TState>(LogLevel logLevel, EventId eventId, TState state,
+            Exception exception, Func<TState, Exception, string> formatter)
+        {
+            var msg = formatter(state, exception);
+            _channel.Writer.TryWrite(new LogMessage { Message = msg, Category = _category });
+        }
+    }
+
+    private sealed class LogMessage
+    {
+        public string Message;
+        public string Category;
     }
 }