diff --git a/EonaCat.Logger/EonaCat.Logger.csproj b/EonaCat.Logger/EonaCat.Logger.csproj index efda1e7..b1862b4 100644 --- a/EonaCat.Logger/EonaCat.Logger.csproj +++ b/EonaCat.Logger/EonaCat.Logger.csproj @@ -13,8 +13,8 @@ EonaCat (Jeroen Saey) EonaCat;Logger;EonaCatLogger;Log;Writer;Jeroen;Saey - 1.6.7 - 1.6.7 + 1.6.8 + 1.6.8 README.md True LICENSE @@ -25,7 +25,7 @@ - 1.6.7+{chash:10}.{c:ymd} + 1.6.8+{chash:10}.{c:ymd} true true v[0-9]* diff --git a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs index bba28e7..1437872 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs @@ -4,48 +4,45 @@ using EonaCat.Logger.Managers; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Text; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; [ProviderAlias("EonaCatFileLogger")] -public sealed class FileLoggerProvider : BatchingLoggerProvider +public sealed class AsyncFileLoggerProvider : ILoggerProvider { private readonly string _path; private readonly string _fileNamePrefix; private readonly int _maxFileSize; private readonly int _maxRetainedFiles; - private readonly int _maxRolloverFiles; - private readonly LoggerScopedContext _context = new LoggerScopedContext(); - private readonly ConcurrentDictionary _files = new ConcurrentDictionary(); + private readonly Channel _channel; + private readonly CancellationTokenSource _cts = new(); + private readonly Task _writerTask; + private readonly ConcurrentDictionary _files = new(); - private const int BufferSize = 256 * 1024; + private const int BufferSize = 4 * 1024 * 1024; // 4 MB buffer for large messages private static readonly Encoding Utf8 = new UTF8Encoding(false); public bool IncludeCorrelationId { get; } public bool EnableCategoryRouting { get; } - public string LogFile => _files.TryGetValue(string.Empty, out var state) ? state.FilePath : null; - - public event EventHandler OnError; - public event EventHandler OnRollOver; - private sealed class FileState { - public string FilePath; + public FileStream Stream; + public byte[] Buffer = ArrayPool.Shared.Rent(BufferSize); + public int BufferPosition; public long Size; public DateTime Date; - public byte[] Buffer = new byte[BufferSize]; - public int BufferPosition; - public FileStream Stream; + public string FilePath; } - public FileLoggerProvider(IOptions options) : base(options) + public AsyncFileLoggerProvider(IOptions options) { var o = options.Value ?? throw new ArgumentNullException(nameof(options)); @@ -53,155 +50,111 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider _fileNamePrefix = o.FileNamePrefix; _maxFileSize = o.FileSizeLimit; _maxRetainedFiles = o.RetainedFileCountLimit; - _maxRolloverFiles = o.MaxRolloverFiles; IncludeCorrelationId = o.IncludeCorrelationId; EnableCategoryRouting = o.EnableCategoryRouting; Directory.CreateDirectory(_path); - var defaultState = CreateFileState(DateTime.UtcNow.Date, o.Category); - _files[string.Empty] = defaultState; + _channel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + SingleWriter = false + }); + + // Start writer task + _writerTask = Task.Run(ProcessQueueAsync); + + // Start background cleanup task + Task.Run(BackgroundCleanupAsync); } - internal override Task WriteMessagesAsync(IReadOnlyList messages, CancellationToken token) + public ILogger CreateLogger(string category) => new Logger(_channel, category); + + public void Dispose() { - try + _cts.Cancel(); + _writerTask.Wait(); + + foreach (var state in _files.Values) { - if (EnableCategoryRouting) + FlushAndDispose(state); + } + } + + private async Task ProcessQueueAsync() + { + await foreach (var msg in _channel.Reader.ReadAllAsync(_cts.Token)) + { + var categoryKey = EnableCategoryRouting ? SanitizeCategory(msg.Category) : string.Empty; + var state = _files.GetOrAdd(categoryKey, _ => CreateFileState(categoryKey)); + + var text = BuildMessage(msg); + var bytes = Utf8.GetBytes(text); + + if (bytes.Length > BufferSize) { - var grouped = messages.GroupBy(m => SanitizeCategory(m.Category)); - - foreach (var group in grouped) - { - var categoryKey = group.Key; - - var state = _files.GetOrAdd(categoryKey, - _ => CreateFileState(DateTime.UtcNow.Date, categoryKey)); - - WriteBatch(state, group, categoryKey); - } + await state.Stream.WriteAsync(bytes, 0, bytes.Length, _cts.Token); + state.Size += bytes.Length; } else { - var state = _files.GetOrAdd(string.Empty, - _ => CreateFileState(DateTime.UtcNow.Date, string.Empty)); - - WriteBatch(state, messages, string.Empty); - } - - DeleteOldLogFiles(); - } - catch (Exception ex) - { - OnError?.Invoke(this, new ErrorMessage { Exception = ex, Message = ex.Message }); - } - - return Task.CompletedTask; - } - - private void WriteBatch(FileState state, IEnumerable messages, string categoryKey) - { - if (!File.Exists(state.FilePath)) - { - RecreateFile(state, categoryKey); - } - - foreach (var msg in messages) - { - var date = msg.Timestamp.UtcDateTime.Date; - - if (state.Date != date) - { - FlushBuffer(state); - RotateByDate(state, date, categoryKey); - } - - WriteMessageToBuffer(state, msg); - - if (state.BufferPosition >= BufferSize - 1024 || state.Size >= _maxFileSize) - { - FlushBuffer(state); - - if (state.Size >= _maxFileSize) + if (state.BufferPosition + bytes.Length > BufferSize) { - RollOver(state, categoryKey); + await FlushBufferAsync(state); } + + Array.Copy(bytes, 0, state.Buffer, state.BufferPosition, bytes.Length); + state.BufferPosition += bytes.Length; + state.Size += bytes.Length; + } + + if (state.Size >= _maxFileSize) + { + await RollOverAsync(state, categoryKey); } } - - FlushBuffer(state); } - private FileState CreateFileState(DateTime date, string category) + private FileState CreateFileState(string category) { - var path = GetFullName(date, category); + var path = GetFullPath(category, DateTime.UtcNow); + var stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, + 4096, useAsync: true); return new FileState { + Stream = stream, + Date = DateTime.UtcNow.Date, FilePath = path, - Date = date, - Size = GetFileSize(path), - Stream = OpenFileWithRetry(path) + Size = stream.Length }; } - private static FileStream OpenFileWithRetry(string path) + private async Task FlushBufferAsync(FileState state) { - const int retries = 3; - - for (int i = 0; i < retries; i++) + if (state.BufferPosition == 0) { - try - { - return new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.SequentialScan); - } - catch - { - Thread.Sleep(5); - } + return; } - throw new IOException("Unable to open log file."); - } - - private void RecreateFile(FileState state, string category) - { - FlushBuffer(state); - state.Stream?.Dispose(); - - state.FilePath = GetFullName(DateTime.UtcNow.Date, category); - state.Size = 0; + await state.Stream.WriteAsync(state.Buffer, 0, state.BufferPosition, _cts.Token); state.BufferPosition = 0; - - state.Stream = OpenFileWithRetry(state.FilePath); + await state.Stream.FlushAsync(_cts.Token); } - private void RotateByDate(FileState state, DateTime newDate, string category) + private async Task RollOverAsync(FileState state, string category) { - state.Stream?.Dispose(); - - state.Date = newDate; - state.FilePath = GetFullName(newDate, category); - state.Size = GetFileSize(state.FilePath); - state.BufferPosition = 0; - - state.Stream = OpenFileWithRetry(state.FilePath); - } - - private void RollOver(FileState state, string category) - { - FlushBuffer(state); - state.Stream?.Dispose(); + await FlushBufferAsync(state); + state.Stream.Dispose(); var dir = Path.GetDirectoryName(state.FilePath); var name = Path.GetFileNameWithoutExtension(state.FilePath); var ext = Path.GetExtension(state.FilePath); - for (int i = _maxRolloverFiles - 1; i >= 1; i--) + for (int i = _maxRetainedFiles - 1; i >= 1; i--) { var src = Path.Combine(dir, $"{name}.{i}{ext}"); var dst = Path.Combine(dir, $"{name}.{i + 1}{ext}"); - if (File.Exists(dst)) { File.Delete(dst); @@ -219,32 +172,29 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider File.Move(state.FilePath, first); } - OnRollOver?.Invoke(this, state.FilePath); - + state.FilePath = GetFullPath(category, DateTime.UtcNow); + state.Stream = new FileStream(state.FilePath, FileMode.Append, FileAccess.Write, + FileShare.ReadWrite | FileShare.Delete, 4096, useAsync: true); state.Size = 0; state.BufferPosition = 0; - state.Stream = OpenFileWithRetry(state.FilePath); } - private void WriteMessageToBuffer(FileState state, LogMessage msg) + private string GetFullPath(string category, DateTime date) { - var text = BuildMessage(msg); - var byteCount = Utf8.GetByteCount(text); + var datePart = date.ToString("yyyyMMdd"); + var machine = Environment.MachineName; - if (state.BufferPosition + byteCount > BufferSize) + if (!EnableCategoryRouting || string.IsNullOrWhiteSpace(category)) { - FlushBuffer(state); + return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{datePart}.log"); } - var written = Utf8.GetBytes(text, 0, text.Length, state.Buffer, state.BufferPosition); - state.BufferPosition += written; - state.Size += written; + var safeCategory = SanitizeCategory(category); + return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{safeCategory}_{datePart}.log"); } private string BuildMessage(LogMessage msg) { - var settings = msg.Settings ?? LoggerSettings; - if (!IncludeCorrelationId) { return msg.Message + Environment.NewLine; @@ -256,9 +206,8 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider return msg.Message + Environment.NewLine; } - var sb = new StringBuilder(256); + var sb = new StringBuilder(msg.Message.Length + 128); sb.Append(msg.Message).Append(" ["); - bool first = true; foreach (var kv in ctx) { @@ -270,59 +219,10 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider sb.Append(kv.Key).Append('=').Append(kv.Value); first = false; } - sb.Append(']').AppendLine(); return sb.ToString(); } - private static void FlushBuffer(FileState state) - { - if (state.BufferPosition == 0 || state.Stream == null) - { - return; - } - - state.Stream.Write(state.Buffer, 0, state.BufferPosition); - state.Stream.Flush(); - state.BufferPosition = 0; - } - - private static long GetFileSize(string path) - => File.Exists(path) ? new FileInfo(path).Length : 0; - - private void DeleteOldLogFiles() - { - if (_maxRetainedFiles <= 0) - { - return; - } - - var files = new DirectoryInfo(_path) - .GetFiles($"{_fileNamePrefix}*") - .OrderByDescending(f => f.LastWriteTimeUtc) - .Skip(_maxRetainedFiles); - - foreach (var f in files) - { - try { f.Delete(); } catch { } - } - } - - private string GetFullName(DateTime date, string category) - { - var datePart = date.ToString("yyyyMMdd"); - var machine = Environment.MachineName; - - if (!EnableCategoryRouting || string.IsNullOrWhiteSpace(category)) - { - return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{datePart}.log"); - } - - var safeCategory = SanitizeCategory(category); - - return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{safeCategory}_{datePart}.log"); - } - private static string SanitizeCategory(string category) { foreach (var c in Path.GetInvalidFileNameChars()) @@ -333,21 +233,69 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider return category.Replace('.', '_'); } - protected override void OnShutdownFlush() + private void FlushAndDispose(FileState state) { - foreach (var state in _files.Values) + try + { + if (state.BufferPosition > 0) + { + state.Stream.Write(state.Buffer, 0, state.BufferPosition); + state.Stream.Flush(); + } + state.Stream.Dispose(); + ArrayPool.Shared.Return(state.Buffer); + } + catch { } + } + + private async Task BackgroundCleanupAsync() + { + while (!_cts.Token.IsCancellationRequested) { try { - FlushBuffer(state); - state.Stream?.Dispose(); - } - catch - { - // Do nothing during shutdown flush + foreach (var file in new DirectoryInfo(_path).GetFiles($"{_fileNamePrefix}*")) + { + var files = new DirectoryInfo(_path).GetFiles($"{_fileNamePrefix}*"); + Array.Sort(files, (a, b) => b.LastWriteTimeUtc.CompareTo(a.LastWriteTimeUtc)); + + for (int i = _maxRetainedFiles; i < files.Length; i++) + { + try { files[i].Delete(); } catch { } + } + } } + catch { } + + await Task.Delay(TimeSpan.FromSeconds(30), _cts.Token); // cleanup every 30s + } + } + + private sealed class Logger : ILogger + { + private readonly Channel _channel; + private readonly string _category; + + public Logger(Channel channel, string category) + { + _channel = channel; + _category = category; } - _files.Clear(); + public IDisposable BeginScope(TState state) => null; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, + Exception exception, Func formatter) + { + var msg = formatter(state, exception); + _channel.Writer.TryWrite(new LogMessage { Message = msg, Category = _category }); + } + } + + private sealed class LogMessage + { + public string Message; + public string Category; } }