diff --git a/EonaCat.Logger/EonaCat.Logger.csproj b/EonaCat.Logger/EonaCat.Logger.csproj index dc399b2..efda1e7 100644 --- a/EonaCat.Logger/EonaCat.Logger.csproj +++ b/EonaCat.Logger/EonaCat.Logger.csproj @@ -13,8 +13,8 @@ EonaCat (Jeroen Saey) EonaCat;Logger;EonaCatLogger;Log;Writer;Jeroen;Saey - 1.6.6 - 1.6.6 + 1.6.7 + 1.6.7 README.md True LICENSE @@ -25,7 +25,7 @@ - 1.6.6+{chash:10}.{c:ymd} + 1.6.7+{chash:10}.{c:ymd} true true v[0-9]* diff --git a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs index 0fdba8c..b3bd064 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/FileLoggerProvider.cs @@ -22,54 +22,32 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider private readonly int _maxRolloverFiles; private readonly LoggerScopedContext _context = new LoggerScopedContext(); - private readonly Dictionary _files = new Dictionary(); - private static readonly ConcurrentDictionary _fileLocks = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _files = new ConcurrentDictionary(); - // High-performance buffer pool - private static readonly ConcurrentBag _bufferPool = new ConcurrentBag(); - private const int BufferSize = 256 * 1024; // 256KB buffers - private const int MaxPoolSize = 100; - - // Pre-allocated StringBuilder pool for low memory - private static readonly ConcurrentBag _stringBuilderPool = new ConcurrentBag(); - private const int MaxStringBuilderPoolSize = 50; + private const int BufferSize = 256 * 1024; + private static readonly Encoding Utf8 = new UTF8Encoding(false); public bool IncludeCorrelationId { get; } public bool EnableCategoryRouting { get; } + public string LogFile => _files.TryGetValue(string.Empty, out var state) ? state.FilePath : null; + public event EventHandler OnError; public event EventHandler OnRollOver; - public string LogFile - { - get - { - FileState state; - return _files.TryGetValue(string.Empty, out state) ? state.FilePath : string.Empty; - } - } - private sealed class FileState { public string FilePath; public long Size; public DateTime Date; - public readonly SemaphoreSlim WriteLock = new SemaphoreSlim(1, 1); - - // Buffered writing for high throughput - public byte[] Buffer; + public byte[] Buffer = new byte[BufferSize]; public int BufferPosition; public FileStream Stream; } - public FileLoggerProvider(IOptions options) - : base(options) + public FileLoggerProvider(IOptions options) : base(options) { - var o = options.Value; - if (o == null) - { - throw new ArgumentNullException("options"); - } + var o = options.Value ?? throw new ArgumentNullException(nameof(options)); _path = o.LogDirectory; _fileNamePrefix = o.FileNamePrefix; @@ -81,207 +59,139 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider Directory.CreateDirectory(_path); - // Initialize - var defaultState = CreateFileState(DateTime.UtcNow.Date, options.Value.Category); + var defaultState = CreateFileState(DateTime.UtcNow.Date, o.Category); _files[string.Empty] = defaultState; } - internal override async Task WriteMessagesAsync(IReadOnlyList messages, CancellationToken token) + internal override Task WriteMessagesAsync(IReadOnlyList messages, CancellationToken token) { - // Group messages by category for batch processing - if (EnableCategoryRouting) - { - var grouped = messages.GroupBy(m => SanitizeCategory(m.Category)); - foreach (var group in grouped) - { - var categoryKey = group.Key; - - FileState state; - if (!_files.TryGetValue(categoryKey, out state)) - { - var date = DateTime.UtcNow.Date; - state = CreateFileState(date, categoryKey); - _files[categoryKey] = state; - } - - await WriteBatchAsync(state, group, categoryKey, token); - } - } - else - { - var categoryKey = string.Empty; - - FileState state; - if (!_files.TryGetValue(categoryKey, out state)) - { - var date = DateTime.UtcNow.Date; - state = CreateFileState(date, categoryKey); - _files[categoryKey] = state; - } - - await WriteBatchAsync(state, messages, categoryKey, token); - } - - DeleteOldLogFiles(); - } - - private async Task WriteBatchAsync(FileState state, IEnumerable messages, string categoryKey, CancellationToken token) - { - await state.WriteLock.WaitAsync(token); try { - foreach (var msg in messages) + if (EnableCategoryRouting) { - if (token.IsCancellationRequested) - break; + var grouped = messages.GroupBy(m => SanitizeCategory(m.Category)); - var date = msg.Timestamp.UtcDateTime.Date; - - // Rotate file by date - if (state.Date != date) + foreach (var group in grouped) { - await FlushBufferAsync(state); - RotateByDate(state, date, categoryKey); - } + var categoryKey = group.Key; - WriteMessageToBuffer(state, msg); + var state = _files.GetOrAdd(categoryKey, + _ => CreateFileState(DateTime.UtcNow.Date, categoryKey)); - // Flush buffer if nearly full or file size limit reached - if (state.BufferPosition >= BufferSize - 1024 || state.Size >= _maxFileSize) - { - await FlushBufferAsync(state); - - if (state.Size >= _maxFileSize) - { - RollOver(state, categoryKey); - } + WriteBatch(state, group, categoryKey); } } + else + { + var state = _files.GetOrAdd(string.Empty, + _ => CreateFileState(DateTime.UtcNow.Date, string.Empty)); - // Final flush for this batch - await FlushBufferAsync(state); + WriteBatch(state, messages, string.Empty); + } + + DeleteOldLogFiles(); } catch (Exception ex) { - if (OnError != null) + OnError?.Invoke(this, new ErrorMessage { Exception = ex, Message = ex.Message }); + } + + return Task.CompletedTask; + } + + private void WriteBatch(FileState state, IEnumerable messages, string categoryKey) + { + if (!File.Exists(state.FilePath)) + { + RecreateFile(state, categoryKey); + } + + foreach (var msg in messages) + { + var date = msg.Timestamp.UtcDateTime.Date; + + if (state.Date != date) { - OnError.Invoke(this, new ErrorMessage { Exception = ex, Message = ex.Message }); + FlushBuffer(state); + RotateByDate(state, date, categoryKey); + } + + WriteMessageToBuffer(state, msg); + + if (state.BufferPosition >= BufferSize - 1024 || state.Size >= _maxFileSize) + { + FlushBuffer(state); + + if (state.Size >= _maxFileSize) + { + RollOver(state, categoryKey); + } } } - finally - { - state.WriteLock.Release(); - } + + FlushBuffer(state); } private FileState CreateFileState(DateTime date, string category) { var path = GetFullName(date, category); - var buffer = GetBuffer(); - var stream = new FileStream( - path, - FileMode.Append, - FileAccess.Write, - FileShare.ReadWrite, - BufferSize, - FileOptions.Asynchronous | FileOptions.SequentialScan); - - var state = new FileState + return new FileState { - Stream = stream, FilePath = path, - Size = GetFileSize(path), Date = date, - Buffer = buffer, - BufferPosition = 0 + Size = GetFileSize(path), + Stream = OpenFileWithRetry(path) }; - - return state; } - private static byte[] GetBuffer() + private static FileStream OpenFileWithRetry(string path) { - byte[] buffer; - if (!_bufferPool.TryTake(out buffer)) + const int retries = 3; + + for (int i = 0; i < retries; i++) { - buffer = new byte[BufferSize]; + try + { + return new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.SequentialScan); + } + catch + { + Thread.Sleep(5); + } } - return buffer; + + throw new IOException("Unable to open log file."); } - private static void ReturnBuffer(byte[] buffer) + private void RecreateFile(FileState state, string category) { - if (_bufferPool.Count < MaxPoolSize) - { - Array.Clear(buffer, 0, buffer.Length); - _bufferPool.Add(buffer); - } - } + FlushBuffer(state); + state.Stream?.Dispose(); - private static StringBuilder GetStringBuilder() - { - StringBuilder sb; - if (!_stringBuilderPool.TryTake(out sb)) - { - sb = new StringBuilder(512); - } - else - { - sb.Clear(); - } - return sb; - } + state.FilePath = GetFullName(DateTime.UtcNow.Date, category); + state.Size = 0; + state.BufferPosition = 0; - private static void ReturnStringBuilder(StringBuilder sb) - { - if (_stringBuilderPool.Count < MaxStringBuilderPoolSize && sb.Capacity <= 2048) - { - sb.Clear(); - _stringBuilderPool.Add(sb); - } - } - - private static long GetFileSize(string path) - { - try - { - return File.Exists(path) ? new FileInfo(path).Length : 0; - } - catch - { - return 0; - } + state.Stream = OpenFileWithRetry(state.FilePath); } private void RotateByDate(FileState state, DateTime newDate, string category) { - if (state.Stream != null) - { - state.Stream.Dispose(); - } + state.Stream?.Dispose(); state.Date = newDate; state.FilePath = GetFullName(newDate, category); state.Size = GetFileSize(state.FilePath); state.BufferPosition = 0; - state.Stream = new FileStream( - state.FilePath, - FileMode.Append, - FileAccess.Write, - FileShare.ReadWrite, - BufferSize, - FileOptions.Asynchronous | FileOptions.SequentialScan); + state.Stream = OpenFileWithRetry(state.FilePath); } private void RollOver(FileState state, string category) { - if (state.Stream != null) - { - state.Stream.Dispose(); - state.Stream = null; - } + FlushBuffer(state); + state.Stream?.Dispose(); var dir = Path.GetDirectoryName(state.FilePath); var name = Path.GetFileNameWithoutExtension(state.FilePath); @@ -289,8 +199,8 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider for (int i = _maxRolloverFiles - 1; i >= 1; i--) { - var src = Path.Combine(dir, string.Format("{0}.{1}{2}", name, i, ext)); - var dst = Path.Combine(dir, string.Format("{0}.{1}{2}", name, i + 1, ext)); + var src = Path.Combine(dir, $"{name}.{i}{ext}"); + var dst = Path.Combine(dir, $"{name}.{i + 1}{ext}"); if (File.Exists(dst)) { @@ -303,90 +213,80 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider } } - var first = Path.Combine(dir, string.Format("{0}.1{1}", name, ext)); + var first = Path.Combine(dir, $"{name}.1{ext}"); if (File.Exists(state.FilePath)) { File.Move(state.FilePath, first); } - if (OnRollOver != null) - { - OnRollOver.Invoke(this, state.FilePath); - } + OnRollOver?.Invoke(this, state.FilePath); state.Size = 0; state.BufferPosition = 0; - - state.Stream = new FileStream( - state.FilePath, - FileMode.Append, - FileAccess.Write, - FileShare.ReadWrite, - BufferSize, - FileOptions.Asynchronous | FileOptions.SequentialScan); + state.Stream = OpenFileWithRetry(state.FilePath); } private void WriteMessageToBuffer(FileState state, LogMessage msg) { - var sb = GetStringBuilder(); + var text = BuildMessage(msg); + var byteCount = Utf8.GetByteCount(text); - try + if (state.BufferPosition + byteCount > BufferSize) { - sb.Append(msg.Message); - - if (IncludeCorrelationId) - { - var ctx = _context.GetAll(); - if (ctx.Count > 0) - { - sb.Append(" ["); - bool first = true; - foreach (var kv in ctx) - { - if (!first) - { - sb.Append(' '); - } - - sb.Append(kv.Key).Append('=').Append(kv.Value); - first = false; - } - sb.Append(']'); - } - } - - sb.AppendLine(); - - var text = sb.ToString(); - var byteCount = Encoding.UTF8.GetByteCount(text); - - // If message won't fit in buffer, flush first - if (state.BufferPosition + byteCount > BufferSize) - { - FlushBufferAsync(state).Wait(); - } - - // Write to buffer - var written = Encoding.UTF8.GetBytes(text, 0, text.Length, state.Buffer, state.BufferPosition); - state.BufferPosition += written; - state.Size += written; - } - finally - { - ReturnStringBuilder(sb); + FlushBuffer(state); } + + var written = Utf8.GetBytes(text, 0, text.Length, state.Buffer, state.BufferPosition); + state.BufferPosition += written; + state.Size += written; } - private async Task FlushBufferAsync(FileState state) + private string BuildMessage(LogMessage msg) + { + if (!IncludeCorrelationId) + { + return msg.Message + Environment.NewLine; + } + + var ctx = _context.GetAll(); + if (ctx.Count == 0) + { + return msg.Message + Environment.NewLine; + } + + var sb = new StringBuilder(256); + sb.Append(msg.Message).Append(" ["); + + bool first = true; + foreach (var kv in ctx) + { + if (!first) + { + sb.Append(' '); + } + + sb.Append(kv.Key).Append('=').Append(kv.Value); + first = false; + } + + sb.Append(']').AppendLine(); + return sb.ToString(); + } + + private static void FlushBuffer(FileState state) { if (state.BufferPosition == 0 || state.Stream == null) + { return; + } - await state.Stream.WriteAsync(state.Buffer, 0, state.BufferPosition); - await state.Stream.FlushAsync(); + state.Stream.Write(state.Buffer, 0, state.BufferPosition); state.BufferPosition = 0; } + private static long GetFileSize(string path) + => File.Exists(path) ? new FileInfo(path).Length : 0; + private void DeleteOldLogFiles() { if (_maxRetainedFiles <= 0) @@ -395,18 +295,8 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider } var files = new DirectoryInfo(_path) - .GetFiles(string.Format("{0}*", _fileNamePrefix)) - .OrderByDescending(f => - { - var name = Path.GetFileNameWithoutExtension(f.Name); - var parts = name.Split('_'); - var datePart = parts.LastOrDefault(); - DateTime dt; - return DateTime.TryParseExact(datePart, "yyyyMMdd", null, - System.Globalization.DateTimeStyles.None, out dt) - ? dt - : DateTime.MinValue; - }) + .GetFiles($"{_fileNamePrefix}*") + .OrderByDescending(f => f.LastWriteTimeUtc) .Skip(_maxRetainedFiles); foreach (var f in files) @@ -422,16 +312,12 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider if (!EnableCategoryRouting || string.IsNullOrWhiteSpace(category)) { - return string.IsNullOrWhiteSpace(_fileNamePrefix) - ? Path.Combine(_path, string.Format("{0}_{1}.log", machine, datePart)) - : Path.Combine(_path, string.Format("{0}_{1}_{2}.log", _fileNamePrefix, machine, datePart)); + return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{datePart}.log"); } var safeCategory = SanitizeCategory(category); - return string.IsNullOrWhiteSpace(_fileNamePrefix) - ? Path.Combine(_path, string.Format("{0}_{1}_{2}.log", machine, safeCategory, datePart)) - : Path.Combine(_path, string.Format("{0}_{1}_{2}_{3}.log", _fileNamePrefix, machine, safeCategory, datePart)); + return Path.Combine(_path, $"{_fileNamePrefix}_{machine}_{safeCategory}_{datePart}.log"); } private static string SanitizeCategory(string category) @@ -446,29 +332,19 @@ public sealed class FileLoggerProvider : BatchingLoggerProvider protected override void OnShutdownFlush() { - foreach (var kvp in _files) + foreach (var state in _files.Values) { - var state = kvp.Value; - if (state != null) + try { - try - { - FlushBufferAsync(state).Wait(); - if (state.Stream != null) - { - state.Stream.Dispose(); - } - if (state.Buffer != null) - { - ReturnBuffer(state.Buffer); - } - } - catch - { - // Ignore errors during shutdown - } + FlushBuffer(state); + state.Stream?.Dispose(); + } + catch + { + // Do nothing during shutdown flush } } + _files.Clear(); } -} \ No newline at end of file +} diff --git a/EonaCat.Logger/EonaCatCoreLogger/XmlFileLogger.cs b/EonaCat.Logger/EonaCatCoreLogger/XmlFileLogger.cs index 77e762a..2be2626 100644 --- a/EonaCat.Logger/EonaCatCoreLogger/XmlFileLogger.cs +++ b/EonaCat.Logger/EonaCatCoreLogger/XmlFileLogger.cs @@ -93,7 +93,7 @@ namespace EonaCat.Logger.EonaCatCoreLogger try { - using (var fs = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.Read, 4096, true)) + using (var fs = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 4096, true)) using (var sw = new StreamWriter(fs, Encoding.UTF8)) { await sw.WriteAsync(logString).ConfigureAwait(false); diff --git a/Testers/EonaCat.Logger.Test.Web/Logger.cs b/Testers/EonaCat.Logger.Test.Web/Logger.cs index 851ec7f..804babc 100644 --- a/Testers/EonaCat.Logger.Test.Web/Logger.cs +++ b/Testers/EonaCat.Logger.Test.Web/Logger.cs @@ -58,19 +58,19 @@ public class Logger { var logFileName = logName + ".log"; - await using var fS = new FileStream(Path.Combine(ConvertToAbsolutePath(LogFolder), logFileName), FileMode.Open, - FileAccess.Read, FileShare.ReadWrite, 64 * 1024, true); + await using var fileStream = new FileStream(Path.Combine(ConvertToAbsolutePath(LogFolder), logFileName), FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 64 * 1024, true); + var response = context.Response; response.ContentType = "text/plain"; response.Headers.ContentDisposition = "attachment;filename=" + logFileName; - if (limit > fS.Length || limit < 1) + if (limit > fileStream.Length || limit < 1) { - limit = fS.Length; + limit = fileStream.Length; } - var oFS = new OffsetStream(fS, 0, limit); + var oFS = new OffsetStream(fileStream, 0, limit); var request = context.Request; Stream stream; @@ -109,7 +109,7 @@ public class Logger { await oFS.CopyToAsync(stream).ConfigureAwait(false); - if (fS.Length > limit) + if (fileStream.Length > limit) { await stream.WriteAsync("\r\n####___TRUNCATED___####"u8.ToArray()).ConfigureAwait(false); }