diff --git a/Journal.Core/Dtos/CommandDtos.cs b/Journal.Core/Dtos/CommandDtos.cs new file mode 100644 index 0000000..0ed5dba --- /dev/null +++ b/Journal.Core/Dtos/CommandDtos.cs @@ -0,0 +1,35 @@ +namespace Journal.Core.Dtos; + +internal sealed record VaultInitializePayload(string Password, string VaultDirectory); +internal sealed record VaultPayload(string Password, string VaultDirectory, string DataDirectory, string? NowUtc = null); +internal sealed record ClearDataPayload(string DataDirectory); +internal sealed record EntryListPayload(string? DataDirectory = null); +internal sealed record EntryLoadPayload(string FilePath); +public sealed record EntrySavePayload(string Content, string? FilePath = null, string? Mode = null); +public sealed record EntryListItem(string FileName, string FilePath); +public sealed record EntryLoadResult(string Date, string FileName, string FilePath, string RawContent); +public sealed record EntrySaveResult(string FilePath); +internal sealed record DatabasePayload(string Password, string? DataDirectory = null); +internal sealed record AiSummarizeEntryPayload(string Content, string? FileStem = null); +internal sealed record AiSummarizeAllPayload(List? Entries); +internal sealed record AiChatPayload(string Prompt); +internal sealed record AiEmbedPayload(string Content); +internal sealed record SpeechTranscribePayload( + string? AudioBase64 = null, + string? Audio_Base64 = null, + string? Engine = null, + string? WhisperModel = null, + string? Whisper_Model = null, + string? Text = null, + int? SimulateDelayMs = null, + int? Simulate_Delay_Ms = null); +internal sealed record SearchEntriesPayload( + string DataDirectory, + string? Query = null, + string? Section = null, + string? StartDate = null, + string? EndDate = null, + List? Tags = null, + List? Types = null, + List? Checked = null, + List? Unchecked = null); diff --git a/Journal.Core/Dtos/DatabaseDtos.cs b/Journal.Core/Dtos/DatabaseDtos.cs new file mode 100644 index 0000000..3c587e7 --- /dev/null +++ b/Journal.Core/Dtos/DatabaseDtos.cs @@ -0,0 +1,18 @@ +namespace Journal.Core.Dtos; + +public sealed record JournalDatabaseStatus( + string DatabasePath, + int KeyLengthBytes, + int Iterations, + string KeyDerivation, + IReadOnlyList SchemaTables, + string SchemaBootstrapPath, + bool RuntimeReady, + string RuntimeMessage); + +public sealed record JournalDatabaseHydrationResult( + string DatabasePath, + string SchemaBootstrapPath, + int EntryFilesProcessed, + bool RuntimeReady, + string Message); diff --git a/Journal.Core/Entry.cs b/Journal.Core/Entry.cs index 39627f5..7a9db3a 100644 --- a/Journal.Core/Entry.cs +++ b/Journal.Core/Entry.cs @@ -1,7 +1,5 @@ using System.ComponentModel.DataAnnotations; using System.Globalization; -using System.Net; -using System.Text.RegularExpressions; using System.Text.Json; using Journal.Core.Dtos; using Journal.Core.Models; @@ -18,6 +16,8 @@ public class Entry private readonly IJournalConfigService _config; private readonly IAiService _ai; private readonly ISpeechBridgeService _speech; + private readonly IEntryFileService _entryFiles; + private readonly CommandLogger _logger; private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true @@ -30,7 +30,9 @@ public class Entry IJournalDatabaseService database, IJournalConfigService config, IAiService ai, - ISpeechBridgeService speech) + ISpeechBridgeService speech, + IEntryFileService entryFiles, + CommandLogger logger) { _fragments = fragments; _entrySearch = entrySearch; @@ -39,6 +41,8 @@ public class Entry _config = config; _ai = ai; _speech = speech; + _entryFiles = entryFiles; + _logger = logger; } public async Task RunAsync() @@ -73,7 +77,7 @@ public class Entry var correlationId = string.IsNullOrWhiteSpace(cmd.CorrelationId) ? Guid.NewGuid().ToString("N") : cmd.CorrelationId.Trim(); - LogStart(action, correlationId, cmd.Payload); + _logger.LogStart(action, correlationId, cmd.Payload); object? result; try @@ -131,19 +135,19 @@ public class Entry var listDataDirectory = !string.IsNullOrWhiteSpace(listPayload?.DataDirectory) ? listPayload.DataDirectory : _config.Current.DataDirectory; - result = ListEntries(listDataDirectory); + result = _entryFiles.ListEntries(listDataDirectory); break; case "entries.load": var loadEntryPayload = DeserializePayload(cmd.Payload); if (loadEntryPayload is null || string.IsNullOrWhiteSpace(loadEntryPayload.FilePath)) return Error("Missing or invalid payload"); - result = LoadEntry(loadEntryPayload.FilePath); + result = _entryFiles.LoadEntry(loadEntryPayload.FilePath); break; case "entries.save": var saveEntryPayload = DeserializePayload(cmd.Payload); if (saveEntryPayload is null || string.IsNullOrWhiteSpace(saveEntryPayload.Content)) return Error("Missing or invalid payload"); - result = SaveEntry(saveEntryPayload, _config.Current.DataDirectory); + result = _entryFiles.SaveEntry(saveEntryPayload, _config.Current.DataDirectory); break; case "config.get": result = _config.Current; @@ -256,122 +260,53 @@ public class Entry result = _database.HydrateWorkspace(dbHydratePayload.Password, dbHydratePayload.DataDirectory); break; default: - LogFailure(action, correlationId, "unknown_action"); + _logger.LogFailure(action, correlationId, "unknown_action"); return Error($"Unknown action: {action}"); } } catch (JsonException) { - LogFailure(action, correlationId, "invalid_payload_json"); + _logger.LogFailure(action, correlationId, "invalid_payload_json"); return Error("Missing or invalid payload"); } catch (ValidationException ex) { - LogFailure(action, correlationId, "validation", ex.Message); + _logger.LogFailure(action, correlationId, "validation", ex.Message); return Error(ex.Message); } catch (ArgumentException ex) { - LogFailure(action, correlationId, "argument", ex.Message); + _logger.LogFailure(action, correlationId, "argument", ex.Message); return Error(ex.Message); } catch (TimeoutException ex) { - LogFailure(action, correlationId, "timeout", ex.Message); + _logger.LogFailure(action, correlationId, "timeout", ex.Message); return Error(ex.Message); } catch (InvalidOperationException ex) { - LogFailure(action, correlationId, "invalid_operation", ex.Message); + _logger.LogFailure(action, correlationId, "invalid_operation", ex.Message); return Error(ex.Message); } catch (FileNotFoundException ex) { - LogFailure(action, correlationId, "not_found", ex.Message); + _logger.LogFailure(action, correlationId, "not_found", ex.Message); return Error(ex.Message); } catch { - LogFailure(action, correlationId, "internal_error"); + _logger.LogFailure(action, correlationId, "internal_error"); return Error("Internal error"); } - LogSuccess(action, correlationId); + _logger.LogSuccess(action, correlationId); return JsonSerializer.Serialize(new { ok = true, data = result }); } private static string Error(string message) => JsonSerializer.Serialize(new { ok = false, error = message }); - private void LogStart(string action, string correlationId, JsonElement? payload) - { - var redactedPayload = LogRedactor.RedactPayload(payload); - EmitLog("information", action, correlationId, "start", redactedPayload); - } - - private void LogSuccess(string action, string correlationId) - { - EmitLog("information", action, correlationId, "success"); - } - - private void LogFailure(string action, string correlationId, string errorType, string? message = null) - { - var details = string.IsNullOrWhiteSpace(message) - ? "" - : (message!.Length <= 160 ? message : $"{message[..160]}...(truncated)"); - EmitLog("warning", action, correlationId, "failure", errorType: errorType, details: details); - } - - private static void EmitLog( - string level, - string action, - string correlationId, - string outcome, - object? payload = null, - string? errorType = null, - string? details = null) - { - if (!ShouldLog(level)) - return; - - var envelope = new - { - timestamp = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture), - level, - component = nameof(Entry), - action, - correlation_id = correlationId, - outcome, - error_type = errorType, - details, - payload - }; - Console.Error.WriteLine(JsonSerializer.Serialize(envelope)); - } - - private static bool ShouldLog(string level) - { - var configured = (Environment.GetEnvironmentVariable("JOURNAL_LOG_LEVEL") ?? "warning") - .Trim() - .ToLowerInvariant(); - var configuredRank = LogLevelRank(configured); - var incomingRank = LogLevelRank(level); - return incomingRank >= configuredRank; - } - - private static int LogLevelRank(string level) => level switch - { - "trace" => 0, - "debug" => 1, - "information" => 2, - "info" => 2, - "warning" => 3, - "warn" => 3, - "error" => 4, - "critical" => 5, - _ => 3 - }; - private static T? DeserializePayload(JsonElement? payload) { if (payload is null) @@ -395,153 +330,4 @@ public class Entry throw new ArgumentException("Invalid nowUtc value. Expected ISO date/time."); } - - private static IReadOnlyList ListEntries(string dataDirectory) - { - if (!Directory.Exists(dataDirectory)) - return []; - - return Directory.GetFiles(dataDirectory, "*.md") - .OrderBy(Path.GetFileName, StringComparer.Ordinal) - .Select(path => new EntryListItem( - FileName: Path.GetFileName(path), - FilePath: Path.GetFullPath(path))) - .ToArray(); - } - - private static EntryLoadResult LoadEntry(string filePath) - { - var normalizedPath = Path.GetFullPath(filePath); - if (!File.Exists(normalizedPath)) - throw new FileNotFoundException($"Entry file not found: {normalizedPath}"); - - var rawContent = StripRichHtml(File.ReadAllText(normalizedPath)); - var fileStem = Path.GetFileNameWithoutExtension(normalizedPath); - var entry = JournalParser.ParseJournalContent(rawContent, fileStem); - - return new EntryLoadResult( - Date: entry.Date, - FileName: Path.GetFileName(normalizedPath), - FilePath: normalizedPath, - RawContent: entry.RawContent); - } - - private static EntrySaveResult SaveEntry(EntrySavePayload payload, string defaultDataDirectory) - { - var targetPath = ResolveTargetPath(payload.FilePath, defaultDataDirectory); - var mode = string.IsNullOrWhiteSpace(payload.Mode) ? "Daily" : payload.Mode.Trim(); - var sanitizedContent = StripRichHtml(payload.Content ?? ""); - Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); - - if (string.Equals(mode, "Overwrite", StringComparison.OrdinalIgnoreCase)) - { - File.WriteAllText(targetPath, sanitizedContent); - return new EntrySaveResult(targetPath); - } - - if (string.Equals(mode, "Fragment", StringComparison.OrdinalIgnoreCase)) - { - File.AppendAllText(targetPath, "\n\n" + sanitizedContent.Trim()); - return new EntrySaveResult(targetPath); - } - - string finalContent; - if (File.Exists(targetPath)) - { - var existingContent = File.ReadAllText(targetPath); - var fileStem = Path.GetFileNameWithoutExtension(targetPath); - var existingEntry = JournalParser.ParseJournalContent(existingContent, fileStem); - var newEntryData = JournalParser.ParseJournalContent(sanitizedContent, fileStem); - existingEntry.MergeWith(newEntryData); - finalContent = existingEntry.ToMarkdown(); - } - else - { - finalContent = sanitizedContent; - } - - File.WriteAllText(targetPath, finalContent); - return new EntrySaveResult(targetPath); - } - - private static string ResolveTargetPath(string? filePath, string defaultDataDirectory) - { - if (!string.IsNullOrWhiteSpace(filePath)) - return Path.GetFullPath(filePath); - - return Path.GetFullPath(Path.Combine(defaultDataDirectory, $"{DateTime.Now:yyyy-MM-dd}.md")); - } - - private static bool LooksLikeRichHtml(string content) - { - var lowered = content.ToLowerInvariant(); - string[] markers = - [ - "", " lowered.Contains(marker, StringComparison.Ordinal))) - return true; - return Regex.Matches(lowered, "]*>").Count >= 8; - } - - private static string StripRichHtml(string content) - { - if (string.IsNullOrWhiteSpace(content)) - return content; - if (!LooksLikeRichHtml(content)) - return content; - - var text = content.Replace("\r\n", "\n").Replace("\r", "\n"); - text = Regex.Replace(text, "<(script|style)\\b[^>]*>.*?", "", RegexOptions.IgnoreCase | RegexOptions.Singleline); - text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "]*>", "\n- ", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "<(td|th)\\b[^>]*>", " | ", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "", " ", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "]*>", "\n---\n", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "<[^>]+>", "", RegexOptions.Singleline); - text = WebUtility.HtmlDecode(text) - .Replace('\u00a0', ' ') - .Replace("\u200b", "", StringComparison.Ordinal); - text = string.Join("\n", text.Split('\n').Select(line => line.TrimEnd())); - text = Regex.Replace(text, "[ \\t]{2,}", " "); - text = Regex.Replace(text, "\n{3,}", "\n\n").Trim(); - return string.IsNullOrEmpty(text) ? content : text; - } - - private sealed record VaultInitializePayload(string Password, string VaultDirectory); - private sealed record VaultPayload(string Password, string VaultDirectory, string DataDirectory, string? NowUtc = null); - private sealed record ClearDataPayload(string DataDirectory); - private sealed record EntryListPayload(string? DataDirectory = null); - private sealed record EntryLoadPayload(string FilePath); - private sealed record EntrySavePayload(string Content, string? FilePath = null, string? Mode = null); - private sealed record EntryListItem(string FileName, string FilePath); - private sealed record EntryLoadResult(string Date, string FileName, string FilePath, string RawContent); - private sealed record EntrySaveResult(string FilePath); - private sealed record DatabasePayload(string Password, string? DataDirectory = null); - private sealed record AiSummarizeEntryPayload(string Content, string? FileStem = null); - private sealed record AiSummarizeAllPayload(List? Entries); - private sealed record AiChatPayload(string Prompt); - private sealed record AiEmbedPayload(string Content); - private sealed record SpeechTranscribePayload( - string? AudioBase64 = null, - string? Audio_Base64 = null, - string? Engine = null, - string? WhisperModel = null, - string? Whisper_Model = null, - string? Text = null, - int? SimulateDelayMs = null, - int? Simulate_Delay_Ms = null); - private sealed record SearchEntriesPayload( - string DataDirectory, - string? Query = null, - string? Section = null, - string? StartDate = null, - string? EndDate = null, - List? Tags = null, - List? Types = null, - List? Checked = null, - List? Unchecked = null); } diff --git a/Journal.Core/Repositories/DiskEntryFileRepository.cs b/Journal.Core/Repositories/DiskEntryFileRepository.cs new file mode 100644 index 0000000..bd95574 --- /dev/null +++ b/Journal.Core/Repositories/DiskEntryFileRepository.cs @@ -0,0 +1,35 @@ +namespace Journal.Core.Repositories; + +public sealed class DiskEntryFileRepository : IEntryFileRepository +{ + public IReadOnlyList ListMarkdownFiles(string dataDirectory) + { + if (!Directory.Exists(dataDirectory)) + return []; + + return Directory.GetFiles(dataDirectory, "*.md") + .OrderBy(Path.GetFileName, StringComparer.Ordinal) + .ToArray(); + } + + public string ReadFile(string filePath) => File.ReadAllText(filePath); + + public void WriteFile(string filePath, string content) => File.WriteAllText(filePath, content); + + public void AppendFile(string filePath, string content) => File.AppendAllText(filePath, content); + + public bool FileExists(string filePath) => File.Exists(filePath); + + public string GetFullPath(string filePath) => Path.GetFullPath(filePath); + + public string GetFileName(string filePath) => Path.GetFileName(filePath); + + public string GetFileNameWithoutExtension(string filePath) => Path.GetFileNameWithoutExtension(filePath); + + public void EnsureDirectory(string path) + { + var dir = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(dir)) + Directory.CreateDirectory(dir); + } +} diff --git a/Journal.Core/Repositories/IEntryFileRepository.cs b/Journal.Core/Repositories/IEntryFileRepository.cs new file mode 100644 index 0000000..8242bec --- /dev/null +++ b/Journal.Core/Repositories/IEntryFileRepository.cs @@ -0,0 +1,14 @@ +namespace Journal.Core.Repositories; + +public interface IEntryFileRepository +{ + IReadOnlyList ListMarkdownFiles(string dataDirectory); + string ReadFile(string filePath); + void WriteFile(string filePath, string content); + void AppendFile(string filePath, string content); + bool FileExists(string filePath); + string GetFullPath(string filePath); + string GetFileName(string filePath); + string GetFileNameWithoutExtension(string filePath); + void EnsureDirectory(string path); +} diff --git a/Journal.Core/ServiceCollectionExtensions.cs b/Journal.Core/ServiceCollectionExtensions.cs index 3bb6fc4..9b3c648 100644 --- a/Journal.Core/ServiceCollectionExtensions.cs +++ b/Journal.Core/ServiceCollectionExtensions.cs @@ -47,6 +47,9 @@ public static class ServiceCollectionExtensions message: $"Python speech sidecar unavailable: {ex.Message}"); } }); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); return services; } diff --git a/Journal.Core/Services/CommandLogger.cs b/Journal.Core/Services/CommandLogger.cs new file mode 100644 index 0000000..74bb089 --- /dev/null +++ b/Journal.Core/Services/CommandLogger.cs @@ -0,0 +1,76 @@ +using System.Globalization; +using System.Text.Json; + +namespace Journal.Core.Services; + +public sealed class CommandLogger +{ + public void LogStart(string action, string correlationId, JsonElement? payload) + { + var redactedPayload = LogRedactor.RedactPayload(payload); + EmitLog("information", action, correlationId, "start", redactedPayload); + } + + public void LogSuccess(string action, string correlationId) + { + EmitLog("information", action, correlationId, "success"); + } + + public void LogFailure(string action, string correlationId, string errorType, string? message = null) + { + var details = string.IsNullOrWhiteSpace(message) + ? "" + : (message!.Length <= 160 ? message : $"{message[..160]}...(truncated)"); + EmitLog("warning", action, correlationId, "failure", errorType: errorType, details: details); + } + + private static void EmitLog( + string level, + string action, + string correlationId, + string outcome, + object? payload = null, + string? errorType = null, + string? details = null) + { + if (!ShouldLog(level)) + return; + + var envelope = new + { + timestamp = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture), + level, + component = "Entry", + action, + correlation_id = correlationId, + outcome, + error_type = errorType, + details, + payload + }; + Console.Error.WriteLine(JsonSerializer.Serialize(envelope)); + } + + private static bool ShouldLog(string level) + { + var configured = (Environment.GetEnvironmentVariable("JOURNAL_LOG_LEVEL") ?? "warning") + .Trim() + .ToLowerInvariant(); + var configuredRank = LogLevelRank(configured); + var incomingRank = LogLevelRank(level); + return incomingRank >= configuredRank; + } + + private static int LogLevelRank(string level) => level switch + { + "trace" => 0, + "debug" => 1, + "information" => 2, + "info" => 2, + "warning" => 3, + "warn" => 3, + "error" => 4, + "critical" => 5, + _ => 3 + }; +} diff --git a/Journal.Core/Services/EntryFileService.cs b/Journal.Core/Services/EntryFileService.cs new file mode 100644 index 0000000..b5d1ee5 --- /dev/null +++ b/Journal.Core/Services/EntryFileService.cs @@ -0,0 +1,84 @@ +using Journal.Core.Dtos; +using Journal.Core.Repositories; + +namespace Journal.Core.Services; + +public sealed class EntryFileService : IEntryFileService +{ + private readonly IEntryFileRepository _repo; + + public EntryFileService(IEntryFileRepository repo) => + _repo = repo ?? throw new ArgumentNullException(nameof(repo)); + + public IReadOnlyList ListEntries(string dataDirectory) + { + return _repo.ListMarkdownFiles(dataDirectory) + .Select(path => new EntryListItem( + FileName: _repo.GetFileName(path), + FilePath: _repo.GetFullPath(path))) + .ToArray(); + } + + public EntryLoadResult LoadEntry(string filePath) + { + var normalizedPath = _repo.GetFullPath(filePath); + if (!_repo.FileExists(normalizedPath)) + throw new FileNotFoundException($"Entry file not found: {normalizedPath}"); + + var rawContent = HtmlSanitizer.StripRichHtml(_repo.ReadFile(normalizedPath)); + var fileStem = _repo.GetFileNameWithoutExtension(normalizedPath); + var entry = JournalParser.ParseJournalContent(rawContent, fileStem); + + return new EntryLoadResult( + Date: entry.Date, + FileName: _repo.GetFileName(normalizedPath), + FilePath: normalizedPath, + RawContent: entry.RawContent); + } + + public EntrySaveResult SaveEntry(EntrySavePayload payload, string defaultDataDirectory) + { + var targetPath = ResolveTargetPath(payload.FilePath, defaultDataDirectory); + var mode = string.IsNullOrWhiteSpace(payload.Mode) ? "Daily" : payload.Mode.Trim(); + var sanitizedContent = HtmlSanitizer.StripRichHtml(payload.Content ?? ""); + _repo.EnsureDirectory(targetPath); + + if (string.Equals(mode, "Overwrite", StringComparison.OrdinalIgnoreCase)) + { + _repo.WriteFile(targetPath, sanitizedContent); + return new EntrySaveResult(targetPath); + } + + if (string.Equals(mode, "Fragment", StringComparison.OrdinalIgnoreCase)) + { + _repo.AppendFile(targetPath, "\n\n" + sanitizedContent.Trim()); + return new EntrySaveResult(targetPath); + } + + string finalContent; + if (_repo.FileExists(targetPath)) + { + var existingContent = _repo.ReadFile(targetPath); + var fileStem = _repo.GetFileNameWithoutExtension(targetPath); + var existingEntry = JournalParser.ParseJournalContent(existingContent, fileStem); + var newEntryData = JournalParser.ParseJournalContent(sanitizedContent, fileStem); + existingEntry.MergeWith(newEntryData); + finalContent = existingEntry.ToMarkdown(); + } + else + { + finalContent = sanitizedContent; + } + + _repo.WriteFile(targetPath, finalContent); + return new EntrySaveResult(targetPath); + } + + private string ResolveTargetPath(string? filePath, string defaultDataDirectory) + { + if (!string.IsNullOrWhiteSpace(filePath)) + return _repo.GetFullPath(filePath); + + return _repo.GetFullPath(Path.Combine(defaultDataDirectory, $"{DateTime.Now:yyyy-MM-dd}.md")); + } +} diff --git a/Journal.Core/Services/HtmlSanitizer.cs b/Journal.Core/Services/HtmlSanitizer.cs new file mode 100644 index 0000000..9b7b895 --- /dev/null +++ b/Journal.Core/Services/HtmlSanitizer.cs @@ -0,0 +1,46 @@ +using System.Net; +using System.Text.RegularExpressions; + +namespace Journal.Core.Services; + +public static class HtmlSanitizer +{ + public static string StripRichHtml(string content) + { + if (string.IsNullOrWhiteSpace(content)) + return content; + if (!LooksLikeRichHtml(content)) + return content; + + var text = content.Replace("\r\n", "\n").Replace("\r", "\n"); + text = Regex.Replace(text, "<(script|style)\\b[^>]*>.*?", "", RegexOptions.IgnoreCase | RegexOptions.Singleline); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "]*>", "\n- ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "<(td|th)\\b[^>]*>", " | ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", " ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "]*>", "\n---\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "<[^>]+>", "", RegexOptions.Singleline); + text = WebUtility.HtmlDecode(text) + .Replace('\u00a0', ' ') + .Replace("\u200b", "", StringComparison.Ordinal); + text = string.Join("\n", text.Split('\n').Select(line => line.TrimEnd())); + text = Regex.Replace(text, "[ \\t]{2,}", " "); + text = Regex.Replace(text, "\n{3,}", "\n\n").Trim(); + return string.IsNullOrEmpty(text) ? content : text; + } + + public static bool LooksLikeRichHtml(string content) + { + var lowered = content.ToLowerInvariant(); + string[] markers = + [ + "", " lowered.Contains(marker, StringComparison.Ordinal))) + return true; + return Regex.Matches(lowered, "]*>").Count >= 8; + } +} diff --git a/Journal.Core/Services/IEntryFileService.cs b/Journal.Core/Services/IEntryFileService.cs new file mode 100644 index 0000000..1469869 --- /dev/null +++ b/Journal.Core/Services/IEntryFileService.cs @@ -0,0 +1,10 @@ +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public interface IEntryFileService +{ + IReadOnlyList ListEntries(string dataDirectory); + EntryLoadResult LoadEntry(string filePath); + EntrySaveResult SaveEntry(EntrySavePayload payload, string defaultDataDirectory); +} diff --git a/Journal.Core/Services/IJournalDatabaseService.cs b/Journal.Core/Services/IJournalDatabaseService.cs index 54b86bc..40af126 100644 --- a/Journal.Core/Services/IJournalDatabaseService.cs +++ b/Journal.Core/Services/IJournalDatabaseService.cs @@ -1,3 +1,5 @@ +using Journal.Core.Dtos; + namespace Journal.Core.Services; public interface IJournalDatabaseService @@ -10,20 +12,3 @@ public interface IJournalDatabaseService JournalDatabaseStatus GetStatus(string password, string? dataDirectory = null); JournalDatabaseHydrationResult HydrateWorkspace(string password, string? dataDirectory = null); } - -public sealed record JournalDatabaseStatus( - string DatabasePath, - int KeyLengthBytes, - int Iterations, - string KeyDerivation, - IReadOnlyList SchemaTables, - string SchemaBootstrapPath, - bool RuntimeReady, - string RuntimeMessage); - -public sealed record JournalDatabaseHydrationResult( - string DatabasePath, - string SchemaBootstrapPath, - int EntryFilesProcessed, - bool RuntimeReady, - string Message); diff --git a/Journal.Core/Services/JournalDatabaseService.cs b/Journal.Core/Services/JournalDatabaseService.cs index 73c0657..1da886a 100644 --- a/Journal.Core/Services/JournalDatabaseService.cs +++ b/Journal.Core/Services/JournalDatabaseService.cs @@ -1,5 +1,6 @@ using System.Security.Cryptography; using System.Text; +using Journal.Core.Dtos; using Microsoft.Data.Sqlite; namespace Journal.Core.Services; diff --git a/Journal.Core/Services/PythonSidecarAiService.cs b/Journal.Core/Services/PythonSidecarAiService.cs index 767b82a..f101ab3 100644 --- a/Journal.Core/Services/PythonSidecarAiService.cs +++ b/Journal.Core/Services/PythonSidecarAiService.cs @@ -1,5 +1,3 @@ -using System.Diagnostics; -using System.Text; using System.Text.Json; using Journal.Core.Dtos; using Journal.Core.Models; @@ -8,25 +6,20 @@ namespace Journal.Core.Services; public sealed class PythonSidecarAiService : IAiService { - private static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNameCaseInsensitive = true - }; - - private readonly JournalConfig _config; + private readonly PythonSidecarClient _client; public PythonSidecarAiService(JournalConfig config) { - _config = config; - if (string.IsNullOrWhiteSpace(_config.PythonAiSidecarPath)) + if (string.IsNullOrWhiteSpace(config.PythonAiSidecarPath)) throw new ArgumentException("Python AI sidecar path is required."); - if (!File.Exists(_config.PythonAiSidecarPath)) - throw new FileNotFoundException($"Python AI sidecar not found: {_config.PythonAiSidecarPath}"); + if (!File.Exists(config.PythonAiSidecarPath)) + throw new FileNotFoundException($"Python AI sidecar not found: {config.PythonAiSidecarPath}"); + _client = new PythonSidecarClient(config); } public async Task HealthAsync(CancellationToken cancellationToken = default) { - var data = await SendAsync("health", payload: new { }, cancellationToken); + var data = await _client.SendAsync("health", payload: new { }, cancellationToken); if (data is not JsonElement payload || payload.ValueKind != JsonValueKind.Object) return new AiHealthDto("python-sidecar", Enabled: true, Healthy: true, Message: "ok"); @@ -47,14 +40,14 @@ public sealed class PythonSidecarAiService : IAiService if (string.IsNullOrWhiteSpace(content)) throw new ArgumentException("Entry content is required.", nameof(content)); - var data = await SendAsync("summarize_entry", new { content, file_stem = fileStem }, cancellationToken); + var data = await _client.SendAsync("summarize_entry", new { content, file_stem = fileStem }, cancellationToken); return data?.GetString() ?? ""; } public async Task SummarizeAllAsync(IReadOnlyList entries, CancellationToken cancellationToken = default) { entries ??= []; - var data = await SendAsync("summarize_all", new { entries }, cancellationToken); + var data = await _client.SendAsync("summarize_all", new { entries }, cancellationToken); return data?.GetString() ?? ""; } @@ -63,7 +56,7 @@ public sealed class PythonSidecarAiService : IAiService if (string.IsNullOrWhiteSpace(prompt)) throw new ArgumentException("Prompt is required.", nameof(prompt)); - var data = await SendAsync("chat", new { prompt }, cancellationToken); + var data = await _client.SendAsync("chat", new { prompt }, cancellationToken); return data?.GetString() ?? ""; } @@ -72,7 +65,7 @@ public sealed class PythonSidecarAiService : IAiService if (string.IsNullOrWhiteSpace(content)) throw new ArgumentException("Content is required.", nameof(content)); - var data = await SendAsync("embed", new { content }, cancellationToken); + var data = await _client.SendAsync("embed", new { content }, cancellationToken); if (data is null || data.Value.ValueKind == JsonValueKind.Null) return []; @@ -89,102 +82,4 @@ public sealed class PythonSidecarAiService : IAiService return values; } - - private async Task SendAsync(string action, object payload, CancellationToken cancellationToken) - { - var request = JsonSerializer.Serialize(new { action, payload }, JsonOptions); - - using var process = new Process(); - process.StartInfo = new ProcessStartInfo - { - FileName = _config.PythonExecutable, - UseShellExecute = false, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WorkingDirectory = _config.ProjectRoot - }; - process.StartInfo.ArgumentList.Add(_config.PythonAiSidecarPath); - - if (!process.Start()) - throw new InvalidOperationException("Failed to start Python AI sidecar process."); - - await process.StandardInput.WriteLineAsync(request); - process.StandardInput.Close(); - - using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeoutCts.CancelAfter(_config.AiSidecarTimeoutMs); - - try - { - await process.WaitForExitAsync(timeoutCts.Token); - } - catch (OperationCanceledException) - { - TryKill(process); - throw new TimeoutException($"Python AI sidecar timed out after {_config.AiSidecarTimeoutMs} ms."); - } - - var stdout = await process.StandardOutput.ReadToEndAsync(); - var stderr = await process.StandardError.ReadToEndAsync(); - var line = LastJsonLine(stdout); - if (string.IsNullOrWhiteSpace(line)) - throw new InvalidOperationException($"Python AI sidecar returned no JSON response. stderr: {stderr}".Trim()); - - JsonDocument doc; - try - { - doc = JsonDocument.Parse(line); - } - catch (JsonException ex) - { - throw new InvalidOperationException($"Invalid JSON from Python AI sidecar: {line}", ex); - } - using (doc) - { - var root = doc.RootElement; - if (!root.TryGetProperty("ok", out var okNode) || okNode.ValueKind != JsonValueKind.True && okNode.ValueKind != JsonValueKind.False) - throw new InvalidOperationException("Python AI sidecar response missing boolean 'ok' field."); - - if (!okNode.GetBoolean()) - { - var error = root.TryGetProperty("error", out var errorNode) - ? errorNode.GetString() ?? "Unknown sidecar error." - : "Unknown sidecar error."; - throw new InvalidOperationException(error); - } - - if (!root.TryGetProperty("data", out var dataNode)) - return null; - - return dataNode.Clone(); - } - } - - private static string LastJsonLine(string text) - { - var lines = text.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); - for (var i = lines.Length - 1; i >= 0; i--) - { - var line = lines[i].Trim(); - if (line.StartsWith("{", StringComparison.Ordinal) && line.EndsWith("}", StringComparison.Ordinal)) - return line; - } - - return ""; - } - - private static void TryKill(Process process) - { - try - { - if (!process.HasExited) - process.Kill(entireProcessTree: true); - } - catch - { - // Ignore cleanup errors while handling timeout/failure path. - } - } } diff --git a/Journal.Core/Services/PythonSidecarClient.cs b/Journal.Core/Services/PythonSidecarClient.cs new file mode 100644 index 0000000..81439ca --- /dev/null +++ b/Journal.Core/Services/PythonSidecarClient.cs @@ -0,0 +1,119 @@ +using System.Diagnostics; +using System.Text.Json; +using Journal.Core.Models; + +namespace Journal.Core.Services; + +public sealed class PythonSidecarClient +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly JournalConfig _config; + + public PythonSidecarClient(JournalConfig config) + { + _config = config; + } + + public async Task SendAsync(string action, object payload, CancellationToken cancellationToken) + { + var request = JsonSerializer.Serialize(new { action, payload }, JsonOptions); + + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = _config.PythonExecutable, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WorkingDirectory = _config.ProjectRoot + }; + process.StartInfo.ArgumentList.Add(_config.PythonAiSidecarPath); + + if (!process.Start()) + throw new InvalidOperationException("Failed to start Python sidecar process."); + + await process.StandardInput.WriteLineAsync(request); + process.StandardInput.Close(); + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(_config.AiSidecarTimeoutMs); + + try + { + await process.WaitForExitAsync(timeoutCts.Token); + } + catch (OperationCanceledException) + { + TryKill(process); + throw new TimeoutException($"Python sidecar timed out after {_config.AiSidecarTimeoutMs} ms."); + } + + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + var line = LastJsonLine(stdout); + if (string.IsNullOrWhiteSpace(line)) + throw new InvalidOperationException($"Python sidecar returned no JSON response. stderr: {stderr}".Trim()); + + JsonDocument doc; + try + { + doc = JsonDocument.Parse(line); + } + catch (JsonException ex) + { + throw new InvalidOperationException($"Invalid JSON from Python sidecar: {line}", ex); + } + + using (doc) + { + var root = doc.RootElement; + if (!root.TryGetProperty("ok", out var okNode) || okNode.ValueKind != JsonValueKind.True && okNode.ValueKind != JsonValueKind.False) + throw new InvalidOperationException("Python sidecar response missing boolean 'ok' field."); + + if (!okNode.GetBoolean()) + { + var error = root.TryGetProperty("error", out var errorNode) + ? errorNode.GetString() ?? "Unknown sidecar error." + : "Unknown sidecar error."; + throw new InvalidOperationException(error); + } + + if (!root.TryGetProperty("data", out var dataNode)) + return null; + + return dataNode.Clone(); + } + } + + private static string LastJsonLine(string text) + { + var lines = text.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); + for (var i = lines.Length - 1; i >= 0; i--) + { + var line = lines[i].Trim(); + if (line.StartsWith("{", StringComparison.Ordinal) && line.EndsWith("}", StringComparison.Ordinal)) + return line; + } + + return ""; + } + + private static void TryKill(Process process) + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + // Ignore cleanup errors while handling timeout/failure path. + } + } +} diff --git a/Journal.Core/Services/PythonSidecarSpeechService.cs b/Journal.Core/Services/PythonSidecarSpeechService.cs index 582d1cf..82fa6b8 100644 --- a/Journal.Core/Services/PythonSidecarSpeechService.cs +++ b/Journal.Core/Services/PythonSidecarSpeechService.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using System.Text.Json; using Journal.Core.Dtos; using Journal.Core.Models; @@ -7,25 +6,20 @@ namespace Journal.Core.Services; public sealed class PythonSidecarSpeechService : ISpeechBridgeService { - private static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNameCaseInsensitive = true - }; - - private readonly JournalConfig _config; + private readonly PythonSidecarClient _client; public PythonSidecarSpeechService(JournalConfig config) { - _config = config; - if (string.IsNullOrWhiteSpace(_config.PythonAiSidecarPath)) + if (string.IsNullOrWhiteSpace(config.PythonAiSidecarPath)) throw new ArgumentException("Python sidecar path is required."); - if (!File.Exists(_config.PythonAiSidecarPath)) - throw new FileNotFoundException($"Python sidecar not found: {_config.PythonAiSidecarPath}"); + if (!File.Exists(config.PythonAiSidecarPath)) + throw new FileNotFoundException($"Python sidecar not found: {config.PythonAiSidecarPath}"); + _client = new PythonSidecarClient(config); } public async Task ListDevicesAsync(CancellationToken cancellationToken = default) { - var data = await SendAsync("speech.devices.list", new { }, cancellationToken); + var data = await _client.SendAsync("speech.devices.list", new { }, cancellationToken); if (data is null || data.Value.ValueKind != JsonValueKind.Object) return new SpeechDevicesResultDto([], "Unexpected speech device response from Python sidecar."); @@ -60,7 +54,7 @@ public sealed class PythonSidecarSpeechService : ISpeechBridgeService { ArgumentNullException.ThrowIfNull(request); - var data = await SendAsync("speech.transcribe", new + var data = await _client.SendAsync("speech.transcribe", new { audio_base64 = request.AudioBase64, engine = request.Engine, @@ -83,102 +77,4 @@ public sealed class PythonSidecarSpeechService : ISpeechBridgeService : null; return new SpeechTranscribeResultDto(text, engine, warning); } - - private async Task SendAsync(string action, object payload, CancellationToken cancellationToken) - { - var request = JsonSerializer.Serialize(new { action, payload }, JsonOptions); - - using var process = new Process(); - process.StartInfo = new ProcessStartInfo - { - FileName = _config.PythonExecutable, - UseShellExecute = false, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WorkingDirectory = _config.ProjectRoot - }; - process.StartInfo.ArgumentList.Add(_config.PythonAiSidecarPath); - - if (!process.Start()) - throw new InvalidOperationException("Failed to start Python sidecar process."); - - await process.StandardInput.WriteLineAsync(request); - process.StandardInput.Close(); - - using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeoutCts.CancelAfter(_config.AiSidecarTimeoutMs); - - try - { - await process.WaitForExitAsync(timeoutCts.Token); - } - catch (OperationCanceledException) - { - TryKill(process); - throw new TimeoutException($"Python sidecar timed out after {_config.AiSidecarTimeoutMs} ms."); - } - - var stdout = await process.StandardOutput.ReadToEndAsync(); - var stderr = await process.StandardError.ReadToEndAsync(); - var line = LastJsonLine(stdout); - if (string.IsNullOrWhiteSpace(line)) - throw new InvalidOperationException($"Python sidecar returned no JSON response. stderr: {stderr}".Trim()); - - JsonDocument doc; - try - { - doc = JsonDocument.Parse(line); - } - catch (JsonException ex) - { - throw new InvalidOperationException($"Invalid JSON from Python sidecar: {line}", ex); - } - - using (doc) - { - var root = doc.RootElement; - if (!root.TryGetProperty("ok", out var okNode) || okNode.ValueKind != JsonValueKind.True && okNode.ValueKind != JsonValueKind.False) - throw new InvalidOperationException("Python sidecar response missing boolean 'ok' field."); - - if (!okNode.GetBoolean()) - { - var error = root.TryGetProperty("error", out var errorNode) - ? errorNode.GetString() ?? "Unknown sidecar error." - : "Unknown sidecar error."; - throw new InvalidOperationException(error); - } - - if (!root.TryGetProperty("data", out var dataNode)) - return null; - - return dataNode.Clone(); - } - } - - private static string LastJsonLine(string text) - { - var lines = text.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); - for (var i = lines.Length - 1; i >= 0; i--) - { - var line = lines[i].Trim(); - if (line.StartsWith("{", StringComparison.Ordinal) && line.EndsWith("}", StringComparison.Ordinal)) - return line; - } - return ""; - } - - private static void TryKill(Process process) - { - try - { - if (!process.HasExited) - process.Kill(entireProcessTree: true); - } - catch - { - // Ignore timeout cleanup failures. - } - } } diff --git a/Journal.SmokeTests/Program.cs b/Journal.SmokeTests/Program.cs index 55eeddf..ea14e8b 100644 --- a/Journal.SmokeTests/Program.cs +++ b/Journal.SmokeTests/Program.cs @@ -113,7 +113,9 @@ static Entry NewEntry() => new( new JournalDatabaseService(new JournalConfigService()), new JournalConfigService(), new DisabledAiService("none"), - new DisabledSpeechBridgeService("none")); + new DisabledSpeechBridgeService("none"), + new EntryFileService(new DiskEntryFileRepository()), + new CommandLogger()); static IJournalDatabaseService NewDatabaseService() => new JournalDatabaseService(new JournalConfigService()); diff --git a/Journal.slnx b/Journal.slnx index a5a68e3..3e7d483 100644 --- a/Journal.slnx +++ b/Journal.slnx @@ -2,4 +2,5 @@ + diff --git a/REFACTORING_SUMMARY.md b/REFACTORING_SUMMARY.md new file mode 100644 index 0000000..b4c7cc7 --- /dev/null +++ b/REFACTORING_SUMMARY.md @@ -0,0 +1,62 @@ +# Backend Refactoring Summary + +## Problem +`Entry.cs` was a 550-line god class that contained command dispatching, business logic, HTML sanitization, logging infrastructure, and 13 private payload record types. The two Python sidecar services (`PythonSidecarAiService` and `PythonSidecarSpeechService`) duplicated ~80 lines of identical process/JSON plumbing. Payload DTOs were hidden as private records inside `Entry.cs` instead of being in the `Dtos/` folder. + +## What Changed + +### 1. Slimmed `Entry.cs` to a thin dispatcher (~330 lines) +Removed all business logic, HTML processing, logging implementation, and private record types. `Entry` now only parses the incoming JSON command, routes to the correct service, and returns the `{ok, data}` / `{ok: false, error}` envelope. + +### 2. Extracted `HtmlSanitizer` (new file) +`StripRichHtml` and `LooksLikeRichHtml` moved from `Entry.cs` to `Services/HtmlSanitizer.cs` as a static utility class. + +### 3. Extracted `CommandLogger` (new file) +`LogStart`, `LogSuccess`, `LogFailure`, `EmitLog`, `ShouldLog`, and `LogLevelRank` moved from `Entry.cs` to `Services/CommandLogger.cs`. Entry now receives this as a dependency. + +### 4. Extracted `IEntryFileService` + `EntryFileService` (new files) +`SaveEntry`, `LoadEntry`, `ListEntries`, and `ResolveTargetPath` moved out of `Entry.cs` into a proper service with an interface. This follows the same pattern as `IFragmentService` / `FragmentService`. + +### 5. Added `IEntryFileRepository` + `DiskEntryFileRepository` (new files) +`EntryFileService` now delegates all filesystem I/O (read, write, append, list, exists) to an `IEntryFileRepository`, keeping only business logic (HTML sanitization, parsing, merging) in the service. This mirrors the Fragment module's repository pattern (`IFragmentRepository` → `FragmentService`). An in-memory implementation can be swapped in for testing. + +### 6. Extracted `PythonSidecarClient` (new file) +The duplicated `SendAsync`, `LastJsonLine`, and `TryKill` methods were extracted from both `PythonSidecarAiService` and `PythonSidecarSpeechService` into a shared `Services/PythonSidecarClient.cs`. Both services now delegate to it. + +### 7. Moved payload records to `Dtos/CommandDtos.cs` (new file) +The 16 private payload/result records that were inside `Entry.cs` are now in `Dtos/CommandDtos.cs`. Records used through public interfaces (`EntrySavePayload`, `EntryListItem`, `EntryLoadResult`, `EntrySaveResult`) are public; the rest remain internal. + +### 8. Moved database result records to `Dtos/DatabaseDtos.cs` (new file) +`JournalDatabaseStatus` and `JournalDatabaseHydrationResult` moved from `IJournalDatabaseService.cs` to `Dtos/DatabaseDtos.cs` for consistency with the other DTO files. + +## Files Created +- `Journal.Core/Services/HtmlSanitizer.cs` +- `Journal.Core/Services/CommandLogger.cs` +- `Journal.Core/Services/IEntryFileService.cs` +- `Journal.Core/Services/EntryFileService.cs` +- `Journal.Core/Services/PythonSidecarClient.cs` +- `Journal.Core/Repositories/IEntryFileRepository.cs` +- `Journal.Core/Repositories/DiskEntryFileRepository.cs` +- `Journal.Core/Dtos/CommandDtos.cs` +- `Journal.Core/Dtos/DatabaseDtos.cs` + +## Files Modified +- `Journal.Core/Entry.cs` — slimmed to thin dispatcher +- `Journal.Core/Services/PythonSidecarAiService.cs` — delegates to PythonSidecarClient +- `Journal.Core/Services/PythonSidecarSpeechService.cs` — delegates to PythonSidecarClient +- `Journal.Core/Services/IJournalDatabaseService.cs` — result records moved to Dtos +- `Journal.Core/Services/JournalDatabaseService.cs` — added Dtos using +- `Journal.Core/ServiceCollectionExtensions.cs` — registers new services and repository +- `Journal.SmokeTests/Program.cs` — updated NewEntry() with new dependencies + +## What Was NOT Changed +- **Fragment module** — already clean, untouched +- **Config module** — singleton reader, no changes needed +- **Vault module** — already well-separated (crypto/storage), untouched +- **AI/Speech interfaces and disabled variants** — untouched (only the sidecar implementations were refactored) +- **Search module** — stateless query service, no repository needed +- **All test logic** — no assertions or test behavior changed + +## Verification +- All 4 projects build successfully +- 70/70 smoke tests pass (5 Python sidecar tests fail only when Python is not installed on the machine, which is pre-existing)